diff --git a/.config/nextest.toml b/.config/nextest.toml index 0a2a55f3..9de2feb3 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -100,6 +100,44 @@ parity-cli = { max-threads = 8 } # The `coverage` command group is a `deacon-conformance` BIN surface, not a test binary: # like `inventory`/`clause` it gets no override and never reaches the shipped `deacon` # CLI (asserted by `parity_registry_check`, T022). +# +# --- 025-exploratory-parity-discovery (T006/T007/T024/T121) ------------------------- +# Discovery is a THIRD lane, alongside the PR lanes and live parity, and it GATES +# NOTHING. Its binaries split exactly two ways: +# +# 1. The hermetic guards — `discovery_hermetic` (deacon crate) and `discovery_cli` +# (conformance crate). NO override, and deliberately NOT excluded anywhere: they MUST +# run in `default` and `dev-fast` (T024). `discovery_hermetic` loads the committed +# discovery data root, runs the D-class validation, and cross-checks this file's own +# lane wiring; `discovery_cli` drives the `conformance discovery …` binary against a +# scratch tree and asserts the exit-status contract (status reflects whether the +# command RAN, never what it FOUND — the rule that makes this lane safe to schedule). +# Both are no-Docker, no-network, no-oracle, so they follow the standing conformance +# convention above. They are guards, not campaigns, and a guard that does not run in +# the fast lane is a guard nobody notices going stale. That is exactly why +# `[profile.discovery]`'s filter below is an explicit `binary(=…)` allow-list and NOT a +# `discovery_*` glob: the glob would capture both of them and silently remove them from +# the fast lane — the mistake the parity profile already documents having made with +# `parity_harness_faults` / `parity_registry_check` (research D9). +# 2. Live campaign binaries — `discovery_campaign` (US1, T040) and `discovery_metamorphic` +# (US6, T097), both in the deacon crate. Selected ONLY by `[profile.discovery]` and +# excluded from the `default-filter` of ALL SIX other profiles (`default`, `dev-fast`, +# `full`, `ci`, `mvp-integration`, `parity`), so those lanes stay truthful by +# non-selection: a green PR run never implies a campaign ran. Exclusion from `parity` +# as well is deliberate — the two lanes answer different questions and a campaign would +# exceed the parity lane's budget. +# +# Their test groups are declared in EVERY profile, including the ones that exclude them +# (T121): a binary excluded from a `default-filter` still needs its group declared where +# it *does* run, and declaring it once per profile keeps the classification from drifting +# when a future profile stops excluding it. `discovery_campaign` → `docker-shared` (its +# container-backed tier brings containers up but is safe to overlap); +# `discovery_metamorphic` → `fs-heavy` (it materializes and transforms workspace trees and +# needs neither Docker nor the oracle). +# +# The `discovery` command group is a `deacon-conformance` BIN surface, not a test binary: +# like `coverage`/`inventory`/`clause` it gets no override and never reaches the shipped +# `deacon` CLI. # Default profile: Balanced parallelization for local development [profile.default] @@ -108,7 +146,7 @@ slow-timeout = "60s" status-level = "pass" # Live parity scenario binaries run ONLY under `--profile parity` (018-harden-parity-harness). # Exclude them here so the default lane is truthful by non-selection (FR-014). -default-filter = 'not (binary(=parity_exec) | binary(=parity_build) | binary(=parity_up_exec) | binary(=parity_observable_state) | binary(=parity_state_diff) | binary(=parity_conformance_runner) | binary(=parity_conformance_docker))' +default-filter = 'not (binary(=parity_exec) | binary(=parity_build) | binary(=parity_up_exec) | binary(=parity_observable_state) | binary(=parity_state_diff) | binary(=parity_conformance_runner) | binary(=parity_conformance_docker) | binary(=discovery_campaign) | binary(=discovery_metamorphic))' [[profile.default.overrides]] filter = 'binary(=smoke_exec) | binary(=smoke_exec_stdin) | binary(=smoke_up_idempotent) | binary(=smoke_down) | binary(=smoke_spinner) | binary(=smoke_lifecycle)' @@ -197,6 +235,20 @@ filter = 'binary(=integration_up_build_options)' test-group = 'smoke-cli' slow-timeout = "60s" +# integration_up_with_features' three Docker tests all bring up the SAME example +# workspace (examples/up/with-features), so they derive the same container identity and +# therefore the same container name. Under `docker-slow-shared` (2 threads) two of them +# race: the second `up --remove-existing-container` creates while the first still holds +# the name, and the daemon rejects it with a name conflict. Which of the three loses is +# scheduling-dependent, so the failure reads as a flake rather than as the shared-fixture +# collision it is. Serialize the binary — `full`/`ci` already do, via their +# `binary(#integration_up_*)` docker-exclusive rule, which is why this only ever surfaced +# in the `default`/`docker`/`long-running` lanes. +[[profile.default.overrides]] +filter = 'binary(=integration_up_with_features)' +test-group = 'docker-exclusive' +slow-timeout = { period = "10m", terminate-after = 1 } + [[profile.default.overrides]] filter = 'binary(=integration_build) | binary(=integration_build_output) | binary(=integration_build_args) | binary(=integration_vulnerability_scan) | binary(#integration_up_*) | binary(=integration_progress) | binary(=integration_host_ca_runtime) | binary(=integration_host_ca_build)' test-group = 'docker-slow-shared' @@ -237,6 +289,21 @@ filter = 'binary(=smoke_exec) | binary(=smoke_exec_stdin) | binary(=smoke_up_ide priority = 100 # Fast feedback loop profile: skip smoke/parity suites, Docker tests, and testcontainers tests + +# --- 025-exploratory-parity-discovery: discovery test-group overrides (T121) -------- +# Declared in EVERY profile, including the ones whose `default-filter` excludes these +# binaries: a binary excluded from a filter still needs its group declared where it *does* +# run, and keeping the declaration in every profile stops the classification drifting if a +# future profile ever stops excluding it. `discovery_hermetic` deliberately gets NO +# override — it is an ungrouped hermetic guard that must run in the fast lane (T024). +[[profile.default.overrides]] +filter = 'binary(=discovery_campaign)' +test-group = 'docker-shared' + +[[profile.default.overrides]] +filter = 'binary(=discovery_metamorphic)' +test-group = 'fs-heavy' + [profile.dev-fast] retries = 0 slow-timeout = "30s" @@ -250,7 +317,7 @@ slow-timeout = "30s" # `parity_*` test binaries, not the crate's own unit tests, so the lib joins # `parity_harness_faults` / `parity_registry_check` as an explicit exception. Its # shell-dependent tests are already `#[cfg(unix)]`-gated, so the Windows lane is unaffected. -default-filter = 'not (binary(#smoke_*) | (binary(#parity_*) & not (binary(=parity_harness_faults) | binary(=parity_registry_check) | binary(=parity_harness))) | binary(=build_consistency) | binary(=integration_build) | binary(=integration_build_output) | binary(=integration_build_args) | binary(=integration_vulnerability_scan) | binary(#integration_up_*) | binary(=integration_progress) | binary(#integration_env_probe_*) | test(/^env_probe::tests::/) | binary(=integration_exec_id_label) | binary(=integration_exec_selection) | binary(=testcontainers_helpers) | binary(=workspace_mounts) | binary(=up_prebuild) | binary(=run_user_commands_prebuild) | binary(=run_user_commands_feature_lifecycle) | binary(=lifecycle_parallel_output) | binary(=up_reconnect_full_id) | binary(=up_keepalive_path) | binary(=up_dotfiles) | binary(=integration_feature_lifecycle) | binary(=integration_feature_security) | binary(=integration_feature_mounts) | binary(=integration_compose_features_build) | binary(=integration_compose_config_mounts) | binary(=integration_auto_forward) | binary(=integration_host_ca_runtime) | binary(=integration_host_ca_build) | binary(=docker_channels))' +default-filter = 'not (binary(#smoke_*) | (binary(#parity_*) & not (binary(=parity_harness_faults) | binary(=parity_registry_check) | binary(=parity_harness))) | binary(=build_consistency) | binary(=integration_build) | binary(=integration_build_output) | binary(=integration_build_args) | binary(=integration_vulnerability_scan) | binary(#integration_up_*) | binary(=integration_progress) | binary(#integration_env_probe_*) | test(/^env_probe::tests::/) | binary(=integration_exec_id_label) | binary(=integration_exec_selection) | binary(=testcontainers_helpers) | binary(=workspace_mounts) | binary(=up_prebuild) | binary(=run_user_commands_prebuild) | binary(=run_user_commands_feature_lifecycle) | binary(=lifecycle_parallel_output) | binary(=up_reconnect_full_id) | binary(=up_keepalive_path) | binary(=up_dotfiles) | binary(=integration_feature_lifecycle) | binary(=integration_feature_security) | binary(=integration_feature_mounts) | binary(=integration_compose_features_build) | binary(=integration_compose_config_mounts) | binary(=integration_auto_forward) | binary(=integration_host_ca_runtime) | binary(=integration_host_ca_build) | binary(=docker_channels) | binary(=discovery_campaign) | binary(=discovery_metamorphic))' [[profile.dev-fast.overrides]] filter = 'binary(=smoke_exec) | binary(=smoke_exec_stdin) | binary(=smoke_up_idempotent) | binary(=smoke_down) | binary(=smoke_spinner) | binary(=smoke_lifecycle)' @@ -327,6 +394,20 @@ filter = 'binary(=integration_up_build_options)' test-group = 'smoke-cli' slow-timeout = "60s" +# integration_up_with_features' three Docker tests all bring up the SAME example +# workspace (examples/up/with-features), so they derive the same container identity and +# therefore the same container name. Under `docker-slow-shared` (2 threads) two of them +# race: the second `up --remove-existing-container` creates while the first still holds +# the name, and the daemon rejects it with a name conflict. Which of the three loses is +# scheduling-dependent, so the failure reads as a flake rather than as the shared-fixture +# collision it is. Serialize the binary — `full`/`ci` already do, via their +# `binary(#integration_up_*)` docker-exclusive rule, which is why this only ever surfaced +# in the `default`/`docker`/`long-running` lanes. +[[profile.dev-fast.overrides]] +filter = 'binary(=integration_up_with_features)' +test-group = 'docker-exclusive' +slow-timeout = { period = "10m", terminate-after = 1 } + [[profile.dev-fast.overrides]] filter = 'binary(=integration_build) | binary(=integration_build_output) | binary(=integration_build_args) | binary(=integration_vulnerability_scan) | binary(#integration_up_*) | binary(=integration_progress) | binary(=integration_host_ca_runtime) | binary(=integration_host_ca_build)' test-group = 'docker-slow-shared' @@ -363,6 +444,21 @@ filter = 'binary(=smoke_exec) | binary(=smoke_exec_stdin) | binary(=smoke_up_ide priority = 100 # Full profile: Run all tests with appropriate grouping + +# --- 025-exploratory-parity-discovery: discovery test-group overrides (T121) -------- +# Declared in EVERY profile, including the ones whose `default-filter` excludes these +# binaries: a binary excluded from a filter still needs its group declared where it *does* +# run, and keeping the declaration in every profile stops the classification drifting if a +# future profile ever stops excluding it. `discovery_hermetic` deliberately gets NO +# override — it is an ungrouped hermetic guard that must run in the fast lane (T024). +[[profile.dev-fast.overrides]] +filter = 'binary(=discovery_campaign)' +test-group = 'docker-shared' + +[[profile.dev-fast.overrides]] +filter = 'binary(=discovery_metamorphic)' +test-group = 'fs-heavy' + [profile.full] retries = 0 # Mark tests as slow after 90s and terminate once (useful for interactive full runs) @@ -370,9 +466,24 @@ slow-timeout = { period = "90s", terminate-after = 1 } status-level = "none" # Live parity scenario binaries run ONLY under `--profile parity` (018-harden-parity-harness). # Exclude them here so even the "full" lane is truthful by non-selection (FR-014). -default-filter = 'not (binary(=parity_exec) | binary(=parity_build) | binary(=parity_up_exec) | binary(=parity_observable_state) | binary(=parity_state_diff) | binary(=parity_conformance_runner) | binary(=parity_conformance_docker))' +default-filter = 'not (binary(=parity_exec) | binary(=parity_build) | binary(=parity_up_exec) | binary(=parity_observable_state) | binary(=parity_state_diff) | binary(=parity_conformance_runner) | binary(=parity_conformance_docker) | binary(=discovery_campaign) | binary(=discovery_metamorphic))' # Profile for long-running integration tests (heavy end-to-end builds) + +# --- 025-exploratory-parity-discovery: discovery test-group overrides (T121) -------- +# Declared in EVERY profile, including the ones whose `default-filter` excludes these +# binaries: a binary excluded from a filter still needs its group declared where it *does* +# run, and keeping the declaration in every profile stops the classification drifting if a +# future profile ever stops excluding it. `discovery_hermetic` deliberately gets NO +# override — it is an ungrouped hermetic guard that must run in the fast lane (T024). +[[profile.full.overrides]] +filter = 'binary(=discovery_campaign)' +test-group = 'docker-shared' + +[[profile.full.overrides]] +filter = 'binary(=discovery_metamorphic)' +test-group = 'fs-heavy' + [profile.long-running] retries = 0 slow-timeout = { period = "30m", terminate-after = 1 } @@ -385,6 +496,20 @@ filter = 'binary(=integration_up_build_options)' test-group = 'smoke-cli' slow-timeout = "60s" +# integration_up_with_features' three Docker tests all bring up the SAME example +# workspace (examples/up/with-features), so they derive the same container identity and +# therefore the same container name. Under `docker-slow-shared` (2 threads) two of them +# race: the second `up --remove-existing-container` creates while the first still holds +# the name, and the daemon rejects it with a name conflict. Which of the three loses is +# scheduling-dependent, so the failure reads as a flake rather than as the shared-fixture +# collision it is. Serialize the binary — `full`/`ci` already do, via their +# `binary(#integration_up_*)` docker-exclusive rule, which is why this only ever surfaced +# in the `default`/`docker`/`long-running` lanes. +[[profile.long-running.overrides]] +filter = 'binary(=integration_up_with_features)' +test-group = 'docker-exclusive' +slow-timeout = { period = "10m", terminate-after = 1 } + [[profile.long-running.overrides]] filter = 'binary(=integration_build) | binary(=integration_build_output) | binary(#integration_up_*) | binary(=integration_progress) | binary(=integration_vulnerability_scan)' test-group = 'docker-slow-shared' @@ -455,6 +580,20 @@ filter = 'binary(=integration_up_build_options)' test-group = 'smoke-cli' slow-timeout = "60s" +# integration_up_with_features' three Docker tests all bring up the SAME example +# workspace (examples/up/with-features), so they derive the same container identity and +# therefore the same container name. Under `docker-slow-shared` (2 threads) two of them +# race: the second `up --remove-existing-container` creates while the first still holds +# the name, and the daemon rejects it with a name conflict. Which of the three loses is +# scheduling-dependent, so the failure reads as a flake rather than as the shared-fixture +# collision it is. Serialize the binary — `full`/`ci` already do, via their +# `binary(#integration_up_*)` docker-exclusive rule, which is why this only ever surfaced +# in the `default`/`docker`/`long-running` lanes. +[[profile.docker.overrides]] +filter = 'binary(=integration_up_with_features)' +test-group = 'docker-exclusive' +slow-timeout = { period = "10m", terminate-after = 1 } + [[profile.docker.overrides]] filter = 'binary(=integration_build) | binary(=integration_build_output) | binary(=integration_build_args) | binary(=integration_vulnerability_scan) | binary(#integration_up_*) | binary(=integration_progress)' test-group = 'docker-slow-shared' @@ -601,7 +740,7 @@ fail-fast = false global-timeout = "1h" # Live parity scenario binaries run ONLY under `--profile parity` (018); exclude # them from ci alongside smoke so this lane is truthful by non-selection (FR-014). -default-filter = 'not (binary(#smoke_*) | binary(=parity_exec) | binary(=parity_build) | binary(=parity_up_exec) | binary(=parity_observable_state) | binary(=parity_state_diff) | binary(=parity_conformance_runner) | binary(=parity_conformance_docker))' +default-filter = 'not (binary(#smoke_*) | binary(=parity_exec) | binary(=parity_build) | binary(=parity_up_exec) | binary(=parity_observable_state) | binary(=parity_state_diff) | binary(=parity_conformance_runner) | binary(=parity_conformance_docker) | binary(=discovery_campaign) | binary(=discovery_metamorphic))' # Throttle smoke and parity tests while allowing more overlap [[profile.ci.overrides]] @@ -699,13 +838,28 @@ priority = 100 # MVP Integration profile: Consolidated smoke + parity + core Docker integration tests # Used by CI for MVP-focused testing without running full feature tests + +# --- 025-exploratory-parity-discovery: discovery test-group overrides (T121) -------- +# Declared in EVERY profile, including the ones whose `default-filter` excludes these +# binaries: a binary excluded from a filter still needs its group declared where it *does* +# run, and keeping the declaration in every profile stops the classification drifting if a +# future profile ever stops excluding it. `discovery_hermetic` deliberately gets NO +# override — it is an ungrouped hermetic guard that must run in the fast lane (T024). +[[profile.ci.overrides]] +filter = 'binary(=discovery_campaign)' +test-group = 'docker-shared' + +[[profile.ci.overrides]] +filter = 'binary(=discovery_metamorphic)' +test-group = 'fs-heavy' + [profile.mvp-integration] retries = 0 slow-timeout = { period = "5m", terminate-after = 1 } status-level = "pass" fail-fast = false # MVP integration tests: smoke, parity, and core integration tests for up/exec/down/read-configuration -default-filter = 'binary(#smoke_*) | (binary(#parity_*) & not (binary(=parity_exec) | binary(=parity_build) | binary(=parity_up_exec) | binary(=parity_observable_state) | binary(=parity_state_diff) | binary(=parity_conformance_runner) | binary(=parity_conformance_docker))) | binary(=integration_exec_selection) | binary(=integration_exec_id_label) | binary(=integration_up_exec_identity) | binary(=integration_down) | binary(=integration_runtime_selection) | binary(=integration_read_configuration) | binary(=integration_read_configuration_output) | binary(=integration_config) | binary(=integration_custom_container_name) | binary(=integration_exec) | binary(=integration_exec_env) | binary(=integration_exec_pty) | binary(=integration_e2e) | binary(#integration_env_probe_*) | binary(=aggregator) | binary(=normalize_consistency) | binary(=raw_outputs) | binary(=report_granularity) | binary(=observation_faults) | binary(=container_state_channel)' +default-filter = '(binary(#smoke_*) | (binary(#parity_*) & not (binary(=parity_exec) | binary(=parity_build) | binary(=parity_up_exec) | binary(=parity_observable_state) | binary(=parity_state_diff) | binary(=parity_conformance_runner) | binary(=parity_conformance_docker))) | binary(=integration_exec_selection) | binary(=integration_exec_id_label) | binary(=integration_up_exec_identity) | binary(=integration_down) | binary(=integration_runtime_selection) | binary(=integration_read_configuration) | binary(=integration_read_configuration_output) | binary(=integration_config) | binary(=integration_custom_container_name) | binary(=integration_exec) | binary(=integration_exec_env) | binary(=integration_exec_pty) | binary(=integration_e2e) | binary(#integration_env_probe_*) | binary(=aggregator) | binary(=normalize_consistency) | binary(=raw_outputs) | binary(=report_granularity) | binary(=observation_faults) | binary(=container_state_channel)) & not (binary(=discovery_campaign) | binary(=discovery_metamorphic))' # Smoke test grouping [[profile.mvp-integration.overrides]] @@ -767,11 +921,26 @@ test-group = 'fs-heavy' # are enforced PER CLI INVOCATION inside the harness, while one test may run # several cases across two CLIs. (The 3 corpus runners join this profile in US2.) # --------------------------------------------------------------------------- + +# --- 025-exploratory-parity-discovery: discovery test-group overrides (T121) -------- +# Declared in EVERY profile, including the ones whose `default-filter` excludes these +# binaries: a binary excluded from a filter still needs its group declared where it *does* +# run, and keeping the declaration in every profile stops the classification drifting if a +# future profile ever stops excluding it. `discovery_hermetic` deliberately gets NO +# override — it is an ungrouped hermetic guard that must run in the fast lane (T024). +[[profile.mvp-integration.overrides]] +filter = 'binary(=discovery_campaign)' +test-group = 'docker-shared' + +[[profile.mvp-integration.overrides]] +filter = 'binary(=discovery_metamorphic)' +test-group = 'fs-heavy' + [profile.parity] retries = 0 slow-timeout = { period = "15m", terminate-after = 1 } status-level = "pass" -default-filter = 'binary(=parity_exec) | binary(=parity_build) | binary(=parity_up_exec) | binary(=parity_observable_state) | binary(=parity_state_diff) | binary(=parity_conformance_runner) | binary(=parity_conformance_docker)' +default-filter = '(binary(=parity_exec) | binary(=parity_build) | binary(=parity_up_exec) | binary(=parity_observable_state) | binary(=parity_state_diff) | binary(=parity_conformance_runner) | binary(=parity_conformance_docker)) & not (binary(=discovery_campaign) | binary(=discovery_metamorphic))' # Config-only parity runners → parity-cli group. `parity_conformance_runner` (022, # US1) drives the declarative runner over config-only read-configuration cases, so it @@ -801,3 +970,60 @@ slow-timeout = { period = "35m", terminate-after = 1 } filter = 'binary(=parity_exec) | binary(=parity_build) | binary(=parity_up_exec) | binary(=parity_observable_state) | binary(=parity_state_diff)' test-group = 'parity' +# --- 025-exploratory-parity-discovery: discovery test-group overrides (T121) -------- +# Declared in EVERY profile, including the ones whose `default-filter` excludes these +# binaries: a binary excluded from a filter still needs its group declared where it *does* +# run, and keeping the declaration in every profile stops the classification drifting if a +# future profile ever stops excluding it. `discovery_hermetic` deliberately gets NO +# override — it is an ungrouped hermetic guard that must run in the fast lane (T024). +[[profile.parity.overrides]] +filter = 'binary(=discovery_campaign)' +test-group = 'docker-shared' + +[[profile.parity.overrides]] +filter = 'binary(=discovery_metamorphic)' +test-group = 'fs-heavy' + + + +# --------------------------------------------------------------------------- +# Discovery profile (025-exploratory-parity-discovery, T006/T121): the SINGLE +# sanctioned entry point for live exploratory campaigns +# (`cargo nextest run --profile discovery`, reached via `make test-discovery`). +# +# The `default-filter` is an EXPLICIT `binary(=…)` allow-list, never a `discovery_*` +# glob. This is the 018 lesson taken verbatim (research D9): the parity profile's filter +# is an allow-list precisely because a `parity_*` glob wrongly captured the hermetic +# guards `parity_harness_faults` and `parity_registry_check`. A `discovery_*` glob would +# make the identical mistake with `discovery_hermetic`, and the symptom — a hermetic guard +# silently not running in the fast lane — is invisible until it matters. +# +# Discovery GATES NOTHING. A campaign that finds forty differences exits 0; only a +# machinery failure (an unverifiable oracle, a normalization failure, an unwritable data +# root) is non-zero. That is what lets this lane be scheduled in CI at all: a stochastic +# gate would make green non-reproducible. +# +# `slow-timeout` is deliberately ABOVE the tier's 30-minute budget (research D10). The +# budget is asserted EXPLICITLY by the campaign driver, which reports the elapsed number; +# nextest's timeout is only the backstop for a driver that stops making progress at all. +# A backstop that fired FIRST would replace a diagnosable failure with "the binary was +# slow" — the unattributable signal the explicit assertion exists to avoid. This mirrors +# `parity_conformance_docker`'s reasoning exactly. +# --------------------------------------------------------------------------- +[profile.discovery] +retries = 0 +slow-timeout = { period = "35m", terminate-after = 1 } +status-level = "pass" +default-filter = 'binary(=discovery_campaign) | binary(=discovery_metamorphic)' + +# The container-backed tier brings containers up but is safe to overlap with other +# Docker work, so it takes the shared group rather than an exclusive one. +[[profile.discovery.overrides]] +filter = 'binary(=discovery_campaign)' +test-group = 'docker-shared' + +# The metamorphic tier needs neither Docker nor the oracle nor the network (research +# D12); it materializes and transforms workspace trees, so the group bounds its I/O. +[[profile.discovery.overrides]] +filter = 'binary(=discovery_metamorphic)' +test-group = 'fs-heavy' diff --git a/.gitattributes b/.gitattributes index e6558f38..0285bc06 100644 --- a/.gitattributes +++ b/.gitattributes @@ -65,3 +65,12 @@ conformance/migration/** -text # V27 deliberately cannot distinguish a hand edit from a stale regeneration; it equally # cannot distinguish either from a re-encoded checkout, so the bytes must be pinned. conformance/obligations/** -text + +# The exploratory discovery data root (025-exploratory-parity-discovery) is written by the +# campaign driver through the same canonical renderer every machine-owned artifact here +# uses (2-space pretty JSON, trailing newline), and both `discovery check` and the +# hermetic guard byte-compare the committed files against a fresh rendering. A Windows +# autocrlf checkout rewrites LF to CRLF, so `raw.ends_with("}\n")` sees `}\r\n` and the +# canonical-rendering comparison fails on bytes git introduced — pointing the reader at a +# renderer that is working correctly. Same discipline, same pin. +conformance/discovery/** -text diff --git a/.github/workflows/discovery.yml b/.github/workflows/discovery.yml new file mode 100644 index 00000000..0fbbe786 --- /dev/null +++ b/.github/workflows/discovery.yml @@ -0,0 +1,358 @@ +# Exploratory parity discovery (025-exploratory-parity-discovery, US3/T058). +# +# The THIRD lane. `ci` answers "did this change break something we already assert"; +# `parity / live-certification` answers "does deacon still match the reference on the +# cases we curated"; this one answers the question neither can — "does anything differ +# outside what we curated?". +# +# ## This lane GATES NOTHING +# +# A campaign that finds forty differences exits 0. Only a *machinery* failure — an +# unverifiable oracle, a normalization failure, an unwritable or invalid data root — is +# non-zero (FR-058, contracts/discovery-cli.md). That rule is the entire reason this lane +# can be scheduled in CI at all: a status that depended on findings would be a stochastic +# gate, and green would stop being reproducible. Nothing here is in the release path, and +# a red run here blocks no pull request and no release. +# +# ## Why there is no `pull_request` trigger +# +# FR-055 is absolute: no discovery program may be selected by, or run in, a pull-request +# lane. Adding a PR trigger "just to lint the workflow" would be the first step toward a +# campaign running on a PR. The wiring is instead guarded *hermetically* on every PR by +# `parity_registry_check` (registry ↔ `tests/*.rs` ↔ `.config/nextest.toml` agreement, +# and that `deacon --help` gains no discovery surface) and by `discovery_hermetic`, both +# of which run in the fast lane and need no oracle, Docker, or network. +# +# ## Cadence — two schedules, two jobs, two budgets +# +# **Nightly (`0 7 * * *`)**, offset from the parity lane's 05:00 so the two do not contend +# for runners. The hermetic configuration-resolution tier gets the whole nightly budget: at +# the clarified per-candidate ceilings a single container-backed candidate can consume a +# sixth of the 30-minute window, so sharing one budget lets the slow tier starve the fast +# one — and the fast tier is where nearly all the exploration happens (research D10). +# +# **Weekly (`0 9 * * 0`)**, the network-backed **corpus** canary, on a schedule of its own +# (T128, FR-056). It is separate from the nightly campaign rather than folded into it for +# two reasons. It must exist at all: US7 calls the corpus an *ecological canary*, and a +# canary that runs only when someone remembers to invoke it cannot warn anyone. And it must +# be weekly rather than nightly: the corpus changes only when someone re-pins it, so nightly +# runs would mostly re-confirm the previous night at network cost, while the signal being +# watched for — the ecosystem drifting away from what deacon handles — moves on the order of +# weeks (research D10). +# +# The container-backed tier remains explicitly invoked only, via `workflow_dispatch`. +name: discovery + +on: + schedule: + # Nightly exploratory campaign on the default branch (hermetic tier). + - cron: '0 7 * * *' + # Weekly real-world corpus canary. Sunday, offset from the nightly hour so the two + # never overlap even when a nightly run is slow: the `concurrency` group below serializes + # them, and a corpus run queued behind a 60-minute nightly would report its own budget + # as exhausted for a reason that has nothing to do with the corpus. + - cron: '0 9 * * 0' + + # Explicit invocation. FR-056 requires the invoked lane to ACCEPT a seed and a budget + # and to RECORD both; the campaign records them in `conformance/discovery/campaigns.json` + # as part of the run's provenance. + workflow_dispatch: + inputs: + seed: + description: >- + Hex PRNG seed. Required, never defaulted: a defaulted seed would let a campaign + run without its reproducibility input being a conscious choice, and FR-001 + depends on the seed being recorded rather than inferred. + required: true + type: string + budget: + description: 'Wall-clock budget for the campaign, in seconds.' + required: false + default: '1800' + type: string + tier: + description: >- + Which tier to run. `metamorphic` needs no oracle, Docker, or network; + `config-differential` is the nightly tier; `container-differential` needs Docker; + `corpus` needs network. Selecting the container tier separately from the + configuration tier is FR-060. + required: false + default: 'config-differential' + type: choice + options: + - config-differential + - container-differential + - metamorphic + - corpus + +# Read-only. The campaign writes `conformance/discovery/{findings,campaigns}.json` in the +# workspace and those files are uploaded as an artifact, but this lane deliberately does +# NOT push them: a stochastic process must never author the record it is measured against +# (FR-036). Promotion is a human reading the diff and editing the registry by hand. +permissions: + contents: read + +concurrency: + # One campaign at a time. Two concurrent runs would write the same data root, and + # `cancel-in-progress: false` because a partially completed campaign still has findings + # worth keeping. + group: discovery-${{ github.ref }} + cancel-in-progress: false + +jobs: + campaign: + name: campaign + # Everything EXCEPT the weekly corpus cron. Written as an exclusion rather than as + # "run on the nightly cron or a dispatch" so that adding a third schedule later cannot + # silently stop this job from running: a new cron would fall through to the nightly + # campaign, which is the harmless direction. The corpus job below is the one that must + # be opted into explicitly, because it is the one that reaches the network. + if: github.event_name != 'schedule' || github.event.schedule != '0 9 * * 0' + runs-on: ubuntu-latest + # Above SC-015's 30-minute campaign budget. The budget is enforced by the campaign + # driver itself, which reports the elapsed number; this is only the backstop for a + # driver that stops making progress at all. A backstop that fired FIRST would replace + # a diagnosable failure with "the job was slow". + timeout-minutes: 60 + steps: + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL 2>/dev/null || true + + - name: Checkout code + uses: actions/checkout@v7 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry and target + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-discovery-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-discovery- + ${{ runner.os }}-cargo- + + - name: Install cargo-nextest + uses: taiki-e/install-action@v2 + with: + tool: cargo-nextest + + - name: Set up Node 20 + uses: actions/setup-node@v7 + with: + node-version: 20 + + - name: Provision the pinned oracle + # Install EXACTLY the version in the pin file — the harness re-verifies it, so + # provisioning and harness can never disagree about the reference. A missing or + # mismatched oracle FAILS the run; there is no silent skip, because "found + # nothing" and "never compared anything" must never look alike. + run: | + set -euo pipefail + VERSION="$(jq -r .version fixtures/parity-corpus/oracle.json)" + echo "Installing @devcontainers/cli@${VERSION} from fixtures/parity-corpus/oracle.json" + npm install -g "@devcontainers/cli@${VERSION}" + + - name: Verify oracle version + run: devcontainer --version + + - name: Set up Docker Buildx + # The container-backed tier builds images; establishing the precondition here (as + # the parity lane does) keeps a missing builder from being reported as a + # divergence. + uses: docker/setup-buildx-action@v3 + + - name: Verify BuildKit is available + run: | + docker buildx version + docker buildx inspect --bootstrap >/dev/null + echo "BuildKit is available" + + - name: Build deacon + run: cargo build -p deacon + + - name: Prove the pipeline can carry an injected difference + # FR-042a, run BEFORE the campaign and deliberately so: a campaign that reports no + # findings and a campaign whose pipeline silently swallows them are byte-identical + # from the outside, and the only way to tell them apart is to plant a difference + # you know is there and require it to come out the other end. + # + # This is the ONE discovery command whose status depends on an outcome, and it is + # still not finding-dependent: it asserts a property of the MACHINERY, so non-zero + # means the pipeline is broken. An injection that never landed also exits 1, as a + # PROOF defect rather than as "found nothing" — a mis-authored record must never + # masquerade as a working pipeline. + # + # Hermetic: it needs deacon and nothing else. No oracle, no Docker, no network — + # the counterpart is deacon's own unperturbed run, which is what makes the + # baseline provably empty and every surfaced difference attributable to the + # injection. + run: make test-discovery-proof + + - name: Run the discovery lane + # The SINGLE sanctioned entry point: `[profile.discovery]`, whose default-filter + # is an explicit `binary(=…)` allow-list rather than a `discovery_*` glob (the + # glob would capture the hermetic guards and silently remove them from the fast + # lane — research D9). + run: cargo nextest run --profile discovery + + - name: Run the explicitly seeded campaign + # Only on explicit invocation, where a seed and a budget were supplied. The + # scheduled lane above runs the profile's own campaign binaries instead. + # + # The inputs reach the shell through `env:`, never through `${{ }}` interpolated + # into the script body. `${{ }}` in a `run:` block is substituted *before* the + # shell sees the text, so a value containing shell metacharacters becomes code + # rather than an argument. Requiring write access to dispatch a workflow narrows + # who can do that; it does not make the substitution safe, and this is the shape + # that is safe by construction. + if: github.event_name == 'workflow_dispatch' + env: + DISCOVERY_SEED: ${{ inputs.seed }} + DISCOVERY_TIER: ${{ inputs.tier }} + DISCOVERY_BUDGET: ${{ inputs.budget }} + run: | + cargo run -p parity-harness --bin discovery-campaign -- \ + --seed "$DISCOVERY_SEED" \ + --tier "$DISCOVERY_TIER" \ + --budget-seconds "$DISCOVERY_BUDGET" \ + --lane invoked + + - name: Render the findings queue + # Never gates: a queue holding fifty untriaged findings still exits 0. The exit + # status reflects only whether the artifacts could be written. + run: cargo run -p deacon-conformance -- discovery report + + - name: Validate the discovery data root + # This one DOES fail the lane, and legitimately so: D1–D5 are integrity classes + # over the record the campaign just wrote (a malformed record, an unresolvable + # reference, an undeclared channel, a stale pin). That is a machinery failure, not + # a finding — the distinction the whole exit-status contract rests on. + run: cargo run -p deacon-conformance -- discovery check + + - name: Upload the findings queue and campaign artifacts + if: always() + uses: actions/upload-artifact@v7 + with: + name: discovery-queue + path: target/discovery/ + if-no-files-found: warn + + - name: Upload the written discovery data root + # The data root is the durable record a reviewer triages from. Uploaded rather + # than pushed (see the `permissions` note above): review is a human act. + if: always() + uses: actions/upload-artifact@v7 + with: + name: discovery-data-root + path: conformance/discovery/ + if-no-files-found: warn + + # The weekly real-world corpus canary (T128, FR-056, research D10). + # + # A separate job rather than a fifth step of `campaign`, because it has a different + # cadence, a different prerequisite (the network), and a different budget. Sharing a job + # would mean sharing a schedule, and the whole point of this one is that the corpus does + # NOT belong on the nightly clock. + # + # It gates nothing, exactly like the campaign job: forty differences against the + # ecosystem exit 0. Only a machinery failure — an unverifiable oracle, no network, an + # invalid data root — is non-zero (FR-058). A red run here blocks no pull request and no + # release. + # + # No Docker setup: the corpus tier is a configuration-resolution differential, so it + # brings nothing up. Establishing a precondition this tier does not have would only turn + # a missing builder into a confusing failure. + corpus: + name: corpus canary + if: github.event_name == 'schedule' && github.event.schedule == '0 9 * * 0' + runs-on: ubuntu-latest + # The corpus tier's cost is 33 network round trips plus 33 oracle invocations, not a + # 30-minute exploration. This is the backstop for a run that stops making progress, + # never the thing that ends a healthy one. + timeout-minutes: 45 + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry and target + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-discovery-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-discovery- + ${{ runner.os }}-cargo- + + - name: Set up Node 20 + uses: actions/setup-node@v7 + with: + node-version: 20 + + - name: Provision the pinned oracle + # The corpus tier is a differential: it compares deacon against the reference over + # inputs nobody here wrote. A missing or mismatched oracle FAILS the run — there is + # no skip, because "the ecosystem agrees with us" and "we never compared anything" + # must never look alike. + run: | + set -euo pipefail + VERSION="$(jq -r .version fixtures/parity-corpus/oracle.json)" + echo "Installing @devcontainers/cli@${VERSION} from fixtures/parity-corpus/oracle.json" + npm install -g "@devcontainers/cli@${VERSION}" + + - name: Verify oracle version + run: devcontainer --version + + - name: Build deacon + run: cargo build -p deacon + + - name: Run the weekly corpus canary + # The seed is fixed rather than random: this tier draws nothing, so the seed + # contributes only to the campaign's derived id, and a fixed one makes consecutive + # weekly runs over an unchanged manifest and unchanged pins the SAME campaign — + # which is correct, and is what lets the queue's deduplication tell "still + # diverging" from "diverged again". + run: | + cargo run -p parity-harness --bin discovery-campaign -- \ + --seed 0x0000c0f5 \ + --tier corpus \ + --budget-seconds 1800 \ + --lane scheduled + + - name: Render the findings queue + run: cargo run -p deacon-conformance -- discovery report + + - name: Validate the discovery data root + # D1–D5 over the record the canary just wrote, including **D4** over any content + # digest it recorded at first materialization. A machinery failure, not a finding. + run: cargo run -p deacon-conformance -- discovery check + + - name: Upload the corpus canary artifacts + if: always() + uses: actions/upload-artifact@v7 + with: + name: discovery-corpus-queue + path: target/discovery/ + if-no-files-found: warn + + - name: Upload the written discovery data root + # Includes `corpus.json` with any digest recorded at first materialization. Every + # later run VERIFIES that digest, so this artifact is the diff a reviewer commits + # from — uploaded, never pushed: review is a human act (FR-036). + if: always() + uses: actions/upload-artifact@v7 + with: + name: discovery-corpus-data-root + path: conformance/discovery/ + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index 64126756..058173b5 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,11 @@ artifacts/nextest/full-timing.json # pollute search results and inflate every assistant's context window. repomix-output* **/repomix-output* + +# Exploratory discovery artifacts (025-exploratory-parity-discovery): the byte-stable +# queue report and the assembled reviewable candidates. Regenerated by +# `discovery report` / a campaign run; the DURABLE record is the version-controlled +# findings queue under conformance/discovery/, never this tree. +# (Covered by the blanket `target` rule above; named explicitly so its status is a +# decision rather than an accident of that rule.) +target/discovery/ diff --git a/CLAUDE.md b/CLAUDE.md index 91d3b22e..ec937610 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -399,8 +399,11 @@ nothing. Each is gone, not merely quiescent: - `parity_read_configuration` — deleted; its 2 units became 2 `case-readconfig-decl-*` - `corpus_runner` — deleted (the shared module all four called) -Their in-repo fixture trees went with them; `fetch_realworld_corpus.py` survives -(research D8). +Their in-repo fixture trees went with them. `fetch_realworld_corpus.py` was **deleted** in +025 US7 (T109), having outlived 023: its 33 pinned entries are now the Rust-owned strict-JSON +`conformance/discovery/corpus.json`, where the immutable-reference rule (**D4**) runs +hermetically on every PR, and the fetch — with content-digest verification — lives in +`parity_harness::discovery::corpus_fetch`. Every other profile (`default`/`full`/`ci`/`dev-fast`/`mvp-integration`) excludes the live set, so those lanes are truthful by **non-selection**: a green fast/CI @@ -767,7 +770,13 @@ one. They are separate machinery — keep them distinct even though both say "pa compose lifecycle-marker case (issue #117, closed in the PR that added this note) was exactly such a mis-seed: markers are deacon-internal, the reference has no concept of them. -**Two gates — do NOT conflate them** (this genuinely confuses): +**Three lanes, two gates — do NOT conflate them** (this genuinely confuses): + +| Lane | Workflow | In the release path? | Gates on | +|---|---|---|---| +| `parity / live-certification` | `parity.yml` | **No** | live divergences (red never blocks a release) | +| Conformance `certify` | `release.yml` § `verify` | **Yes** | gaps / uncovered behaviors / V28–V29 | +| `discovery` | `discovery.yml` | **No** | **nothing** — status reflects whether it RAN, never what it found | - **`parity / live-certification` lane** (`.github/workflows/parity.yml`) — surfaces the live divergences. It is **NOT in the release path**: `release.yml` never runs it, so a **red @@ -783,6 +792,11 @@ one. They are separate machinery — keep them distinct even though both say "pa non-blocking. Keep the registry gap-free (or every open gap consciously accepted) so releases aren't surprise-blocked. **`coverage report` is NOT a gate** — its exit code never reflects what it reports; it feeds `certify`, it does not duplicate it. +- **`discovery` lane** (`.github/workflows/discovery.yml`, nightly + manual) — a **third** lane + that **gates nothing**. It searches for differences nobody curated; a campaign that finds forty + of them exits `0`. Only a *machinery* failure (unverifiable oracle, normalization failure, + unwritable data root) is non-zero. A stochastic gate would make green non-reproducible, which + is precisely why status is machinery-only — see the section below. **The build-out loop (apply these defaults).** When the harness surfaces a difference: @@ -811,6 +825,67 @@ Defaults for the work itself: - **Never** make `certify` non-blocking or silently delete a real gap to go green; that is the one move the whole model exists to prevent. +## Exploratory Parity Discovery (025-exploratory-parity-discovery) + +The registry says whether each *curated* behavior is covered; the coverage model says whether the +*scenario space* is. This searches for differences **nobody curated** — and hands each one over as +a reviewable candidate that a human, never a program, may promote into the record. All commands +are **dev-only**; `parity_registry_check` asserts `deacon --help` gains nothing from any of them. + +**Two data roots, deliberately siblings.** `conformance/discovery/` (`findings.json`, +`campaigns.json`, `corpus.json`) is a sibling of `conformance/registry/`, **not** a child. No +registry loader path reaches it, so nothing a campaign finds can influence `validate` or +`certify`. The one permitted cross-root reference points *out* of the queue +(`Finding::promotedTo` → a registry case), never into it. `conformance/registry/metamorphic.json` +(`mrl-` relation records) is the exception that lives in the registry proper, because a relation +is a curated assertion, not a finding. + +**Never gates — the rule the whole lane rests on.** A discovery command's exit status reflects +**whether it ran**, never **what it found**. A campaign that surfaces forty differences exits +`0`; one that cannot verify the oracle exits non-zero. Any command whose status depends on its +findings becomes a gate the moment someone wires it into CI, and a stochastic gate makes green +non-reproducible. The single exception is `discovery-proof`, and it is *not* finding-dependent: +it asserts a property of the **machinery** (an injected difference must traverse all six stages), +so non-zero means the pipeline is broken — exactly the thing that should fail a lane. + +**Hermetic vs live split.** +- **Hermetic** (`cargo run -p deacon-conformance -- discovery `) + — no network, no Docker, no oracle. `discovery_hermetic` + `discovery_cli` run in the `default` + and `dev-fast` lanes. `check` is read-only by construction; `triage` is the ONLY writer of + `classification`; `scaffold` writes **nothing** (stdout only, `UNREVIEWED` sentinels the loader + rejects). +- **Live** (`cargo run -p parity-harness --bin `) — four + campaign tiers: `metamorphic` (deacon-only, no oracle/Docker/network), `config-differential` + (nightly), `container-differential` (invoked-only; its 5-min per-candidate ceiling would starve + the scheduled window), `corpus` (weekly, network-backed). Selected **only** by + `[profile.discovery]`, whose `default-filter` is an explicit `binary(=…)` allow-list — never a + `discovery_*` glob, which would capture the hermetic guard `discovery_hermetic` and silently + drop it from the fast lane (the exact mistake the parity profile documents making with + `parity_harness_faults` / `parity_registry_check`). + +**D-classes vs V-classes — different roots, different consequences.** V-classes (`validate`) +police the **registry** and block a PR; several feed `certify` and block a release. D-classes +(`discovery check`) police the **discovery root** and block a PR *only* on the integrity of the +queue itself — they can never block a release, because a finding is not coverage. + +| Class | Guards | +|---|---| +| **D1** | derived-id mismatch / duplicate id / empty witnesses; a signature naming an undeclared channel; an unresolvable `firstObserved`/`lastObserved`, witness, `splitFrom`, or campaign `profile`; a `split` ancestor with <2 children; non-finite `spaceCoveredFraction`; empty seed | +| **D2** | a `triaged`/`promoted`/`no-longer-reproducing` finding with no classification; an `untriaged`/`split` one carrying one; a `promoted` finding classified `normalizer-defect` / `fixture-defect` | +| **D3** | a `promoted` finding with no `promotedTo`, or naming a case absent from the registry; a `promotedTo` in any other state | +| **D4** | a corpus `commit` that is not 40-hex (a branch or tag is mutable); malformed `contentDigest`; non-derived or duplicate corpus id/name | +| **D5** | a `schemaPin` / `prosePin` / `oracleVersion` naming a revision absent from `revisions.json` | + +**Promotion is a human act, structurally.** There is no code path from a finding to a registry +write — asserted behaviourally *and* by source scan (`no_discovery_source_references_a_registry_or_snapshot_writer`). +`certify`'s verdict is byte-identical with a queue full of unreviewed findings and with an empty +one. To promote: `discovery scaffold ` (or `--tolerate` for a **scoped** `wvr-` waiver — +a blanket scope is refused, not emitted), then hand-edit the registry using that output as a +starting point, then `validate`. A finding is a *candidate* for an assertion and never blocks; a +gap is missing coverage and always blocks — do not conflate them (`conformance/RULES.md`). + +See `specs/025-exploratory-parity-discovery/quickstart.md`. + ## Pre-Implementation Checklist Before implementing any new subcommand or feature: @@ -1178,6 +1253,8 @@ RUST_LOG=debug cargo run -- up --container-data-folder /tmp/cache - Rust, Edition 2024, MSRV 1.95 (`unsafe_code = "deny"` workspace-wide) + existing workspace deps only — `serde`/`serde_json` (strict-JSON records), `indexmap` (declaration order), `sha2` (unit/case/fixture hashing), `tokio` (bounded async exec in the harness), `thiserror` (domain errors), `tracing`, `toml` (nextest-profile drift check), `tempfile` (dev-dep, isolated workspaces). **No new crates, no new dependencies** (research D6). (023-migrate-parity-to-conformance) - strict-JSON, version-controlled. New: `conformance/migration/baseline.json` (frozen inventory), `conformance/migration/mapping.json` (unit → case/residual), `conformance/registry/residuals.json` (residual records). Extended: `conformance/registry/cases.json`. Generated (git-ignored): `target/conformance/migration-report.{json,md}`, `target/parity/equivalence.json`. All writes atomic (temp file + `fs::rename`). (023-migrate-parity-to-conformance) - strict-JSON, version-controlled. New hand-authored: `conformance/registry/scenario.json` (`sdim-` scenario dimensions), `conformance/registry/applicability.json` (`rule-` exclusions + `hrt-` high-risk triples), `conformance/registry/obligation-dispositions/.json` (`odp-` records), `conformance/registry/regressions.json` (`reg-` records). New machine-owned: `conformance/obligations/obligations.json` (`obl-`, sole output of `coverage generate`). Migrated: `cases.json` → `cases/.json`. Generated (git-ignored): `target/conformance/coverage-{pairwise,triples,operations,observables}.{json,md}`, `target/conformance/regressions.json`. All writes atomic. (024-deterministic-conformance-coverage) +- Rust, Edition 2024, MSRV 1.95 (`unsafe_code = "deny"` workspace-wide) + existing workspace deps only — `serde`/`serde_json` (strict-JSON records), `indexmap` (declaration order), `sha2` (`hash8` signature/fixture ids), `tokio` (bounded async exec), `thiserror` (domain errors), `tracing`, `tempfile` (dev-dep, isolated workspaces). **No new crates** — including no RNG crate (research D2). (025-exploratory-parity-discovery) +- strict-JSON, version-controlled. New root `conformance/discovery/` (queue, campaigns, corpus manifest) — a sibling of `registry/`, deliberately outside it. New registry file `conformance/registry/metamorphic.json` (`mrl-` relation records). Generated artifacts under `target/discovery/` (git-ignored, byte-stable). All writes atomic (temp file + `fs::rename`). (025-exploratory-parity-discovery) ## Recent Changes - 024-deterministic-conformance-coverage: Filled the coverage gaps the migration froze in diff --git a/Makefile b/Makefile index 33528e89..28165fde 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,9 @@ help: ## Show this help @echo "Testing - Other:" @grep -E '^(test-non-smoke|test-smoke|test-parity|test-parity-all|test-parity-regressions|parity):.*?##' $(MAKEFILE_LIST) | sed -E 's/:.*?##/\t- /' @echo "" + @echo "Testing - Exploratory discovery (never gates):" + @grep -E '^(test-discovery|test-discovery-proof|test-discovery-check):.*?##' $(MAKEFILE_LIST) | sed -E 's/:.*?##/\t- /' + @echo "" @echo "Code Quality:" @grep -E '^(fmt|clippy|coverage):.*?##' $(MAKEFILE_LIST) | sed -E 's/:.*?##/\t- /' @echo "" @@ -338,6 +341,50 @@ test-parity-regressions: ## Prove every observable channel can fail (injected-re test-parity-all: ## Alias for test-parity (live parity certification) $(MAKE) test-parity +.PHONY: test-discovery +test-discovery: install-nextest ## Run exploratory discovery campaigns, then render the findings queue + @set -euo pipefail; \ + ./scripts/nextest/assert-installed.sh; \ + # 025-exploratory-parity-discovery (contracts/discovery-cli.md § Make targets). \ + # Step 1 runs every registered discovery binary under the dedicated `discovery` \ + # profile, whose default-filter is an EXPLICIT binary(=…) allow-list — never a \ + # `discovery_*` glob, which would capture the hermetic guard. Step 2 renders \ + # target/discovery/queue.{json,md}. \ + # \ + # This lane GATES NOTHING. A campaign that finds forty differences exits 0; only a \ + # machinery failure is non-zero. It is a THIRD lane alongside the PR lanes and live \ + # parity, and a red run here never blocks a release. \ + # \ + # Needs the pinned oracle for every tier except `metamorphic`, Docker for the \ + # container-backed tier, and network for the corpus tier; each fails loud on a \ + # missing prerequisite, never skips. \ + cargo nextest run --profile discovery; \ + cargo run -p deacon-conformance -- discovery report + +.PHONY: test-discovery-proof +test-discovery-proof: ## Prove the discovery pipeline can surface an injected difference end to end + @set -euo pipefail; \ + # 025-exploratory-parity-discovery (FR-042a): injects a known difference at the \ + # SEALED evidence-source boundary and requires it to traverse generation → \ + # comparison → minimization → candidate → classification → promotable. \ + # \ + # This is the ONE discovery command whose status depends on an outcome — and it is \ + # not a finding-dependent status: it asserts a property of the MACHINERY, so \ + # non-zero means the pipeline is broken, which is exactly the thing that should fail \ + # a lane. An injection that never landed exits 1 as `InjectionInapplicable` rather \ + # than being counted as "found nothing": a mis-authored proof must never masquerade \ + # as a working pipeline. \ + cargo run -p parity-harness --bin discovery-proof + +.PHONY: test-discovery-check +test-discovery-check: ## Validate the discovery data root (hermetic; also runs in the fast lane) + @set -euo pipefail; \ + # 025-exploratory-parity-discovery: the D1–D5 violation classes over \ + # conformance/discovery/. Hermetic — no Docker, no network, no oracle — so it is \ + # safe anywhere and is also asserted by the `discovery_hermetic` guard in the \ + # default/dev-fast lanes. Read-only by construction: `check` never writes. \ + cargo run -p deacon-conformance -- discovery check + .PHONY: test-podman test-podman: ## Run Podman runtime tests via Makefile @set -euo pipefail; \ diff --git a/conformance/RULES.md b/conformance/RULES.md index 8f4bc7c8..e8b38144 100644 --- a/conformance/RULES.md +++ b/conformance/RULES.md @@ -74,6 +74,8 @@ checkable at a glance rather than by reading every section. | **V28** | an applicable obligation with zero dispositions, or with more than one | [Obligation dispositions](#obligation-dispositions-v28--v29) | | **V29** | malformed disposition: filler rationale; a high-risk triple dispositioned by rationale/waiver rather than a case; a disposition whose obligation no longer resolves (stale) | [Obligation dispositions](#obligation-dispositions-v28--v29) | | **V30** | injected-regression integrity: a declared channel with no regression record; a regression targeting a channel with no observer | [Injected-regression harness](#injected-regression-harness-v30) | +| **V31** | metamorphic relation integrity: a missing or unresolvable `ground`; an empty `channels`, or one naming an undeclared channel; a duplicated `transformation`; a `rationale` that is a label rather than an argument | [Metamorphic relation catalogue](#metamorphic-relation-catalogue-v31--v32) | +| **V32** | a mandated metamorphic relation family (FR-044) with no record | [Metamorphic relation catalogue](#metamorphic-relation-catalogue-v31--v32) | **Three distinctions this file keeps apart**, because conflating any pair makes a status unfalsifiable: @@ -992,3 +994,144 @@ this way, and both were channels the registry believed it was covering. | An ordinary run can never inject (FR-070) | injection needs a process-level capability only the `coverage-regressions` bin takes out; the one hook the runner calls is inert without it | | A perturbation is never left applied (FR-066) | an RAII guard reverts on success **and** on unwind, mirroring the Docker workspace guard; the tree is verified unmodified afterwards | | The classification is reproducible (FR-069) | perturbations are data, applied in a deterministic order, and the report is byte-stable | + +--- + +## Metamorphic relation catalogue (V31 – V32) + +`registry/metamorphic.json` is the one piece of the exploratory-discovery machinery that +lives **inside** the registry (025-exploratory-parity-discovery, research D11). Everything +else discovery produces — findings, campaigns, the corpus manifest — sits in +`conformance/discovery/`, a sibling of `registry/` that no loader path reaches, and is +policed by the separately numbered **D1 – D5** classes. + +The split follows from what each thing *is*. A metamorphic relation is an **assertion the +project makes**: "reordering these keys must not change the result, and here is the clause +that says so." That is the same kind of object as an applicability rule — hand-authored, +reviewed, stable, and naming `clu-`/`bhv-` ids only the registry loader resolves. A finding +is a **candidate** for an assertion: machine-produced, unreviewed, possibly wrong, and it +must never reach `certify`. + +### The two effects, and why sensitivity is mandatory + +| `effect` | Assertion | Catches | +|---|---|---| +| `invariance` | the transformation MUST NOT change the normalized result | a tool reading meaning that is not there | +| `sensitivity` | the transformation MUST change the normalized result | a tool ignoring meaning that *is* there | + +**A sensitivity relation is the one thing the differential structurally cannot replace.** +If deacon and the reference *both* wrongly ignore declaration order, the differential is +clean and the defect is invisible to it — both sides agree, and agreeing is exactly what +the differential checks. A sensitivity relation asserts the result *must* change, so +consistent-wrongness fails it. That is why FR-043 mandates both kinds rather than treating +sensitivity as an optional extra, and why `mrl-declaration-order-sensitivity` is in the +mandated set rather than left to judgement. + +### The ground must resolve, and the rationale must argue + +Every relation names a `ground` resolving to a normative clause (`clu-`) or a recorded +behavior (`bhv-`). Without one, a relation records an author's intuition about what *ought* +to be irrelevant — and the failure mode is the quiet one: **an ungrounded invariance +relation that happens to be wrong does not fail, it passes**, silently, while asserting +something the specification never said. A grounded one can be checked by reading the clause. + +A citation nobody argued from is only half of that. The `rationale` has to carry two claims +and the connective between them — what the ground says, and why that makes this +transformation irrelevant (or significant) — so **V31** also rejects a rationale short +enough to be a caption. + +The rationale test is deliberately **not** V26's `ground` test. That check's vague-marker +list is tuned for one-line capability statements, where "later" and "unknown" are evasions; +in a paragraph they are ordinary words, and it rejected the declaration-order rationale for +quoting its own clause ("later files override earlier ones"). A marker list tuned for a +different field rejects correct text — the same reason V26's ground test does not reuse the +out-of-scope marker list either. + +### A behavior is a real ground, not a fallback + +FR-045 admits a normative clause **or** a recorded behavior, and the committed catalogue +uses both — three clauses and four behaviors. The contract's mandated-family table +*nominates* a clause for five of the seven, but the pinned prose does not everywhere say +what those relations assert: no clause unit carries the sentence that fixes the on-disk +format as JSON-with-Comments, so `mrl-formatting-invariance` and `mrl-comment-invariance` +cite `bhv-readconfig-basic-parse` and `bhv-readconfig-malformed-jsonc-rejected` instead. + +Stretching a nearby clause to fill the column would have been the worse outcome, and +precisely the thing the ground requirement exists to prevent: a citation that does not say +what the relation claims reads, to a reviewer, exactly like one that does. Where the prose +is silent, cite the behavior that records what we actually know and say so in the rationale. + +### How a `clu-` ground resolves without the clause inventory + +A `bhv-` ground resolves against `behaviors/*.json`. A `clu-` ground resolves against the +registry's own **clause classifications**: V12 already requires every clause unit to carry +exactly one `clc-` record, so "named by a classification" and "is a clause unit" are the +same set, and V11 separately reports a classification naming a clause that no longer +exists. Resolving this way keeps V31 a pure function of the loaded registry, so `report` +and `certify` see it too — rather than scoping it to the one entry point that happens to +load the committed inventory. + +### The classes + +| Class | Fires on | Remedy | +|---|---|---| +| **V31** | a blank `ground`; a `ground` that is not a well-formed id, is neither a `clu-` nor a `bhv-`, or names a clause/behavior the registry does not carry; an empty `channels`; a `channels` entry absent from `channels.json`; an empty or duplicated `transformation`; a `rationale` under twenty words | Cite a clause or behavior that actually says it, and write the argument connecting the two | +| **V32** | a mandated family (FR-044) with no record | Restore the relation, or change the mandated list deliberately — never silently | + +An **unknown `effect`** is listed under V31 by contracts/metamorphic-catalogue.md but is +refused strictly earlier, at **load**, by the closed enum: same outcome, better diagnosis, +and no record with an unrecognised effect can reach evaluation, where "unknown" would have +to be resolved into one of the two answers by a default. + +The mandated list is a **floor, not a suggestion**. Dropping a family removes an assertion +nothing else makes, and the loss is invisible in the ordinary way — every remaining relation +still passes, so a smaller catalogue looks exactly like a healthy one. V32 is what makes the +removal say so. + +### Scope + +A registry that declares **neither** a catalogue nor a scenario model is silent: a fixture +predating this feature has not opted into the regime, and reporting seven missing families +for each one would say nothing true about them. A registry that declares **either** is +checked — in particular, deleting `metamorphic.json` from the real registry is V32 seven +times over, not a quiet pass. + +### Relations are evaluated against deacon alone + +FR-048 requires it, and it is load-bearing rather than incidental: the metamorphic tier is +the only part of discovery that needs **neither** the pinned oracle **nor** Docker **nor** +the network, so a contributor with no devcontainer CLI installed can develop and test it. +It does **not** license running it in a pull-request lane — FR-055 is absolute, and its +reason is stochasticity, not resource cost. + +## Finding vs gap (do not conflate) + +A **finding** is a *candidate for an assertion*: a difference a campaign surfaced that nobody +has yet decided anything about. A **gap** is an *admission of missing coverage*. They live in +different roots, and the distinction is what lets a stochastic search be wired into CI at all. + +| | `conformance/discovery/findings.json` (`fnd-`) | `conformance/registry/gaps.json` (`gap-`) | +|---|---|---| +| What it admits | a difference **was observed**; nobody has judged it | **coverage is missing** — no evidence exists | +| Evidence behind it | a reproducing witness, at least one | none, by definition | +| Blocks `certify`? | **No**, never — `certify` cannot even read this root | **Yes**, always | +| Blocks a PR? | only via D1–D5, on queue *integrity* — never on its contents | via V5, on the registry | +| Who may create it | a campaign, mechanically | a human, deliberately | +| Resolution | triage, then promote it (a human edit) or classify it away | add real evidence and delete the record | + +**Why they must never merge.** A finding is unreviewed by construction — a campaign that +surfaces forty differences has made forty claims nobody has checked, and some will be +`normalizer-defect` or `fixture-defect`, i.e. defects in the *observer*, not in deacon. If a +finding blocked, every campaign would be a release gate whose verdict changed with its seed, +and green would stop being reproducible. Conversely, if a gap did **not** block, the one +record whose entire purpose is to say "we do not know" would become decorative. + +So the queue is a **triage list, not a defect list**, and the arrow between the roots points +one way only: a finding becomes registry content by a human promoting it (`Finding::promotedTo` +→ a real case, **D3**), never by a program writing one. A finding is never "fixed" by editing +the queue — it is promoted, classified away, or observed to stop reproducing. + +**Corollary — an untriaged finding is not a passing finding.** `discovery report` counts the +untriaged bucket explicitly so that "nothing has been looked at" can never render as "nothing +was found" (the same failure mode `certify` avoids by counting `non-testable` separately from +`covered`). diff --git a/conformance/discovery/campaigns.json b/conformance/discovery/campaigns.json new file mode 100644 index 00000000..9c88b7d3 --- /dev/null +++ b/conformance/discovery/campaigns.json @@ -0,0 +1,4 @@ +{ + "schemaVersion": 1, + "records": [] +} diff --git a/conformance/discovery/corpus.json b/conformance/discovery/corpus.json new file mode 100644 index 00000000..f53777c0 --- /dev/null +++ b/conformance/discovery/corpus.json @@ -0,0 +1,302 @@ +{ + "schemaVersion": 1, + "records": [ + { + "id": "cor-8b1d0ec3", + "name": "images-javascript-node", + "repository": "devcontainers/images", + "commit": "31b61b521d55926d62c748b659f24ae71774c0e3", + "path": "src/javascript-node", + "contentDigest": null, + "notes": "Dockerfile + feature-heavy image recipe with scripts." + }, + { + "id": "cor-eb074204", + "name": "images-python", + "repository": "devcontainers/images", + "commit": "31b61b521d55926d62c748b659f24ae71774c0e3", + "path": "src/python", + "contentDigest": null, + "notes": "Dockerfile build with multiple official features." + }, + { + "id": "cor-4c2f2b92", + "name": "images-go", + "repository": "devcontainers/images", + "commit": "31b61b521d55926d62c748b659f24ae71774c0e3", + "path": "src/go", + "contentDigest": null, + "notes": "Dockerfile build plus Go/Node/common-utils features." + }, + { + "id": "cor-f24adfac", + "name": "images-rust", + "repository": "devcontainers/images", + "commit": "31b61b521d55926d62c748b659f24ae71774c0e3", + "path": "src/rust", + "contentDigest": null, + "notes": "Dockerfile build with Rust feature and lifecycle customizations." + }, + { + "id": "cor-2b59797e", + "name": "images-java", + "repository": "devcontainers/images", + "commit": "31b61b521d55926d62c748b659f24ae71774c0e3", + "path": "src/java", + "contentDigest": null, + "notes": "Dockerfile build with Java/Node features." + }, + { + "id": "cor-d250b81b", + "name": "images-php", + "repository": "devcontainers/images", + "commit": "31b61b521d55926d62c748b659f24ae71774c0e3", + "path": "src/php", + "contentDigest": null, + "notes": "Dockerfile build with a checked-in local feature." + }, + { + "id": "cor-72e54d3d", + "name": "templates-javascript-node-postgres", + "repository": "devcontainers/templates", + "commit": "95f7406a57fc5f0798964a5853c5ac04added322", + "path": "src/javascript-node-postgres", + "contentDigest": null, + "notes": "Compose-based template workspace with app + Postgres services." + }, + { + "id": "cor-a86505dc", + "name": "templates-go-postgres", + "repository": "devcontainers/templates", + "commit": "95f7406a57fc5f0798964a5853c5ac04added322", + "path": "src/go-postgres", + "contentDigest": null, + "notes": "Compose-based template workspace for Go + Postgres." + }, + { + "id": "cor-0f3f2973", + "name": "try-node", + "repository": "microsoft/vscode-remote-try-node", + "commit": "45f5e33e47f4b113804ea808b7ce4c90a6823867", + "path": "", + "contentDigest": null, + "notes": "Small image-based Node workspace used by the reference ecosystem." + }, + { + "id": "cor-918a1ec3", + "name": "try-python", + "repository": "microsoft/vscode-remote-try-python", + "commit": "e351212b72f76fb557c6f31956eb44756300b8b4", + "path": "", + "contentDigest": null, + "notes": "Small image-based Python workspace with app files." + }, + { + "id": "cor-e9236a47", + "name": "try-go", + "repository": "microsoft/vscode-remote-try-go", + "commit": "f4575309350c5ca2f1495c28a13b4b07088e8cea", + "path": "", + "contentDigest": null, + "notes": "Small image-based Go workspace with module files." + }, + { + "id": "cor-ee9911fd", + "name": "try-rust", + "repository": "microsoft/vscode-remote-try-rust", + "commit": "d3cb2a9843af67d20491c9fde829b2c77230847b", + "path": "", + "contentDigest": null, + "notes": "Small image-based Rust workspace with crate sources." + }, + { + "id": "cor-a1a3b240", + "name": "try-java", + "repository": "microsoft/vscode-remote-try-java", + "commit": "4638a925031f05cca946b8ce6ab640c433c93585", + "path": "", + "contentDigest": null, + "notes": "Small image-based Java workspace with Maven sources." + }, + { + "id": "cor-302b0bd3", + "name": "try-dotnetcore", + "repository": "microsoft/vscode-remote-try-dotnetcore", + "commit": "ca7ad4d9216a1bcc1469e4ca3545a66ff3e771a0", + "path": "", + "contentDigest": null, + "notes": "Small image-based .NET workspace." + }, + { + "id": "cor-a3674cd3", + "name": "try-cpp", + "repository": "microsoft/vscode-remote-try-cpp", + "commit": "22af031095a03864b8c2032b6498ff6d21e46c36", + "path": "", + "contentDigest": null, + "notes": "Dockerfile-based C++ sample; .devcontainer contains required support files." + }, + { + "id": "cor-ce2a703b", + "name": "try-php", + "repository": "microsoft/vscode-remote-try-php", + "commit": "9c4c759e95499bb57be2a35e2e0c55f292036908", + "path": "", + "contentDigest": null, + "notes": "Small image-based PHP workspace." + }, + { + "id": "cor-7750fc6f", + "name": "oss-ruff", + "repository": "astral-sh/ruff", + "commit": "82b550741e8766224f23ce6d71fea262b866966b", + "path": "", + "contentDigest": null, + "notes": "Real OSS Rust/Python workspace with volume mounts and a post-create script." + }, + { + "id": "cor-a02f6d4c", + "name": "oss-gh-cli", + "repository": "cli/cli", + "commit": "57b9b207d900114092f30d78020c193b40621dfa", + "path": "", + "contentDigest": null, + "notes": "Real OSS Go workspace with sshd feature and explicit remoteUser." + }, + { + "id": "cor-a8ba9ba5", + "name": "oss-vscode", + "repository": "microsoft/vscode", + "commit": "af752dba42ade664df7d6e01f4f459e0c1718512", + "path": "", + "contentDigest": null, + "notes": "Dockerfile-based workspace with desktop-lite and rust features." + }, + { + "id": "cor-e548260c", + "name": "oss-typescript", + "repository": "microsoft/TypeScript", + "commit": "7964e22f2b85f16e520f0e902c7fd7b6f0c15416", + "path": "", + "contentDigest": null, + "notes": "Image-based workspace with feature metadata and rich VS Code customizations." + }, + { + "id": "cor-f14bc9e3", + "name": "oss-fluentui", + "repository": "microsoft/fluentui", + "commit": "de337cf86b501d2aa7d9b12c472489c7c88d6b24", + "path": "", + "contentDigest": null, + "notes": "Dockerfile-based workspace with build args and feature metadata." + }, + { + "id": "cor-521cedac", + "name": "oss-promptflow", + "repository": "microsoft/promptflow", + "commit": "3928a727b406e66d64ff42621534bb58e0ca18ce", + "path": "", + "contentDigest": null, + "notes": "Dockerfile-based workspace with remoteEnv, runArgs, and Azure CLI feature." + }, + { + "id": "cor-bd264478", + "name": "oss-autogen", + "repository": "microsoft/autogen", + "commit": "027ecf0a379bcc1d09956d46d12d44a3ad9cee14", + "path": "", + "contentDigest": null, + "notes": "Compose-based real repo with many features and explicit workspaceFolder." + }, + { + "id": "cor-f54fe848", + "name": "oss-fhir-server", + "repository": "microsoft/fhir-server", + "commit": "7769ecc5eb170920f384f309afa6e852e7fdee78", + "path": "", + "contentDigest": null, + "notes": "Compose-based real repo with custom workspaceFolder and helper scripts." + }, + { + "id": "cor-c058be43", + "name": "oss-monaco-editor", + "repository": "microsoft/monaco-editor", + "commit": "7374dcb41a787a63d5885a5be5e6bbc2e6bc338c", + "path": "", + "contentDigest": null, + "notes": "Simple image-based workspace with lightweight customizations." + }, + { + "id": "cor-c5539b6b", + "name": "oss-web-dev-for-beginners", + "repository": "microsoft/Web-Dev-For-Beginners", + "commit": "5f220217d35499881cfff61a5b4c2dab033ab228", + "path": "", + "contentDigest": null, + "notes": "Universal-image workspace with a feature and editor customizations." + }, + { + "id": "cor-faa58e07", + "name": "oss-generative-ai-for-beginners", + "repository": "microsoft/generative-ai-for-beginners", + "commit": "61a1240c5de4109ceac54142934411365c67c759", + "path": "", + "contentDigest": null, + "notes": "Universal-image workspace with hostRequirements and post-create script." + }, + { + "id": "cor-d214628f", + "name": "oss-ml-for-beginners", + "repository": "microsoft/ML-For-Beginners", + "commit": "24028ae995117b45fabb883cf42114e721ed65b5", + "path": "", + "contentDigest": null, + "notes": "Dockerfile-based workspace with context '..' and init runArgs." + }, + { + "id": "cor-20c88ab0", + "name": "oss-code-with-engineering-playbook", + "repository": "microsoft/code-with-engineering-playbook", + "commit": "016770e43d8a75be87b98c000c049f07c4a6e6f8", + "path": "", + "contentDigest": null, + "notes": "Dockerfile-based workspace with parent build context." + }, + { + "id": "cor-6130c515", + "name": "oss-procdump-linux", + "repository": "microsoft/ProcDump-for-Linux", + "commit": "f2717626a2960f8e0d0aebd9efe2e95bfe6c43b1", + "path": "", + "contentDigest": null, + "notes": "Dockerfile-based workspace selecting a non-default Dockerfile variant." + }, + { + "id": "cor-be29d36e", + "name": "oss-sample-app-aoai-chatgpt", + "repository": "microsoft/sample-app-aoai-chatGPT", + "commit": "54f0af2c09bfe71f93da4acfb06422e11f984d71", + "path": "", + "contentDigest": null, + "notes": "Image-based workspace with multiple OCI features including latest azd." + }, + { + "id": "cor-793c6056", + "name": "oss-agentic-cookbook", + "repository": "microsoft/AgenticCookBook", + "commit": "32c6b754cff666962b6cd4679a2bdd9183fbe28e", + "path": "", + "contentDigest": null, + "notes": "Nonstandard config path (.devcontainer/.devcontainer.json) with hostRequirements. Config lives at `.devcontainer/.devcontainer.json`, not at the workspace root's standard discovery location." + }, + { + "id": "cor-67e6c98b", + "name": "oss-presidio-anonymizer", + "repository": "microsoft/presidio", + "commit": "83ab7eb85609c49d9b0b17c44b5c025575966876", + "path": "", + "contentDigest": null, + "notes": "Nested config path with workspaceMount and cross-directory Dockerfile/context references. Config lives at `.devcontainer/presidio-anonymizer/devcontainer.json`, not at the workspace root's standard discovery location." + } + ] +} diff --git a/conformance/discovery/findings.json b/conformance/discovery/findings.json new file mode 100644 index 00000000..9c88b7d3 --- /dev/null +++ b/conformance/discovery/findings.json @@ -0,0 +1,4 @@ +{ + "schemaVersion": 1, + "records": [] +} diff --git a/conformance/registry/metamorphic.json b/conformance/registry/metamorphic.json new file mode 100644 index 00000000..b978fb10 --- /dev/null +++ b/conformance/registry/metamorphic.json @@ -0,0 +1,61 @@ +{ + "schemaVersion": 1, + "records": [ + { + "id": "mrl-formatting-invariance", + "transformation": "reindent the configuration document and rewrap its whitespace between tokens, changing no token", + "effect": "invariance", + "ground": "bhv-readconfig-basic-parse", + "channels": ["chan-exit-code", "chan-structured-output"], + "rationale": "The grounding behavior records that a well-formed devcontainer.json resolves to a configuration document, so the configuration is the JSON value the file denotes and not the bytes that spell it. JSON's grammar makes whitespace between tokens non-significant, so reindenting produces the same value; a resolved configuration that changes under reindentation is reading layout, which no clause makes meaningful. The contract's mandated-family table nominates a clause here, but the pinned prose states the on-disk format only in passing and no clause unit carries that sentence, so the recorded behavior is cited instead of stretching a discovery-locations clause into a statement about syntax." + }, + { + "id": "mrl-comment-invariance", + "transformation": "insert JSONC line and block comments and a trailing comma in every non-empty object and array", + "effect": "invariance", + "ground": "bhv-readconfig-malformed-jsonc-rejected", + "channels": ["chan-exit-code", "chan-structured-output"], + "rationale": "The grounding behavior fixes the parse dialect as JSON with Comments and records that only a hard syntax error is rejected. A comment and a trailing comma are well-formed in that dialect and denote no member, so neither can alter the value the document describes. This relation therefore fails in two directions that both matter: rejecting a commented document, and silently changing the resolved configuration when one is accepted." + }, + { + "id": "mrl-key-order-invariance", + "transformation": "reverse the member order of every JSON object in the configuration document", + "effect": "invariance", + "ground": "clu-json-reference-this-property-allows-you-to-override-the-feature-desc-bebdddcd", + "channels": ["chan-exit-code", "chan-structured-output"], + "rationale": "The cited clause exists precisely because the order Features are written in does not decide the order they install in: install order is resolved from installsAfter and overridden by overrideFeatureInstallOrder. That is the specification saying, about the one object where ordering would most plausibly matter, that the object's member order carries no meaning. A resolved configuration that changes under member permutation is reading an order the specification says is not there." + }, + { + "id": "mrl-path-relocation", + "transformation": "materialize the identical workspace tree at a different absolute path, keeping its directory basename", + "effect": "invariance", + "ground": "bhv-readconfig-discovery-locations", + "channels": ["chan-exit-code", "chan-structured-output"], + "rationale": "The grounding behavior records that discovery searches locations relative to the workspace folder, so relocating the workspace relocates every input identically and the host path enters the result only through the declared localWorkspaceFolder substitutions. Comparison is therefore modulo the declared path tokenization, and any residual the tokenization does not absorb is reported rather than tolerated: such a residual is as likely to be a normalizer defect as a deacon regression, and misfiling it as the latter sends someone to fix code that is correct." + }, + { + "id": "mrl-lifecycle-equivalence", + "transformation": "rewrite a lifecycle command from its bare form into the single-entry named-object form denoting the same command", + "effect": "invariance", + "ground": "clu-parallel-lifecycle-the-key-of-the-object-will-be-a-unique-name-for-io-e2509dfa", + "channels": ["chan-exit-code", "chan-structured-output"], + "rationale": "The cited clause says the object form's key is a name for the command and its value is the same string or array command the bare form carries, so a single-entry object naming one command denotes exactly that command. Both spellings must therefore resolve, and switching between them must change nothing outside the lifecycle property itself; a difference anywhere else means the form of a command is leaking into unrelated resolution." + }, + { + "id": "mrl-extends-flattening", + "transformation": "replace a two-link extends chain with its hand-flattened single-document equal", + "effect": "invariance", + "ground": "bhv-readconfig-extends-merged", + "channels": ["chan-exit-code", "chan-structured-output"], + "rationale": "The grounding behavior records that the full extends chain is resolved into the merged configuration, which is the claim that a chain is a way of spelling a configuration rather than a different configuration. The merged block a chain produces must therefore equal the merged block its hand-flattened equal produces; a difference means the resolution is either losing a parent's contribution or letting the chain contribute something the flattened document does not." + }, + { + "id": "mrl-declaration-order-sensitivity", + "transformation": "reverse a declaration-ordered array, the dockerComposeFile overlay list", + "effect": "sensitivity", + "ground": "clu-json-reference-the-order-of-the-array-matters-since-the-content-io-bdf2bf4c", + "channels": ["chan-structured-output"], + "rationale": "The cited clause states outright that the order of this array matters, because later files override earlier ones. Reversing it therefore describes a different configuration and the result must change. This is the assertion the differential structurally cannot make: if deacon and the reference both canonicalized the list, the two sides would agree and the comparison would be clean, so the defect would be invisible to it. A sensitivity relation fails on that consistent wrongness." + } + ] +} diff --git a/conformance/registry/residuals.json b/conformance/registry/residuals.json index 901d263c..56cd7e61 100644 --- a/conformance/registry/residuals.json +++ b/conformance/registry/residuals.json @@ -196,9 +196,9 @@ "realworld::try-python", "realworld::try-rust" ], - "missingCapability": "vendored fixtures for the pinned third-party workspaces: the manifest fetches 33 public workspace snapshots over the network on demand, and vendoring third-party content into this repository is deliberately out of scope, so no case can materialize them", + "missingCapability": "vendored fixtures for the pinned third-party workspaces: the manifest fetches 33 public workspace snapshots over the network on demand, and vendoring third-party content into this repository is deliberately out of scope, so no declarative case can materialize them", "disposition": "permanent", - "outOfScopeRationale": "Vendoring third-party content into this repository is out of scope by decision (research D8): the manifest names 33 public workspace snapshots fetched over the network on demand, and a declarative fixture must be committed, content-hashed input. These entries are inventoried as a recorded coverage SOURCE, never executed, so there is no capability to add \u2014 only a decision already taken.", + "outOfScopeRationale": "Vendoring third-party content into this repository is out of scope by decision (research D8): the manifest names 33 public workspace snapshots fetched over the network on demand, and a declarative fixture must be committed, content-hashed input. The residual is therefore about REPRESENTATION as a declarative case, and it stays permanent \u2014 not about whether the entries are exercised. Since 025-exploratory-parity-discovery (US7) they are: the manifest moved out of the retired fixtures/parity-corpus/fetch_realworld_corpus.py into the Rust-owned conformance/discovery/corpus.json, where violation class D4 rejects any non-immutable reference hermetically on every pull request, and the weekly network-backed `corpus` campaign tier fetches each entry with content-digest verification and compares it against the pinned oracle. What no case can do is carry the workspace as committed input; that has not changed and is not going to.", "behaviors": [] }, { diff --git a/crates/conformance/Cargo.toml b/crates/conformance/Cargo.toml index a1ad850c..6772e677 100644 --- a/crates/conformance/Cargo.toml +++ b/crates/conformance/Cargo.toml @@ -28,6 +28,13 @@ jiff = "0.2" [dev-dependencies] tempfile = { workspace = true } +# NOTE: no async runtime, deliberately — not even as a dev-dependency. `discovery_hermetic` +# asserts that this crate declares no way to speak a network protocol, and its argument is +# that the capability must be ABSENT rather than merely unused (SC-013). The shrink +# strategy's predicate is async (the live one runs two CLIs), and its unit tests drive that +# future with a five-line `block_on` over `Waker::noop()` instead — see +# `discovery::shrink::tests`. A runtime would be a far larger hammer for a future that +# never suspends, and it would cost exactly the guarantee that guard exists to hold. [lints] workspace = true diff --git a/crates/conformance/src/baseline.rs b/crates/conformance/src/baseline.rs index fb87cdef..d6855238 100644 --- a/crates/conformance/src/baseline.rs +++ b/crates/conformance/src/baseline.rs @@ -29,8 +29,11 @@ //! function drifts the baseline and must be acknowledged; //! - the declarative runner's units come from the registry's declarative cases, with //! their channels, fixtures and difference classes derived from the case record; -//! - the external manifest's units come from `fetch_realworld_corpus.py`'s pinned -//! entries (research D8). +//! - the external manifest's units come from the pinned entries of the real-world corpus +//! manifest (research D8). That manifest moved from `fetch_realworld_corpus.py` to the +//! Rust-owned `conformance/discovery/corpus.json` in 025 US7 (T109); the same 33 names +//! in the same order, so the frozen file still regenerates byte-for-byte, now read from +//! a record whose pins this repository validates on every pull request. //! //! `assertion`, and the per-case `channels` / `errorPath` / `fixtures` / `diffClasses` //! for the scenario and guard programs, are **authored once, here, at freeze**. They @@ -71,7 +74,14 @@ pub const UNFROZEN_REVISION: &str = "unfrozen"; const PARITY_REGISTRY_REL: &str = "fixtures/parity-corpus/registry.json"; /// Repo-relative location of the pinned external real-world corpus manifest (D8). -const REALWORLD_MANIFEST_REL: &str = "fixtures/parity-corpus/fetch_realworld_corpus.py"; +/// +/// **Re-pointed at the Rust-owned manifest** when `fetch_realworld_corpus.py` was retired +/// (025 US7, T109). The enumeration is unchanged in substance: the same 33 entry *names* +/// in the same order, so `baseline generate` still reproduces the frozen file +/// byte-for-byte. What changed is where the names are read from — a strict-JSON record +/// this repository validates on every pull request, rather than a Python tuple nothing +/// checked. +const REALWORLD_MANIFEST_REL: &str = "conformance/discovery/corpus.json"; /// Repo-relative directory holding the parity test binaries' sources. const TESTS_DIR_REL: &str = "crates/deacon/tests"; @@ -987,6 +997,43 @@ const FUNCTION_UNITS: &[(&str, &[(&str, Authored)])] = &[ diff_classes: DIFF_NONE, }, ), + // 025 US3 (T054/T056/T057), on the same terms as 024 Block B above and 023 + // User Story 4 before it. Membership is DERIVED — the enumeration scans this + // program's real `#[test]` functions — so a guard added after the freeze + // cannot be silently absent from the baseline; it appears as a unit with no + // authored assertion, which is a hard error. Authoring the sentence here is + // the only way to describe it, and describing it is the point: the baseline + // says what each unit ASSERTS, not merely that it exists. + ( + "the_discovery_lane_selects_the_campaigns_and_no_pull_request_lane_does", + Authored { + assertion: "the discovery nextest profile selects every live discovery campaign binary and captures no hermetic discovery guard, and no pull-request profile selects a campaign while both fast lanes select every guard", + channels: CH_NONE, + error_path: false, + fixtures: FX_NONE, + diff_classes: DIFF_NONE, + }, + ), + ( + "the_discovery_lane_is_wired_in_registry_tests_and_nextest", + Authored { + assertion: "every discovery binary is registered, has a source file at its declared test directory, and is wired into the nextest profiles; every discovery source file is registered; and the parity profile selects no discovery campaign", + channels: CH_NONE, + error_path: false, + fixtures: FX_NONE, + diff_classes: DIFF_NONE, + }, + ), + ( + "no_discovery_command_reaches_the_shipped_cli", + Authored { + assertion: "no discovery command introduced by the exploratory-discovery tooling appears at any depth of the shipped `deacon` command tree", + channels: CH_NONE, + error_path: false, + fixtures: FX_NONE, + diff_classes: DIFF_NONE, + }, + ), ], ), ( @@ -1543,23 +1590,19 @@ fn scan_test_functions(source: &str) -> Vec { out } -/// Scan the external corpus manifest for its pinned entry names — the `name="…"` -/// field of each `CorpusEntry(…)`, in declaration order. +/// Scan the external corpus manifest for its pinned entry names, in declaration order. +/// +/// Since T109 the manifest is `conformance/discovery/corpus.json` and the scan is a strict +/// deserialization rather than a line-oriented grep over Python source. A malformed +/// manifest therefore yields **no** names, which [`realworld_units`] turns into a hard +/// [`BaselineError::NoManifestEntries`]: a manifest that stopped parsing must never be +/// indistinguishable from one that lists nothing, because the second reads as "these 33 +/// recorded coverage sources were retired" and nobody decided that. fn scan_manifest_entry_names(source: &str) -> Vec { - let mut out = Vec::new(); - for line in source.lines() { - let trimmed = line.trim(); - let Some(rest) = trimmed.strip_prefix("name=\"") else { - continue; - }; - let Some(name) = rest.split('"').next() else { - continue; - }; - if !name.is_empty() { - out.push(name.to_string()); - } - } - out + let Ok(file) = serde_json::from_str::(source) else { + return Vec::new(); + }; + file.records.into_iter().map(|entry| entry.name).collect() } fn to_strings(values: &[&str]) -> Vec { @@ -1785,24 +1828,46 @@ mod tests { } #[test] - fn scan_manifest_entry_names_reads_pinned_entries() { - let src = r#" -ENTRIES = ( - CorpusEntry( - name="images-python", - repo="devcontainers/images", - ), - CorpusEntry( - name="try-node", - ), -) -"#; + fn scan_manifest_entry_names_reads_pinned_entries_in_declaration_order() { + let src = r#"{ + "schemaVersion": 1, + "records": [ + { + "id": "cor-eb074204", + "name": "images-python", + "repository": "devcontainers/images", + "commit": "31b61b521d55926d62c748b659f24ae71774c0e3", + "path": "src/python", + "contentDigest": null, + "notes": "n" + }, + { + "id": "cor-e0e0c8be", + "name": "try-node", + "repository": "microsoft/vscode-remote-try-node", + "commit": "45f5e33e47f4b113804ea808b7ce4c90a6823867", + "path": "", + "contentDigest": null, + "notes": "n" + } + ] +}"#; assert_eq!( scan_manifest_entry_names(src), vec!["images-python", "try-node"] ); } + #[test] + fn a_manifest_that_stopped_parsing_yields_no_names_rather_than_a_partial_list() { + // The caller turns an empty list into a hard error. A lenient scan that recovered + // *some* names would silently retire the rest — and a retired coverage source is + // exactly the claim nobody made. + assert!(scan_manifest_entry_names("not json at all").is_empty()); + assert!(scan_manifest_entry_names(r#"{"schemaVersion": 99, "records": []}"#).is_empty()); + assert!(scan_manifest_entry_names(r#"{"records": []}"#).is_empty()); + } + #[test] fn compare_names_added_removed_and_changed_units() { let committed = file(vec![unit("p::a"), unit("p::b")]); diff --git a/crates/conformance/src/bin/conformance.rs b/crates/conformance/src/bin/conformance.rs index 4133eeae..4fcac19d 100644 --- a/crates/conformance/src/bin/conformance.rs +++ b/crates/conformance/src/bin/conformance.rs @@ -31,6 +31,8 @@ use deacon_conformance::coverage_report::{build_coverage_reports, write_coverage use deacon_conformance::diff::{ diff, render_json as render_diff_json, render_md as render_diff_md, }; +use deacon_conformance::discovery::queue::{self, DiscoveryData}; +use deacon_conformance::discovery::report as discovery_report_mod; use deacon_conformance::inventory::{ InventoryDrift, compare, generate_inventory, render, write_inventory, }; @@ -136,6 +138,84 @@ enum Command { #[command(subcommand)] command: CoverageCommand, }, + /// Exploratory parity discovery tooling (025-exploratory-parity-discovery, + /// contracts/discovery-cli.md). Hermetic: no network, no Docker, no reference oracle. + /// Dev-only — NEVER part of the shipped `deacon` consumer CLI. + /// + /// Operates on `conformance/discovery/`, a SIBLING of the registry that no registry + /// loader path reaches, so nothing here can influence `certify`. + Discovery { + #[command(subcommand)] + command: DiscoveryCommand, + }, +} + +/// `discovery ` (contracts/discovery-cli.md). +/// +/// **The exit-status rule for every command here**: the status reflects *whether the +/// command ran*, never *what it found*. A queue holding fifty untriaged findings still +/// exits `0` from `report`. Any command whose status depended on its findings would +/// become a gate the moment someone wired it into CI — and a stochastic gate makes green +/// non-reproducible. `check` is the one exception, and it is a *structural* verdict on +/// the records, not a verdict on what was discovered. +#[derive(Debug, Subcommand)] +enum DiscoveryCommand { + /// Validate the discovery data root (violation classes D1–D5). Read-only by + /// construction. Reports ALL violations in one pass, matching `validate`. + Check { + /// Emit a single JSON document (`{ "ok", "violations" }`) on stdout instead of + /// one violation per line; logs still go to stderr. + #[arg(long)] + json: bool, + }, + /// Render the findings queue and campaign history to `target/discovery/queue.{json,md}` + /// (git-ignored, byte-stable). **Never gates**: the exit status reflects only whether + /// the artifacts were produced. + Report { + /// Directory to write `queue.json` and `queue.md` into. Defaults to + /// `/target/discovery/`. + #[arg(long, value_name = "DIR")] + out_dir: Option, + }, + /// Record a reviewer's classification. The ONLY writer of `classification`. + Triage { + /// The finding to classify (`fnd-`). + #[arg(value_name = "FND-ID")] + finding: String, + /// One of the six closed classifications. + #[arg(long, value_name = "CLASSIFICATION")] + classification: String, + /// Reviewer prose recorded alongside the classification. + #[arg(long, value_name = "TEXT")] + notes: Option, + }, + /// Split a signature-merged finding whose witnesses turn out to have different + /// causes (FR-032). The parent becomes an inert ancestor. + Split { + /// The finding to split (`fnd-`). + #[arg(value_name = "FND-ID")] + finding: String, + }, + /// Emit a skeleton behavior + case + fixture layout for promoting a finding, to + /// **stdout only**, with `UNREVIEWED` sentinels the registry loader rejects. + /// + /// Stdout-only is the same discipline as `inventory scaffold` / `clause scaffold`: + /// generation never writes a hand-authored file. Promotion is a human editing the + /// registry with this output as a starting point, which is what makes FR-036 hold — + /// there is no code path from a finding to a registry write. + Scaffold { + /// The finding to scaffold a promotion for (`fnd-`). + #[arg(value_name = "FND-ID")] + finding: String, + /// Scaffold a **tolerance** instead of a promotion: a scoped `wvr-` waiver + /// (rationale + `expires`) plus the `allowedDifferences` entry that references + /// it (FR-041). Still stdout only. + /// + /// A blanket or unscoped tolerance is refused rather than emitted — a bare-channel + /// `observablePath` is a global ignore list wearing a waiver id. + #[arg(long)] + tolerate: bool, + }, } /// `coverage ` (contracts/coverage-cli.md). `generate` @@ -484,9 +564,505 @@ fn run(cli: Cli) -> i32 { } CoverageCommand::Scaffold {} => coverage_scaffold(®istry_dir, &today), }, + Command::Discovery { command } => match command { + DiscoveryCommand::Check { json } => discovery_check(®istry_dir, json), + DiscoveryCommand::Report { out_dir } => discovery_report(®istry_dir, out_dir), + DiscoveryCommand::Triage { + finding, + classification, + notes, + } => discovery_triage(®istry_dir, &finding, &classification, notes.as_deref()), + DiscoveryCommand::Split { finding } => discovery_split(®istry_dir, &finding), + DiscoveryCommand::Scaffold { finding, tolerate } => { + discovery_scaffold(®istry_dir, &finding, tolerate) + } + }, + } +} + +// --------------------------------------------------------------------------- +// discovery (025-exploratory-parity-discovery) +// --------------------------------------------------------------------------- + +/// Load the registry a `discovery` subcommand needs to resolve the queue's outward +/// references (declared channels, recorded revisions, and — from US5 — cases). +/// +/// The direction is one-way and load-bearing: discovery reads the registry, the registry +/// never reads discovery. That asymmetry is what makes the queue unreachable from +/// `certify` (research D6). +fn load_for_discovery(registry_dir: &Path) -> Result { + // Deliberately NOT `load_for_coverage`: its diagnostic ends "…before generating + // obligations", which names the wrong subject entirely when a `discovery` subcommand + // is what failed. A message that misidentifies what the operator was doing sends them + // to the wrong file. + match Registry::load(registry_dir) { + Ok(registry) => Ok(registry), + Err(LoadError::Schema(errors)) => { + for violation in deacon_conformance::validate::schema_violations(&errors) { + eprintln!( + "{} {}: {}", + violation.code, violation.record, violation.message + ); + } + eprintln!( + "error: {} is schema-invalid; discovery resolves its channels and pins \ + against the registry, so fix the registry records first", + registry_dir.display() + ); + Err(1) + } + Err(other) => { + eprintln!( + "error: cannot read registry {}: {other}", + registry_dir.display() + ); + Err(2) + } + } +} + +/// A discovery data-root load failure, rendered per output mode by the caller. +/// +/// Carrying the failure rather than printing it is what lets `--json` keep its contract: +/// the violation report belongs on **stdout** as a single JSON document, and a helper that +/// printed to stderr would leave `… --json | jq .ok` with a parse error instead of `false` +/// exactly when something is wrong. +struct DiscoveryLoadFailure { + /// One `(class, record, message)` triple per failing file. + violations: Vec<(&'static str, String, String)>, + /// The process exit code this failure maps to. + code: i32, +} + +/// Load the discovery data root, reporting located schema failures as **D1**. +/// +/// A malformed file is D1 rather than a bare load error because that is what it is: the +/// records are the queue, and a record that does not parse is a malformed record. +fn load_discovery_data(discovery_dir: &Path) -> Result { + match DiscoveryData::load(discovery_dir) { + Ok(data) => Ok(data), + Err(LoadError::Schema(errors)) => Err(DiscoveryLoadFailure { + violations: errors + .iter() + .map(|e| ("D1", e.file.display().to_string(), e.message.clone())) + .collect(), + code: 1, + }), + Err(other) => Err(DiscoveryLoadFailure { + violations: vec![( + "D1", + discovery_dir.display().to_string(), + format!("cannot read discovery data root: {other}"), + )], + code: 2, + }), + } +} + +/// Print a load failure as plain text on stderr and return its exit code. Used by every +/// discovery subcommand except `check --json`, which renders it as JSON on stdout. +fn report_discovery_load_failure(failure: &DiscoveryLoadFailure, discovery_dir: &Path) -> i32 { + for (class, record, message) in &failure.violations { + eprintln!("{class} {record}: {message}"); + } + eprintln!( + "error: {} could not be loaded; the discovery data root is strict JSON and rejects \ + unknown fields, a missing `records` array, and an unsupported `schemaVersion`", + discovery_dir.display() + ); + failure.code +} + +/// `discovery check` (contracts/discovery-cli.md): validate the discovery data root. +/// +/// Read-only by construction — it never writes. Exit `0` when there are no D-class +/// violations, `1` when there are (all reported in one pass), `2` on an unreadable root +/// (the usage/IO code every other command in this binary uses). +/// +/// Under `--json` the violation report is a single JSON document on **stdout** in every +/// failing case, including the case where the data root itself would not load — a +/// consumer piping to `jq` must get `{"ok": false, …}`, never an empty stream. +fn discovery_check(registry_dir: &Path, json: bool) -> i32 { + let render = |violations: &[(&str, String, String)], ok: bool| -> Result<(), i32> { + if json { + let document = serde_json::json!({ + "ok": ok, + "violations": violations + .iter() + .map(|(class, record, message)| serde_json::json!({ + "class": class, + "record": record, + "message": message, + })) + .collect::>(), + }); + match serde_json::to_string_pretty(&document) { + Ok(text) => println!("{text}"), + Err(e) => { + eprintln!("error: could not render the check result: {e}"); + return Err(2); + } + } + } else { + for (class, record, message) in violations { + println!("{class} {record}: {message}"); + } + } + Ok(()) + }; + + let registry = match load_for_discovery(registry_dir) { + Ok(registry) => registry, + Err(code) => { + // The registry failure is already reported as located `V…`/`SCHEMA` lines on + // stderr by `load_for_discovery`; emit a well-formed `ok: false` document so + // the JSON contract holds even here. + if let Err(render_code) = render(&[], false) { + return render_code; + } + return code; + } + }; + let discovery_dir = deacon_conformance::discovery_dir_for(registry_dir); + let data = match load_discovery_data(&discovery_dir) { + Ok(data) => data, + Err(failure) => { + let violations: Vec<(&str, String, String)> = failure + .violations + .iter() + .map(|(c, r, m)| (*c, r.clone(), m.clone())) + .collect(); + if let Err(render_code) = render(&violations, false) { + return render_code; + } + if !json { + // Text mode already printed the violations above; add the remedy line. + eprintln!( + "error: {} could not be loaded; the discovery data root is strict JSON \ + and rejects unknown fields, a missing `records` array, and an \ + unsupported `schemaVersion`", + discovery_dir.display() + ); + } + return failure.code; + } + }; + + let checked = queue::check(&data, &queue::RegistryView::from_registry(®istry)); + let violations: Vec<(&str, String, String)> = checked + .iter() + .map(|v| (v.class(), v.record().to_string(), v.to_string())) + .collect(); + if let Err(code) = render(&violations, violations.is_empty()) { + return code; + } + let violations = checked; + + if violations.is_empty() { + eprintln!( + // The corpus count is named alongside the queue's: a run that validated 33 + // pinned entries and reported only "0 findings, 0 campaigns" would understate + // what it checked, and the D4 immutable-reference clause is the one thing here + // that must visibly run on every pull request. + "ok: {} validates clean ({} finding(s), {} campaign(s), {} corpus entr(ies))", + discovery_dir.display(), + data.findings.len(), + data.campaigns.len(), + data.corpus.len() + ); + 0 + } else { + eprintln!( + "error: {} has {} discovery violation(s)", + discovery_dir.display(), + violations.len() + ); + 1 + } +} + +/// `discovery report` (contracts/discovery-cli.md): render the queue and the campaign +/// history to `target/discovery/queue.{json,md}`. +/// +/// **Never gates.** The exit status reflects only whether the artifacts were produced: +/// `0` when written, `1` when they could not be (including when the inputs could not be +/// read, since without them there is nothing to write). A queue holding fifty untriaged +/// findings still exits `0`. +fn discovery_report(registry_dir: &Path, out_dir: Option) -> i32 { + let registry = match load_for_discovery(registry_dir) { + Ok(registry) => registry, + // `report` has only two statuses in the contract, so an unreadable input is a + // "could not write" — there is nothing to write from. + Err(_) => return 1, + }; + let discovery_dir = deacon_conformance::discovery_dir_for(registry_dir); + let data = match load_discovery_data(&discovery_dir) { + Ok(data) => data, + Err(failure) => { + report_discovery_load_failure(&failure, &discovery_dir); + // The contract gives `report` exactly two statuses, so every failure — an + // unreadable root included — collapses to "could not write". Passing the + // loader's `2` through would invent a third status the contract does not have. + return 1; + } + }; + + let pins = discovery_report_mod::CurrentPins::from_registry(®istry); + // The behavior index is what lets promoted findings be *reported* grouped under the + // behavior their case names (FR-031) — a reviewed mapping, never one the report + // invents, because a finding itself never names a behavior (FR-025). + let behaviors = discovery_report_mod::BehaviorIndex::from_registry(®istry); + let report = discovery_report_mod::build_queue_report_with_behaviors(&data, &pins, &behaviors); + let dir = out_dir.unwrap_or_else(|| workspace_root().join("target").join("discovery")); + + match discovery_report_mod::write_queue_report(&dir, &report) { + Ok(written) => { + for path in &written { + println!("{}", path.display()); + } + eprintln!( + "wrote {} discovery artifact(s) to {} ({} finding(s): {} untriaged, {} triaged, \ + {} split, {} promoted, {} no-longer-reproducing, {} pin-stale; {} campaign(s), \ + {} signature(s) suppressed)", + written.len(), + dir.display(), + report.total, + report.untriaged.len(), + report.triaged.len(), + report.split.len(), + report.promoted.len(), + report.no_longer_reproducing.len(), + report.pin_stale.len(), + report.campaigns.len(), + report.signatures_suppressed, + ); + 0 + } + Err(e) => { + eprintln!( + "error: could not write the discovery report to {}: {e}", + dir.display() + ); + 1 + } + } +} + +/// `discovery triage` (contracts/discovery-cli.md): record a reviewer's classification. +/// +/// The **only** writer of `classification`. Interactive-equivalent, never automated: no +/// campaign invokes it, which is what keeps promotion a human act. +/// +/// Exit `0` when recorded; `1` on an unknown finding, an invalid classification, or a +/// finding already in a terminal state. +fn discovery_triage( + registry_dir: &Path, + finding_id: &str, + classification: &str, + notes: Option<&str>, +) -> i32 { + let Some(classification) = queue::Classification::parse(classification) else { + let names: Vec<&str> = queue::Classification::all() + .iter() + .map(|c| c.as_str()) + .collect(); + eprintln!( + "error: `{classification}` is not a classification; expected one of: {}", + names.join(", ") + ); + return 1; + }; + + let discovery_dir = deacon_conformance::discovery_dir_for(registry_dir); + let mut data = match load_discovery_data(&discovery_dir) { + Ok(data) => data, + Err(failure) => return report_discovery_load_failure(&failure, &discovery_dir), + }; + + let Some(finding) = data.finding_mut(finding_id) else { + eprintln!( + "error: no finding `{finding_id}` in {}", + discovery_dir.display() + ); + return 1; + }; + + // The state machine owns which states may be classified and which advance + // (`crates/conformance/src/discovery/queue.rs`, T069). Re-deciding that here would put + // the lifecycle in two places, and the copy the checker does not read is the one that + // drifts. + let previous_state = finding.state; + let new_state = match finding.triage(classification, notes) { + Ok(state) => state, + Err(e) => { + eprintln!("error: {e}"); + return 1; + } + }; + + match queue::write_findings(&discovery_dir, &data.findings) { + Ok(()) => { + eprintln!( + "recorded `{}` on {finding_id} in {}", + classification.as_str(), + queue::findings_path(&discovery_dir).display() + ); + if previous_state == new_state { + eprintln!( + "note: {finding_id} stays in state `{}` — only a campaign that \ + reproduces it may change that", + new_state.as_str() + ); + } + 0 + } + Err(e) => { + eprintln!("error: could not write the findings queue: {e}"); + 1 + } } } +/// `discovery split` (contracts/discovery-cli.md): split a signature-merged finding whose +/// witnesses turn out to have different causes (FR-032). +/// +/// The parent becomes an inert ancestor (invariant Q10): it keeps its witnesses as +/// historical record, stops accepting new ones (enforced by [`queue::upsert_finding`], which +/// refuses the whole lineage, not just this record), and **surrenders its classification to +/// its children** — a split exists precisely because one classification could not describe +/// them all, so a parent that kept one would assert the judgement the split rejected. +/// +/// One child per witness, each keyed to `parent ‖ its own witness` and starting untriaged. +/// See [`queue::split_finding`] for why the finest partition is the only one derivable from +/// what the reviewer actually said. +/// +/// Exit `0` when the split was written; `1` on an unknown finding, a finding with fewer +/// than two witnesses (there is nothing to split), or a finding in a state the lifecycle +/// does not let a split leave. +fn discovery_split(registry_dir: &Path, finding_id: &str) -> i32 { + let discovery_dir = deacon_conformance::discovery_dir_for(registry_dir); + let mut data = match load_discovery_data(&discovery_dir) { + Ok(data) => data, + Err(failure) => return report_discovery_load_failure(&failure, &discovery_dir), + }; + + let split = match queue::split_finding(&mut data.findings, finding_id) { + Ok(split) => split, + Err(e) => { + eprintln!("error: {e}"); + if matches!(e, queue::TransitionError::UnknownFinding { .. }) { + eprintln!(" (searched {})", discovery_dir.display()); + } + return 1; + } + }; + + match queue::write_findings(&discovery_dir, &data.findings) { + Ok(()) => { + eprintln!( + "split {finding_id} into {} child finding(s); it is now an inert ancestor \ + that keeps its witnesses as historical record, accepts no new ones, and \ + carries no classification of its own.", + split.children.len() + ); + for child in &split.children { + eprintln!(" {child}"); + } + eprintln!( + "Each child is untriaged: classify them separately with `discovery triage`. \ + The split is permanent — the deduplication rule never re-merges this \ + lineage, so your judgement is not reverted by the next campaign." + ); + 0 + } + Err(e) => { + eprintln!("error: could not write the findings queue: {e}"); + 1 + } + } +} + +/// `discovery scaffold` (contracts/discovery-cli.md): emit a skeleton for promoting — or, +/// with `--tolerate`, for *tolerating* — a finding, to **stdout only**. +/// +/// Writes **nothing**. That is the same discipline as `inventory scaffold` / +/// `clause scaffold`, and here it is what makes FR-036 hold: there is no code path from a +/// finding to a registry write, so a stochastic process cannot author the record it is +/// tested against. Every field a human must decide carries the `UNREVIEWED` sentinel the +/// registry loader rejects, so scaffolded output cannot be committed unedited. +/// +/// The skeletons themselves live in `deacon_conformance::discovery::promote`, not here: +/// they are pure `Finding → JSON` transformations, and keeping them in the library is what +/// lets the hermetic guards assert the FR-041 scoping rule without spawning a process. This +/// function is the process boundary and nothing else — load, dispatch, render, explain. +/// +/// Exit `0` when emitted; `1` on an unknown finding, a non-promotable classification +/// (`normalizer-defect` / `fixture-defect`, FR-035), or a tolerance that would be blanket +/// or unscoped (FR-041). +fn discovery_scaffold(registry_dir: &Path, finding_id: &str, tolerate: bool) -> i32 { + use deacon_conformance::discovery::promote; + + let discovery_dir = deacon_conformance::discovery_dir_for(registry_dir); + let data = match load_discovery_data(&discovery_dir) { + Ok(data) => data, + Err(failure) => return report_discovery_load_failure(&failure, &discovery_dir), + }; + + let Some(finding) = data.finding(finding_id) else { + eprintln!( + "error: no finding `{finding_id}` in {}", + discovery_dir.display() + ); + return 1; + }; + + let document = if tolerate { + promote::tolerance_skeleton(finding) + } else { + promote::promotion_skeleton(finding) + }; + let document = match document { + Ok(document) => document, + Err(e) => { + eprintln!("error: {e}"); + return 1; + } + }; + + match serde_json::to_string_pretty(&document) { + Ok(text) => println!("{text}"), + Err(e) => { + eprintln!("error: could not render the scaffold: {e}"); + return 2; + } + } + + if tolerate { + eprintln!( + "scaffolded a TOLERANCE skeleton for {finding_id}; stdout only — nothing was \ + written. The waiver is scoped to the observable path the difference occurs at, \ + never to the whole channel (FR-041): a bare-channel scope is a global ignore \ + list wearing a waiver id, and it is refused rather than emitted. Fill in \ + `rationale`, `expires`, the behavior, and the binary/fixture the scope names, \ + then add the `allowedDifferences` entry to the case that observes this path. \ + Every `{SCAFFOLD_SENTINEL}` field is rejected by the registry loader, so this \ + cannot be committed unedited. Tolerating records that the difference is \ + ACCEPTABLE — the waiver self-invalidates as stale the moment it stops \ + reproducing, so this is not a way to make a difference go away." + ); + } else { + eprintln!( + "scaffolded a promotion skeleton for {finding_id}; stdout only — nothing was written. \ + Every `{SCAFFOLD_SENTINEL}` field carries the sentinel the registry loader rejects. \ + Author the behavior's three axes by hand: a finding tells you what DIFFERS, not \ + whether deacon is wrong, the reference is wrong, or the spec is silent — that is the \ + review. Then copy the fixture, author the case with a FULL `scenarioContext` (V26), \ + and flip the `odp-cmb-*` dispositions it covers off `gap` in the same commit." + ); + } + 0 +} + // --------------------------------------------------------------------------- // coverage (024-deterministic-conformance-coverage) // --------------------------------------------------------------------------- diff --git a/crates/conformance/src/discovery/corpus.rs b/crates/conformance/src/discovery/corpus.rs new file mode 100644 index 00000000..9edebfe1 --- /dev/null +++ b/crates/conformance/src/discovery/corpus.rs @@ -0,0 +1,504 @@ +//! Real-world corpus manifest (`cor-`) — `conformance/discovery/corpus.json` +//! (025-exploratory-parity-discovery, data-model.md § 8, US7). +//! +//! The manifest is Rust-owned strict JSON rather than a Python tuple so the +//! immutable-reference check (**D4**) runs **hermetically**, on every pull request, +//! without network access: a validation that only runs when the network is up is a +//! validation that does not run (research D8). Corpus *content* is never vendored — this +//! file records provenance, not bytes (FR-053). +//! +//! ## What **D4** actually asserts +//! +//! Two clauses, checked in two different places because they are answerable in two +//! different places: +//! +//! 1. **Non-immutable reference** (FR-050) — a branch, a moving tag, `HEAD`, `latest`, an +//! abbreviated SHA, or anything else that is not a 40-hex object name. This is a +//! property of the manifest *alone*, so [`check`] answers it from a single load, +//! hermetically, with no network and no history. That is the whole reason the manifest +//! moved into Rust. +//! 2. **A digest recorded and then removed** — a `contentDigest` that was `sha256:…` and +//! is now `null`. This is a property of a *change*, not of a file, so no single-load +//! check can see it. [`check_drift`] answers it against an explicit baseline, and the +//! fetch path (`parity_harness::discovery::corpus_fetch`) is the caller that has one: +//! it holds the committed manifest while it materializes the new digests. +//! +//! Splitting them is deliberate rather than an omission. Folding clause 2 into [`check`] +//! would require the checker to reconstruct history — from git, or from a second committed +//! copy — and both make a hermetic validator depend on something outside the file it +//! validates. +//! +//! ## Why the id is derived +//! +//! `cor-` (data-model.md § 0). Derivation makes two +//! entries naming the same upstream workspace *unrepresentable*: they collide on id, and +//! the check rejects the duplicate. A hand-chosen id would let one snapshot be fetched, +//! digested, and compared twice under two names, and a divergence found through both would +//! look like two. + +use std::path::Path; + +use serde::{Deserialize, Serialize}; + +use crate::DiscoveryError; +use crate::discovery::hash8; + +/// Length of a git object name in hex characters. +const OBJECT_NAME_LEN: usize = 40; + +/// The `sha256:` prefix every recorded content digest carries. +pub const DIGEST_PREFIX: &str = "sha256:"; + +/// Length of the hex body of a SHA-256 digest. +const DIGEST_BODY_LEN: usize = 64; + +/// One pinned third-party workspace (data-model.md § 8). +/// +/// Field order here is the emitted JSON field order, so recording a digest produces a +/// reviewable one-line diff rather than a reformat. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CorpusEntry { + /// Derived, never authored: `cor-`. + pub id: String, + /// The short human name the entry is referred to by (`try-node`, `oss-ruff`, …), and + /// the name its frozen `realworld::` baseline unit carries. + pub name: String, + /// `owner/repo` on GitHub. + pub repository: String, + /// The **immutable** commit: a 40-hex object name. A branch, a tag, `HEAD`, or + /// `latest` is **D4** (FR-050). + pub commit: String, + /// The workspace root within the repository. Empty means the repository root. + pub path: String, + /// `sha256:<64-hex>` over the materialized workspace, or `null` until the entry has + /// been materialized once. Verified on every later fetch (FR-051). + pub content_digest: Option, + /// Why this workspace was selected, and anything about its shape a reader needs. + pub notes: String, +} + +impl CorpusEntry { + /// The derived id: `cor-`. + /// + /// Those three parts are exactly what identifies an upstream workspace snapshot. + /// `name` is deliberately **not** a part: renaming an entry for clarity must not + /// re-key it, or the rename would read as one snapshot removed and another added. + pub fn derive_id(repository: &str, commit: &str, path: &str) -> String { + format!("cor-{}", hash8(&[repository, commit, path])) + } + + /// This entry's id as derived from its own substance. + pub fn derived_id(&self) -> String { + CorpusEntry::derive_id(&self.repository, &self.commit, &self.path) + } + + /// The directory this entry materializes into, under `root`. + /// + /// Keyed on the **id** rather than the name so two entries can never share a + /// directory: the id is a function of the upstream identity, the name is a label. + pub fn workspace_dir(&self, root: &Path) -> std::path::PathBuf { + root.join(&self.id) + } +} + +/// Whether `reference` is an immutable git object name (FR-050). +/// +/// Exactly 40 **lowercase** hex characters. Uppercase is rejected rather than folded: +/// git renders object names in lowercase, so an uppercase spelling means the value was +/// retyped or transformed by hand, and a manifest whose pins have been retyped is exactly +/// the one worth looking at twice. Everything else — `main`, `v1.2.3`, `HEAD`, `latest`, +/// an abbreviated SHA — is mutable or ambiguous and therefore refused. +pub fn is_immutable_reference(reference: &str) -> bool { + reference.len() == OBJECT_NAME_LEN + && reference + .bytes() + .all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()) +} + +/// Whether `digest` is a well-formed `sha256:<64 lowercase hex>` value. +pub fn is_well_formed_digest(digest: &str) -> bool { + let Some(body) = digest.strip_prefix(DIGEST_PREFIX) else { + return false; + }; + body.len() == DIGEST_BODY_LEN + && body + .bytes() + .all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()) +} + +/// `sha256:<64-hex>` over a materialized workspace tree, keyed by workspace-relative +/// POSIX path. +/// +/// Lives here rather than beside the fetch so the digest **format** and the digest +/// **computation** are one definition: [`is_well_formed_digest`] and this function must +/// never be able to disagree about what a recorded digest looks like. +/// +/// Every part is **length-prefixed** before hashing, for the same reason +/// [`hash8`] is: a path and a payload concatenated without a length are not injective, so +/// two different trees could digest identically — and a verification that cannot +/// distinguish them verifies nothing. A separator byte is not enough here either, because +/// file *contents* are arbitrary bytes and can contain any separator. +pub fn digest_of(content: &std::collections::BTreeMap>) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update((content.len() as u64).to_le_bytes()); + for (path, bytes) in content { + hasher.update((path.len() as u64).to_le_bytes()); + hasher.update(path.as_bytes()); + hasher.update((bytes.len() as u64).to_le_bytes()); + hasher.update(bytes); + } + format!("{DIGEST_PREFIX}{:x}", hasher.finalize()) +} + +/// The `corpus.json` envelope. `records` is mandatory for the same reason as the findings +/// queue's: a truncated file must not load as "this repository pins no corpus". +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CorpusFile { + /// Schema version of the file format — rejected at load unless it is the current + /// [`SCHEMA_VERSION`](crate::discovery::queue::SCHEMA_VERSION). + #[serde(deserialize_with = "supported_schema_version")] + pub schema_version: u32, + /// The pinned entries, in file order. + pub records: Vec, +} + +impl Default for CorpusFile { + fn default() -> Self { + CorpusFile { + schema_version: crate::discovery::queue::SCHEMA_VERSION, + records: Vec::new(), + } + } +} + +fn supported_schema_version<'de, D>(de: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let value = u32::deserialize(de)?; + let current = crate::discovery::queue::SCHEMA_VERSION; + if value != current { + return Err(serde::de::Error::custom(format!( + "unsupported schemaVersion {value}: this build reads and writes version \ + {current}, and writing would stamp {current} over it" + ))); + } + Ok(value) +} + +/// Render a corpus file in its canonical, byte-stable form (2-space pretty JSON, +/// trailing newline) — the same rendering every machine-touched artifact in this crate +/// uses, so recording a digest produces a reviewable one-line diff. +pub fn render(file: &CorpusFile) -> String { + let mut out = serde_json::to_string_pretty(file) + .unwrap_or_else(|e| unreachable!("corpus record serialization is infallible: {e}")); + out.push('\n'); + out +} + +/// Atomically write the corpus manifest to `dir/corpus.json`. +/// +/// Delegates to the single [`crate::atomic_write`] primitive (unique temp file + +/// `fs::rename`): a shorter payload written over a longer one must never leave trailing +/// bytes. +pub fn write(dir: &Path, entries: &[CorpusEntry]) -> std::io::Result<()> { + let file = CorpusFile { + schema_version: crate::discovery::queue::SCHEMA_VERSION, + records: entries.to_vec(), + }; + crate::atomic_write(&crate::discovery::queue::corpus_path(dir), &render(&file)) +} + +/// **D4** over a loaded manifest — everything answerable from the file alone. +/// +/// Four shapes, one class, because all four are the same defect: an entry that does not +/// name a retrievable, verifiable snapshot. +/// +/// - a `commit` that is not a 40-hex object name (FR-050); +/// - a `contentDigest` that is present but not `sha256:<64-hex>` — a malformed digest is +/// not a weaker check, it is one that can never disagree; +/// - an `id` that does not derive from `repository ‖ commit ‖ path`, which detaches the +/// record from the snapshot it claims to identify; +/// - a duplicate id or name, which makes one snapshot two entries. +pub fn check(entries: &[CorpusEntry]) -> Vec { + let mut out = Vec::new(); + let mut seen_ids: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new(); + let mut seen_names: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new(); + + for entry in entries { + if !is_immutable_reference(&entry.commit) { + out.push(DiscoveryError::CorpusIntegrity { + record: entry.id.clone(), + cause: format!( + "`commit` is `{}`, which is not a 40-character lowercase-hex object \ + name. A branch, a tag, `HEAD`, `latest`, or an abbreviated SHA names \ + different content tomorrow, so a finding recorded against it is a \ + claim about content nobody can retrieve", + entry.commit + ), + }); + } + if let Some(digest) = &entry.content_digest + && !is_well_formed_digest(digest) + { + out.push(DiscoveryError::CorpusIntegrity { + record: entry.id.clone(), + cause: format!( + "`contentDigest` is `{digest}`, which is not `{DIGEST_PREFIX}<64 \ + lowercase hex>`. A malformed digest is not a weaker verification, it \ + is one that can never disagree" + ), + }); + } + let derived = entry.derived_id(); + if entry.id != derived { + out.push(DiscoveryError::CorpusIntegrity { + record: entry.id.clone(), + cause: format!( + "id does not match its substance (expected `{derived}` from `{}` ‖ \ + `{}` ‖ `{}`)", + entry.repository, entry.commit, entry.path + ), + }); + } + if !seen_ids.insert(entry.id.as_str()) { + out.push(DiscoveryError::CorpusIntegrity { + record: entry.id.clone(), + cause: "duplicate corpus entry id — every by-id lookup takes the first \ + match, so the second record would be fetched, digested, and \ + compared under an identity nothing resolves to" + .to_string(), + }); + } + if !seen_names.insert(entry.name.as_str()) { + out.push(DiscoveryError::CorpusIntegrity { + record: entry.id.clone(), + cause: format!( + "duplicate corpus entry name `{}` — the name is what a reviewer and \ + the frozen `realworld::` baseline units refer to, and two \ + entries under one name make that reference ambiguous", + entry.name + ), + }); + } + } + out +} + +/// **D4**'s second clause: a digest that was recorded and then removed. +/// +/// Answerable only against a baseline, so it is a separate entry point that takes one +/// explicitly (see this module's header). `previous` is the manifest as committed; +/// `current` is the manifest about to be written. +/// +/// Re-recording a *different* digest is deliberately **not** flagged here — that is the +/// FR-051 mismatch, which the fetch reports for the entry against real content. Silently +/// dropping the digest is different in kind: it does not disagree with anything, it +/// deletes the thing a later fetch would have disagreed with. +pub fn check_drift(previous: &[CorpusEntry], current: &[CorpusEntry]) -> Vec { + let mut out = Vec::new(); + for before in previous { + let Some(digest) = &before.content_digest else { + continue; + }; + let Some(after) = current.iter().find(|e| e.id == before.id) else { + // A removed entry is a deliberate re-pin, not a lost digest: the id is a + // function of the commit, so re-pinning necessarily removes the old record, + // and the digest goes with the snapshot it described. + continue; + }; + if after.content_digest.is_none() { + out.push(DiscoveryError::CorpusIntegrity { + record: before.id.clone(), + cause: format!( + "`contentDigest` was recorded as `{digest}` and is now null. A digest \ + is recorded once, at first materialization, and verified on every \ + later fetch; removing it does not weaken the check, it deletes the \ + only thing a later fetch could have disagreed with" + ), + }); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(repository: &str, commit: &str, path: &str) -> CorpusEntry { + CorpusEntry { + id: CorpusEntry::derive_id(repository, commit, path), + name: format!("{repository}@{path}"), + repository: repository.to_string(), + commit: commit.to_string(), + path: path.to_string(), + content_digest: None, + notes: String::new(), + } + } + + const SHA: &str = "31b61b521d55926d62c748b659f24ae71774c0e3"; + + #[test] + fn only_a_40_hex_lowercase_object_name_is_immutable() { + assert!(is_immutable_reference(SHA)); + // Every floating reference FR-050 names, plus the near-misses. + for mutable in [ + "main", + "master", + "HEAD", + "latest", + "v1.2.3", + "refs/heads/main", + "31b61b5", // abbreviated + "31B61B521D55926D62C748B659F24AE71774C0E3", // uppercase + "31b61b521d55926d62c748b659f24ae71774c0e", // 39 + "31b61b521d55926d62c748b659f24ae71774c0e33", // 41 + "31b61b521d55926d62c748b659f24ae71774c0g3", // non-hex + "", + ] { + assert!( + !is_immutable_reference(mutable), + "`{mutable}` must not pass as an immutable reference" + ); + } + } + + #[test] + fn a_mutable_reference_is_d4() { + let e = entry("microsoft/vscode-remote-try-node", "main", ""); + let violations = check(std::slice::from_ref(&e)); + assert!( + violations + .iter() + .any(|v| v.class() == "D4" + && v.to_string().contains("not a 40-character lowercase-hex")), + "a branch reference must be D4: {violations:?}" + ); + } + + #[test] + fn a_pinned_entry_with_no_digest_is_clean() { + // `contentDigest: null` is the state of every entry before its first + // materialization. It must not be a violation, or the manifest could never be + // authored in the first place. + assert!(check(&[entry("devcontainers/images", SHA, "src/go")]).is_empty()); + } + + #[test] + fn a_malformed_digest_is_d4() { + let mut e = entry("devcontainers/images", SHA, "src/go"); + e.content_digest = Some("deadbeef".to_string()); + assert!( + check(std::slice::from_ref(&e)) + .iter() + .any(|v| v.class() == "D4" && v.to_string().contains("contentDigest")) + ); + e.content_digest = Some(format!("{DIGEST_PREFIX}{}", "a".repeat(64))); + assert!(check(std::slice::from_ref(&e)).is_empty()); + } + + #[test] + fn a_hand_chosen_id_is_d4() { + let mut e = entry("devcontainers/images", SHA, "src/go"); + e.id = "cor-deadbeef".to_string(); + assert!( + check(std::slice::from_ref(&e)).iter().any( + |v| v.class() == "D4" && v.to_string().contains("does not match its substance") + ) + ); + } + + #[test] + fn two_entries_naming_one_snapshot_collide_on_id() { + let a = entry("devcontainers/images", SHA, "src/go"); + let b = entry("devcontainers/images", SHA, "src/go"); + assert_eq!(a.id, b.id); + assert!( + check(&[a, b]) + .iter() + .any(|v| v.to_string().contains("duplicate corpus entry id")) + ); + } + + #[test] + fn a_removed_digest_is_d4_against_the_baseline() { + let mut before = entry("devcontainers/images", SHA, "src/go"); + before.content_digest = Some(format!("{DIGEST_PREFIX}{}", "b".repeat(64))); + let after = entry("devcontainers/images", SHA, "src/go"); + + let violations = check_drift(std::slice::from_ref(&before), std::slice::from_ref(&after)); + assert!( + violations + .iter() + .any(|v| v.class() == "D4" && v.to_string().contains("is now null")), + "removing a recorded digest must be D4: {violations:?}" + ); + + // Re-recording the SAME digest, or a different one, is not this clause: a + // disagreement is the FR-051 mismatch the fetch reports against real content. + let mut same = after.clone(); + same.content_digest = before.content_digest.clone(); + assert!(check_drift(std::slice::from_ref(&before), &[same]).is_empty()); + let mut different = after.clone(); + different.content_digest = Some(format!("{DIGEST_PREFIX}{}", "c".repeat(64))); + assert!(check_drift(std::slice::from_ref(&before), &[different]).is_empty()); + + // A re-pin REMOVES the entry (its id is a function of the commit), and the digest + // goes with the snapshot it described. + assert!(check_drift(std::slice::from_ref(&before), &[]).is_empty()); + } + + #[test] + fn the_id_ignores_the_name_but_not_the_upstream_identity() { + let a = entry("devcontainers/images", SHA, "src/go"); + let mut renamed = a.clone(); + renamed.name = "something-else".to_string(); + assert_eq!(a.id, renamed.derived_id(), "a rename must not re-key"); + + assert_ne!(a.id, entry("devcontainers/images", SHA, "src/rust").id); + assert_ne!(a.id, entry("devcontainers/templates", SHA, "src/go").id); + assert_ne!( + a.id, + entry( + "devcontainers/images", + "0000000000000000000000000000000000000000", + "src/go" + ) + .id + ); + } + + #[test] + fn the_rendered_file_is_byte_stable_and_newline_terminated() { + let file = CorpusFile { + schema_version: crate::discovery::queue::SCHEMA_VERSION, + records: vec![entry("devcontainers/images", SHA, "src/go")], + }; + let once = render(&file); + assert_eq!(once, render(&file)); + assert!(once.ends_with("}\n")); + let round: CorpusFile = serde_json::from_str(&once).expect("round-trips"); + assert_eq!(round, file); + } + + #[test] + fn an_unknown_field_and_a_future_schema_version_are_both_refused() { + let unknown = serde_json::json!({ + "schemaVersion": crate::discovery::queue::SCHEMA_VERSION, + "records": [], + "extra": true + }); + assert!(serde_json::from_value::(unknown).is_err()); + + let future = serde_json::json!({ + "schemaVersion": crate::discovery::queue::SCHEMA_VERSION + 1, + "records": [] + }); + assert!(serde_json::from_value::(future).is_err()); + } +} diff --git a/crates/conformance/src/discovery/generate.rs b/crates/conformance/src/discovery/generate.rs new file mode 100644 index 00000000..993062a4 --- /dev/null +++ b/crates/conformance/src/discovery/generate.rs @@ -0,0 +1,1357 @@ +//! Constrained candidate generation from the pinned grammar +//! (025-exploratory-parity-discovery, T030, FR-006/FR-007/FR-008a/FR-011/FR-012). +//! +//! Draws from [`super::grammar`] so that `required` keys are satisfied for **valid** +//! candidates and violated deliberately for **near-valid** ones — the distinction the +//! `required` constraint kind exists to make (research D1). Nothing here is hand-authored +//! schema knowledge: the branch set comes from the inventory's `union-alternative` shape, +//! the property set from its `property-existence` units, the value domain from its `type` +//! and `enum` units, and the valid/near-valid line from its `required` units. +//! +//! ## Why the grammar rather than a hand-written generator +//! +//! A hand-authored grammar generates the shapes its author thought of, which is exactly +//! what the curated fixtures already do and exactly the maintainer imagination this +//! feature exists to escape (research D1). Drawing from the committed inventory also +//! means a re-vendored schema pin changes `grammarVersion`, which correctly invalidates +//! every finding bound to the old value with no separate bookkeeping. +//! +//! ## Three candidate kinds, one stream +//! +//! | Kind | Source | What it probes | +//! |---|---|---| +//! | [`CandidateKind::Valid`] | grammar draw satisfying one branch's `required` set | agreement on inputs both implementations should accept | +//! | [`CandidateKind::NearValid`] | the same draw with one `required` key removed | agreement on *rejection*, the place strictness divergences live | +//! | [`CandidateKind::MutatedFixture`] | a committed fixture plus mutation operators | the near-miss neighbourhood of a document known to work | +//! +//! The mutation seed corpus is the **committed deterministic fixtures only** (FR-008a), +//! embedded with `include_str!` so generation is reproducible from the recorded seed and +//! pinned input set with no filesystem or network access at all. The real-world corpus is +//! deliberately not a seed source: it would make the candidate stream depend on a fetch. +//! +//! ## Category coverage is structural, not statistical +//! +//! The primary mutation category cycles round-robin through a seed-shuffled permutation of +//! the eleven, and the generator retries across base documents until that category has a +//! target. SC-003 ("every declared category applied at least once") is therefore a +//! property of the schedule rather than a bet on a long-enough run — a campaign that ends +//! early still covers every category it had budget to reach, and a category reported as +//! never applied is a real generation deficiency rather than a short-run artifact. + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value, json}; + +use super::grammar::Grammar; +use super::hash8; +use super::mutate::{self, Mutation, MutationCategory}; +use super::rng::{Prng, prng_identity}; +use super::shrink::reduction_catalogue_identity; +use crate::model::ConstraintKind; + +/// The revision of *this module's* derivation — bumped whenever a draw could move. +/// +/// Distinct from the PRNG's own version: the stream can stay identical while the order in +/// which the generator consumes it changes, and both determine the candidate sequence. +pub const GENERATOR_VERSION: u32 = 1; + +/// The `generatorVersion` element of a campaign's pinned input set (data-model.md § 4). +/// +/// Covers the two things that determine output but are neither a grammar nor a mutation: +/// the pseudorandom stream's algorithm identity (FR-001 depends on it) and the reduction +/// catalogue's *order* (FR-020 depends on it). Folding either into +/// `mutationCatalogVersion` would name it for something it is not, so a deliberate change +/// to reduction order would look like a change to the mutation operators. +pub fn generator_identity() -> String { + format!( + "{}+{}+generator/v{GENERATOR_VERSION}", + prng_identity(), + reduction_catalogue_identity() + ) +} + +// --------------------------------------------------------------------------- +// Grammar anchors +// --------------------------------------------------------------------------- + +/// Properties shared by every branch (`devContainerCommon`). +const GROUP_COMMON: &str = "/definitions/devContainerCommon"; +/// Properties shared by the non-Compose branches (`nonComposeBase`). +const GROUP_NON_COMPOSE: &str = "/definitions/nonComposeBase"; +/// The `imageContainer` branch. +const GROUP_IMAGE: &str = "/definitions/imageContainer"; +/// The `composeContainer` branch. +const GROUP_COMPOSE: &str = "/definitions/composeContainer"; +/// The `dockerfileContainer` branch's canonical (nested `build`) spelling. +const GROUP_DOCKERFILE: &str = "/definitions/dockerfileContainer/oneOf/0"; +/// The nested `build` object of the canonical Dockerfile spelling. +const GROUP_BUILD: &str = "/definitions/dockerfileContainer/oneOf/0/properties/build/allOf/0"; + +/// The three configuration-source branches the schema's `oneOf` declares. +/// +/// Named rather than derived from the `union-alternative` units because the *names* are +/// what a candidate and a finding report; the branch **contents** — which keys exist, +/// which are required, what types they hold — all come from the grammar. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConfigBranch { + /// `image`. + Image, + /// `build.dockerfile` (the canonical containers.dev spelling). + Dockerfile, + /// `dockerComposeFile` + `service` + `workspaceFolder`. + Compose, +} + +impl ConfigBranch { + /// Every branch, in declaration order. + pub fn all() -> &'static [ConfigBranch; 3] { + &[ + ConfigBranch::Image, + ConfigBranch::Dockerfile, + ConfigBranch::Compose, + ] + } + + /// The stable wire spelling. + pub fn as_str(self) -> &'static str { + match self { + ConfigBranch::Image => "image", + ConfigBranch::Dockerfile => "dockerfile", + ConfigBranch::Compose => "compose", + } + } + + /// The grammar groups whose properties this branch may carry, in draw order. + fn groups(self) -> &'static [&'static str] { + match self { + ConfigBranch::Image => &[GROUP_COMMON, GROUP_NON_COMPOSE, GROUP_IMAGE], + ConfigBranch::Dockerfile => &[GROUP_COMMON, GROUP_NON_COMPOSE, GROUP_DOCKERFILE], + ConfigBranch::Compose => &[GROUP_COMMON, GROUP_COMPOSE], + } + } +} + +// --------------------------------------------------------------------------- +// The committed mutation seed corpus (FR-008a) +// --------------------------------------------------------------------------- + +/// One committed fixture, embedded at compile time. +/// +/// `include_str!` rather than a filesystem read so generation needs no I/O at all: FR-008a +/// requires the seed corpus to be the committed deterministic fixtures, and a compile-time +/// embed makes "committed" and "deterministic" properties of the build rather than of the +/// working tree. +struct SeedFixture { + name: &'static str, + raw: &'static str, +} + +/// The committed seed corpus. Plain-JSON fixtures only — the two JSONC fixtures +/// (`fixtures/config/{basic,with-variables}/devcontainer.jsonc`) are deliberately absent: +/// parsing them here would need a second JSONC parser, and a second parser is a second +/// opinion on what a document says (the same argument FR-015 makes about normalization). +const SEED_FIXTURES: &[SeedFixture] = &[ + SeedFixture { + name: "config-image-reference", + raw: include_str!("../../../../fixtures/config/build/image-reference/devcontainer.json"), + }, + SeedFixture { + name: "config-compose-service-target", + raw: include_str!( + "../../../../fixtures/config/build/compose-service-target/devcontainer.json" + ), + }, + SeedFixture { + name: "config-compose-multiservice", + raw: include_str!( + "../../../../fixtures/config/compose-multiservice/.devcontainer/devcontainer.json" + ), + }, + SeedFixture { + name: "up-single-container", + raw: include_str!( + "../../../../fixtures/devcontainer-up/single-container/devcontainer.json" + ), + }, + SeedFixture { + name: "up-feature-and-dotfiles", + raw: include_str!( + "../../../../fixtures/devcontainer-up/feature-and-dotfiles/devcontainer.json" + ), + }, +]; + +/// The seed corpus, parsed. A fixture that stops parsing is a hard error rather than a +/// silently smaller corpus: a corpus that shrinks without anyone noticing explores less +/// and reports the same "found nothing". +fn seed_corpus() -> Vec<(&'static str, Value)> { + SEED_FIXTURES + .iter() + .map(|f| { + let value: Value = serde_json::from_str(f.raw).unwrap_or_else(|e| { + unreachable!( + "committed seed fixture `{}` no longer parses as JSON: {e}. It is \ + embedded with include_str!, so this is a compile-time-committed \ + document that changed shape.", + f.name + ) + }); + (f.name, value) + }) + .collect() +} + +/// The names of the committed seed fixtures, for reporting and tests. +pub fn seed_fixture_names() -> Vec<&'static str> { + SEED_FIXTURES.iter().map(|f| f.name).collect() +} + +// --------------------------------------------------------------------------- +// Candidates +// --------------------------------------------------------------------------- + +/// One consumer-surface invocation a candidate is executed under. +/// +/// `${WORKSPACE}` is the same token the declarative conformance runner uses, so a +/// candidate's operations read the same way a case's do. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Operation { + /// The consumer subcommand (`read-configuration`, `up`, …). + pub subcommand: String, + /// Its argv, `${WORKSPACE}`-tokenized. + pub argv: Vec, +} + +impl Operation { + /// The configuration-resolution operation every hermetic-tier candidate runs. + pub fn read_configuration() -> Operation { + Operation { + subcommand: "read-configuration".to_string(), + argv: vec![ + "read-configuration".to_string(), + "--workspace-folder".to_string(), + "${WORKSPACE}".to_string(), + ], + } + } +} + +/// How a candidate's **base document** was produced. +/// +/// It describes the base, not the final document: a [`CandidateKind::Valid`] draw that +/// then had a mutation operator applied is very likely no longer valid. That is deliberate +/// — the kind and [`Candidate::mutations`] are two independent facts, and collapsing them +/// into one label would lose the ability to say "this started from a valid draw" — but it +/// means a consumer asking "was this input deliberately malformed?" must consult **both** +/// (see `parity_harness::discovery::campaign`, which does). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum CandidateKind { + /// A grammar draw satisfying its branch's `required` set. + Valid, + /// A grammar draw with one `required` key deliberately removed. + NearValid, + /// A committed fixture used as a mutation base. + MutatedFixture, +} + +impl CandidateKind { + /// The stable wire spelling. + pub fn as_str(self) -> &'static str { + match self { + CandidateKind::Valid => "valid", + CandidateKind::NearValid => "near-valid", + CandidateKind::MutatedFixture => "mutated-fixture", + } + } +} + +/// A generated candidate: the document, its operations, and its full provenance. +#[derive(Debug, Clone, PartialEq)] +pub struct Candidate { + /// `cnd-` over `canonical(document) ‖ canonical(operations)`. + pub id: String, + /// The candidate's position in the seed's stream, from zero. + pub index: u64, + /// How it was produced. + pub kind: CandidateKind, + /// The branch it draws from, when it is a grammar draw. + pub branch: Option, + /// The committed fixture it mutates, when it is a mutated fixture. + pub fixture: Option<&'static str>, + /// The configuration document. + pub document: Value, + /// Every mutation operator applied, in application order (FR-009 attribution). + pub mutations: Vec, + /// The `required` keys deliberately removed, when near-valid. + pub violated_required: Vec, + /// The ordered operations it is executed under. + pub operations: Vec, +} + +impl Candidate { + /// The `mop-` operator ids this candidate carries, in application order. + pub fn operator_ids(&self) -> Vec { + self.mutations.iter().map(|m| m.operator.clone()).collect() + } + + /// Derive the `cnd-` id from the document and operations. + /// + /// The canonical form is `serde_json`'s own rendering, which **preserves key order** + /// (this crate enables `preserve_order`). Sorting keys first would be wrong here, not + /// merely different: `mop-ordering-change` deliberately produces documents that differ + /// only in the order of a declaration-ordered collection, and a key-sorting canonical + /// form would give those the same id — collapsing exactly the candidates that category + /// exists to explore. + pub fn derive_id(document: &Value, operations: &[Operation]) -> String { + let doc = serde_json::to_string(document) + .unwrap_or_else(|e| unreachable!("a candidate document always serializes: {e}")); + let ops = serde_json::to_string(operations) + .unwrap_or_else(|e| unreachable!("candidate operations always serialize: {e}")); + format!("cnd-{}", hash8(&[&doc, &ops])) + } +} + +// --------------------------------------------------------------------------- +// The generator +// --------------------------------------------------------------------------- + +/// The deterministic candidate stream for one seed. +/// +/// Two generators built from the same seed and the same grammar produce the identical +/// ordered sequence forever — the property SC-001 verifies end to end and FR-001 requires. +#[derive(Debug)] +pub struct Generator<'a> { + grammar: &'a Grammar, + prng: Prng, + /// The seed-shuffled category schedule; candidate `n` gets `schedule[n % 11]` as its + /// primary category. + schedule: Vec, + seeds: Vec<(&'static str, Value)>, + produced: u64, +} + +impl<'a> Generator<'a> { + /// Build a generator for `seed` over `grammar`. + pub fn new(grammar: &'a Grammar, seed: u64) -> Generator<'a> { + let mut prng = Prng::from_seed(seed); + let mut schedule: Vec = MutationCategory::all().to_vec(); + prng.shuffle(&mut schedule); + Generator { + grammar, + prng, + schedule, + seeds: seed_corpus(), + produced: 0, + } + } + + /// The next candidate in the stream. + /// + /// Always yields: a generator that could return `None` would let a campaign end early + /// for a reason nobody recorded, and "the generator stopped" would be indistinguishable + /// from "the budget ran out". + pub fn next_candidate(&mut self) -> Candidate { + let index = self.produced; + self.produced += 1; + + // The primary category cycles round-robin, so eleven consecutive candidates cover + // the whole catalogue regardless of how the draws fall (SC-003). + let primary = self.schedule[(index as usize) % self.schedule.len()]; + + // A base document: a committed fixture two candidates in three, a grammar draw + // otherwise. Mutating a document known to work probes the near-miss neighbourhood; + // drawing from the grammar reaches shapes no fixture author wrote. + let use_fixture = self.prng.next_bounded(3).unwrap_or(0) < 2; + let (mut document, mut kind, mut branch, mut fixture, mut violated) = if use_fixture { + let (name, doc) = self.draw_fixture(); + ( + doc, + CandidateKind::MutatedFixture, + None, + Some(name), + Vec::new(), + ) + } else { + let branch = *self + .prng + .choose(ConfigBranch::all()) + .unwrap_or(&ConfigBranch::Image); + let document = self.draw_document(branch); + ( + document, + CandidateKind::Valid, + Some(branch), + None, + Vec::new(), + ) + }; + + // One candidate in four is near-valid: a `required` key the grammar names is + // removed. This is the line the `required` constraint kind draws, and it is where + // the strictness divergences between the two implementations live. + if self.prng.next_bounded(4) == Some(0) { + if let Some(removed) = self.violate_required(&mut document, branch) { + kind = CandidateKind::NearValid; + violated.push(removed); + } + } + + // Apply the primary category, retrying across base documents when it has no + // target here — so the schedule's coverage guarantee survives a document that + // happens not to suit it. + let mut mutations = Vec::new(); + match mutate::apply(primary, &document, &mut self.prng) { + Some(applied) => { + document = applied.document; + mutations.push(applied.mutation); + } + None => { + if let Some(applied) = self.retry_primary(primary) { + document = applied.0; + fixture = applied.2; + branch = applied.3; + kind = if fixture.is_some() { + CandidateKind::MutatedFixture + } else { + CandidateKind::Valid + }; + violated.clear(); + mutations.push(applied.1); + } + } + } + + // A second, freely drawn operator on some candidates, so operator *interactions* + // are reachable — a defect that needs two mutations to surface is invisible to a + // stream that only ever applies one. + if self.prng.next_bool() { + let extra = *self + .prng + .choose(MutationCategory::all().as_slice()) + .unwrap_or(&primary); + if let Some(applied) = mutate::apply(extra, &document, &mut self.prng) { + document = applied.document; + mutations.push(applied.mutation); + } + } + + let operations = vec![Operation::read_configuration()]; + Candidate { + id: Candidate::derive_id(&document, &operations), + index, + kind, + branch, + fixture, + document, + mutations, + violated_required: violated, + operations, + } + } + + /// Draw a committed fixture. + fn draw_fixture(&mut self) -> (&'static str, Value) { + let index = self.prng.next_index(self.seeds.len()).unwrap_or(0); + let (name, doc) = &self.seeds[index]; + (name, doc.clone()) + } + + /// Try every base document in turn until `primary` applies to one. + /// + /// Returns the mutated document, the mutation, the fixture name (if the base was a + /// fixture), and the branch (if it was a grammar draw). Bases are tried in a fixed + /// order — fixtures first, then each branch — so the retry is as reproducible as the + /// first attempt. + #[allow(clippy::type_complexity)] + fn retry_primary( + &mut self, + primary: MutationCategory, + ) -> Option<(Value, Mutation, Option<&'static str>, Option)> { + for index in 0..self.seeds.len() { + let (name, base) = self.seeds[index].clone(); + if let Some(applied) = mutate::apply(primary, &base, &mut self.prng) { + return Some((applied.document, applied.mutation, Some(name), None)); + } + } + for branch in ConfigBranch::all() { + let base = self.draw_document(*branch); + if let Some(applied) = mutate::apply(primary, &base, &mut self.prng) { + return Some((applied.document, applied.mutation, None, Some(*branch))); + } + } + None + } + + /// Draw a document satisfying `branch`'s `required` set, plus a random subset of the + /// optional properties the grammar declares for its groups. + pub fn draw_document(&mut self, branch: ConfigBranch) -> Value { + let mut object = Map::new(); + + // Every configuration carries a name: it is the one property that makes a + // generated document legible in a failure message. + object.insert( + "name".to_string(), + Value::String(format!("discovery-{}", branch.as_str())), + ); + + // Required first, so an optional draw can never displace one. + for key in required_keys(self.grammar, branch) { + let value = self.draw_property(&key, branch); + object.insert(key, value); + } + + // Then a subset of the optional properties. Roughly a third are included, which + // keeps documents small enough to read and large enough to interact. + for spec in optional_properties(self.grammar, branch) { + if self.prng.next_bounded(3) != Some(0) { + continue; + } + if object.contains_key(&spec.name) || !is_generatable(&spec.name) { + continue; + } + if let Some(value) = self.draw_from_spec(&spec) { + object.insert(spec.name.clone(), value); + } + } + + Value::Object(object) + } + + /// Remove one `required` key the grammar declares for the document's branch. + /// + /// Returns the removed key, or `None` when the document carries none (a fixture with + /// no branch, or a draw already missing them). Silently reporting a near-valid + /// candidate that violates nothing would make the kind a lie. + fn violate_required( + &mut self, + document: &mut Value, + branch: Option, + ) -> Option { + let object = document.as_object_mut()?; + let branches: Vec = match branch { + Some(b) => vec![b], + None => ConfigBranch::all().to_vec(), + }; + let mut present: Vec = branches + .iter() + .flat_map(|b| required_keys(self.grammar, *b)) + .filter(|k| object.contains_key(k)) + .collect(); + present.sort_unstable(); + present.dedup(); + let key = self.prng.choose(&present)?.clone(); + object.shift_remove(&key); + Some(key) + } + + /// Draw a value for a `required` key, preferring the grammar's own declaration. + fn draw_property(&mut self, key: &str, branch: ConfigBranch) -> Value { + for group in branch.groups() { + let pointer = format!("{group}/properties/{key}"); + if let Some(spec) = property_spec(self.grammar, &pointer, key) + && let Some(value) = self.draw_from_spec(&spec) + { + return value; + } + } + // `build` is the one required key whose value is itself an object with its own + // required set, so it is composed from the nested group rather than drawn flat. + if key == "build" { + return self.draw_build_object(); + } + Value::String(format!("discovery-{key}")) + } + + /// Compose the nested `build` object from its own grammar group. + fn draw_build_object(&mut self) -> Value { + let mut build = Map::new(); + for unit in self.grammar.of_kind(ConstraintKind::Required) { + let Some(rest) = unit + .pointer + .strip_prefix(&format!("{GROUP_BUILD}/required/")) + else { + continue; + }; + if rest.contains('/') { + continue; + } + let value = curated_values(rest) + .first() + .cloned() + .unwrap_or_else(|| Value::String(format!("discovery-{rest}"))); + build.insert(rest.to_string(), value); + } + if self.prng.next_bool() { + build.insert("context".to_string(), Value::String(".".to_string())); + } + Value::Object(build) + } + + /// Draw a value for one property spec: an `enum`/`const` value when the grammar + /// declares one, else a curated value for that property name, else a generic value of + /// one of its declared `type`s. + fn draw_from_spec(&mut self, spec: &PropertySpec) -> Option { + if !spec.allowed_values.is_empty() { + return self.prng.choose(&spec.allowed_values).cloned(); + } + let curated = curated_values(&spec.name); + if !curated.is_empty() { + return self.prng.choose(&curated).cloned(); + } + let ty = self.prng.choose(&spec.types)?.clone(); + Some(generic_value(&ty)) + } +} + +// --------------------------------------------------------------------------- +// Grammar projection +// --------------------------------------------------------------------------- + +/// One property as the grammar declares it. +#[derive(Debug, Clone, PartialEq)] +pub struct PropertySpec { + /// The property name. + pub name: String, + /// The schema pointer that declares it. + pub pointer: String, + /// Its declared JSON types (`type` units), possibly several. + pub types: Vec, + /// Its exact legal values, when the grammar declares an `enum` or a `const`. + pub allowed_values: Vec, +} + +/// The `required` keys the grammar declares for `branch`, sorted. +/// +/// Read from the inventory's `required` units rather than restated: the schema is the +/// authority on which keys a *valid* instance must carry, and restating it here would be +/// a second view of the pinned surface that could disagree with the first (research D1's +/// argument, applied to the valid/near-valid line specifically). +pub fn required_keys(grammar: &Grammar, branch: ConfigBranch) -> Vec { + let mut keys: Vec = Vec::new(); + for group in branch.groups() { + let prefix = format!("{group}/required/"); + for unit in grammar.of_kind(ConstraintKind::Required) { + let Some(rest) = unit.pointer.strip_prefix(&prefix) else { + continue; + }; + if rest.contains('/') { + continue; + } + if let Some(key) = unit.substance.get("required").and_then(Value::as_str) { + keys.push(key.to_string()); + } + } + } + keys.sort_unstable(); + keys.dedup(); + keys +} + +/// Every property `branch` may carry that is not `required`, sorted by name. +pub fn optional_properties(grammar: &Grammar, branch: ConfigBranch) -> Vec { + let required = required_keys(grammar, branch); + let mut specs: Vec = Vec::new(); + for group in branch.groups() { + let prefix = format!("{group}/properties/"); + for unit in grammar.of_kind(ConstraintKind::PropertyExistence) { + let Some(name) = unit.pointer.strip_prefix(&prefix) else { + continue; + }; + if name.contains('/') || required.iter().any(|r| r == name) { + continue; + } + if let Some(spec) = property_spec(grammar, &unit.pointer, name) { + specs.push(spec); + } + } + } + specs.sort_by(|a, b| a.name.cmp(&b.name)); + specs.dedup_by(|a, b| a.name == b.name); + specs +} + +/// Build a [`PropertySpec`] from the units at `pointer`, or `None` when the grammar +/// declares no type there (nothing to draw from). +fn property_spec(grammar: &Grammar, pointer: &str, name: &str) -> Option { + let mut types: Vec = Vec::new(); + for unit in grammar.at_pointer_of_kind(pointer, ConstraintKind::Type) { + match unit.substance.get("type") { + Some(Value::String(t)) => types.push(t.clone()), + Some(Value::Array(items)) => { + types.extend(items.iter().filter_map(Value::as_str).map(str::to_string)); + } + _ => {} + } + } + let mut allowed_values: Vec = Vec::new(); + for unit in grammar.at_pointer_of_kind(pointer, ConstraintKind::Enum) { + if let Some(Value::Array(items)) = unit.substance.get("enum") { + allowed_values.extend(items.iter().cloned()); + } + } + for unit in grammar.at_pointer_of_kind(pointer, ConstraintKind::Const) { + if let Some(v) = unit.substance.get("const") { + allowed_values.push(v.clone()); + } + } + if types.is_empty() && allowed_values.is_empty() { + return None; + } + types.sort_unstable(); + types.dedup(); + Some(PropertySpec { + name: name.to_string(), + pointer: pointer.to_string(), + types, + allowed_values, + }) +} + +/// Whether generation may emit this property at all. +/// +/// `initializeCommand` is excluded **by construction**: it executes on the developer's +/// host before any container sandboxing, and a machine-generated host command is exactly +/// the class deacon's workspace-trust gate exists to refuse (see `crates/core/src/trust.rs` +/// and SECURITY.md). Note that this is not the same as never producing one — the +/// `lifecycle-shape` mutation operator can still introduce it — which is deliberate: the +/// campaign's unsafe-candidate guard (FR-011) then discards and *counts* it, so the guard +/// is exercised by real traffic rather than being a branch nothing ever reaches. +/// +/// `$schema` and `additionalProperties` are excluded as pure noise: neither affects +/// configuration resolution, so a difference at either would be a finding about the echo +/// rather than about the tool. +fn is_generatable(name: &str) -> bool { + !matches!( + name, + "initializeCommand" | "$schema" | "additionalProperties" + ) +} + +/// Curated values for a property, chosen so a drawn document reaches configuration +/// resolution rather than dying at the document-syntax stage (FR-007, SC-002). +/// +/// These are **values within the grammar's declared type**, never additional structure: +/// the grammar decides what may appear and of what type; this decides which of the legal +/// values is worth drawing. A generic string for `image` would be legal and useless. +fn curated_values(name: &str) -> Vec { + match name { + "image" => vec![ + json!("alpine:3.19"), + json!("debian:bookworm-slim"), + json!("mcr.microsoft.com/devcontainers/base:ubuntu-22.04"), + ], + "dockerfile" | "dockerFile" => vec![json!("Dockerfile")], + "context" => vec![json!("."), json!("..")], + "build" => vec![json!({ "dockerfile": "Dockerfile" })], + "dockerComposeFile" => vec![ + json!("docker-compose.yml"), + json!(["docker-compose.yml"]), + json!(["docker-compose.yml", "docker-compose.override.yml"]), + ], + "service" => vec![json!("app")], + "runServices" => vec![json!([]), json!(["db"]), json!(["db", "cache"])], + "workspaceFolder" => vec![json!("/workspace"), json!("/workspaces/project")], + "workspaceMount" => vec![json!( + "source=${localWorkspaceFolder},target=/workspace,type=bind" + )], + "name" => vec![json!("discovery candidate")], + "remoteUser" | "containerUser" => vec![json!("root"), json!("vscode")], + "forwardPorts" => vec![json!([3000]), json!([3000, 8080]), json!(["db:5432"])], + "appPort" => vec![json!(3000), json!([3000, 8080]), json!("3000:3000")], + "runArgs" => vec![json!(["--init"]), json!(["--init", "--rm"])], + "capAdd" => vec![json!(["SYS_PTRACE"])], + "securityOpt" => vec![json!(["seccomp=unconfined"])], + "mounts" => vec![ + json!(["source=${localWorkspaceFolder}/.cache,target=/cache,type=bind"]), + json!([{ "source": "${localWorkspaceFolder}/.cache", "target": "/cache", "type": "bind" }]), + ], + "containerEnv" | "remoteEnv" => vec![ + json!({}), + json!({ "DISCOVERY": "1" }), + json!({ "DISCOVERY_PATH": "${containerEnv:PATH}" }), + ], + "features" => vec![ + json!({}), + json!({ "ghcr.io/devcontainers/features/git:1": {} }), + json!({ "ghcr.io/devcontainers/features/common-utils:2": { "installZsh": false } }), + ], + "overrideFeatureInstallOrder" => vec![ + json!([]), + json!(["ghcr.io/devcontainers/features/common-utils"]), + ], + "customizations" => vec![ + json!({}), + json!({ "vscode": { "extensions": ["rust-lang.rust-analyzer"] } }), + ], + "portsAttributes" => vec![ + json!({}), + json!({ "3000": { "label": "web", "onAutoForward": "notify" } }), + ], + "otherPortsAttributes" => vec![json!({}), json!({ "onAutoForward": "silent" })], + "hostRequirements" => vec![json!({}), json!({ "cpus": 2 })], + "secrets" => vec![json!({}), json!({ "TOKEN": { "description": "a token" } })], + "onCreateCommand" + | "updateContentCommand" + | "postCreateCommand" + | "postStartCommand" + | "postAttachCommand" => vec![ + json!("echo discovery"), + json!(["echo", "discovery"]), + json!({ "step": "echo discovery" }), + ], + _ => Vec::new(), + } +} + +/// A generic value of a declared JSON type, for a property with no curated pool. +fn generic_value(ty: &str) -> Value { + match ty { + "string" => json!("discovery"), + "boolean" => json!(true), + "integer" | "number" => json!(1), + "array" => json!([]), + "object" => json!({}), + _ => Value::Null, + } +} + +// --------------------------------------------------------------------------- +// FR-011 / FR-012 — safety and pinning predicates +// --------------------------------------------------------------------------- + +/// Why a candidate must not be executed (FR-011). +/// +/// Pure and hermetic so the guard is unit-testable without a campaign, and so the campaign +/// driver's only job is to *discard and count* — a guard whose decision lives inside the +/// driver can only be tested by running one. +pub fn unsafe_reasons(document: &Value, container_backed: bool) -> Vec { + let Some(object) = document.as_object() else { + return vec!["the document is not a JSON object".to_string()]; + }; + let mut reasons = Vec::new(); + + // `initializeCommand` executes on the developer's HOST before any container + // sandboxing. A generated host command is precisely what deacon's workspace-trust + // gate refuses, and a discovery campaign must never be the thing that runs one. + if object.contains_key("initializeCommand") { + reasons.push( + "carries `initializeCommand`, which executes on the host before any container \ + sandboxing — a machine-generated host command is never executed" + .to_string(), + ); + } + + if !container_backed { + // The remaining hazards are all about what a container would be granted. A + // configuration-resolution comparison starts no container, so flagging them there + // would discard candidates for a risk the tier does not take. + return reasons; + } + + if object.get("privileged") == Some(&Value::Bool(true)) { + reasons.push("requests `privileged: true`".to_string()); + } + if let Some(Value::Array(args)) = object.get("runArgs") { + for arg in args.iter().filter_map(Value::as_str) { + if arg == "--privileged" || arg.starts_with("--device") || arg.starts_with("--pid=host") + { + reasons.push(format!("`runArgs` contains `{arg}`")); + } + } + } + for mount in mount_sources(object) { + if is_sensitive_host_path(&mount) { + reasons.push(format!("binds the sensitive host path `{mount}`")); + } + } + + reasons +} + +/// Image inputs that are not pinned (FR-012). +/// +/// An unpinned input makes the comparison non-reproducible in the one way the pinned input +/// set cannot record: `alpine:latest` is a different image tomorrow, so a finding recorded +/// against it is a claim about content nobody can retrieve. The same rule the declarative +/// runner enforces as **V18** for committed Docker cases. +pub fn unpinned_image_inputs(document: &Value) -> Vec { + let Some(object) = document.as_object() else { + return Vec::new(); + }; + let mut unpinned = Vec::new(); + if let Some(Value::String(image)) = object.get("image") + && !is_pinned_image(image) + { + unpinned.push(image.clone()); + } + unpinned +} + +/// Whether an image reference is pinned: a digest, or a concrete tag that is not +/// `latest`. A tag-less reference resolves to `latest` and is therefore unpinned too. +pub fn is_pinned_image(image: &str) -> bool { + if image.contains("@sha256:") { + return true; + } + // The tag is what follows the last `:` — unless that `:` is inside a registry host's + // port (`localhost:5000/img`), which is why the tail must not contain a `/`. + match image.rsplit_once(':') { + Some((_, tag)) if !tag.contains('/') => !tag.is_empty() && tag != "latest", + _ => false, + } +} + +/// Every mount source string a document declares, in both the string and object forms. +fn mount_sources(object: &Map) -> Vec { + let mut sources = Vec::new(); + let mut collect = |value: &Value| match value { + Value::String(spec) => { + for part in spec.split(',') { + if let Some(rest) = part.trim().strip_prefix("source=") { + sources.push(rest.to_string()); + } + } + } + Value::Object(map) => { + if let Some(Value::String(s)) = map.get("source") { + sources.push(s.clone()); + } + } + _ => {} + }; + if let Some(Value::Array(mounts)) = object.get("mounts") { + for mount in mounts { + collect(mount); + } + } + if let Some(mount) = object.get("workspaceMount") { + collect(mount); + } + sources +} + +/// Host paths a generated candidate is never allowed to bind. +const SENSITIVE_HOST_PATHS: &[&str] = &[ + "/", + "/etc", + "/root", + "/home", + "/var/run/docker.sock", + "/var/run", + "/proc", + "/sys", + "/dev", +]; + +/// Whether a mount source names a sensitive host path. +/// +/// `${localWorkspaceFolder}`-rooted sources are the candidate's own workspace and are +/// always allowed; an absolute path is compared against the declared list exactly or as a +/// parent directory, so `/etc/passwd` is caught by `/etc`. +fn is_sensitive_host_path(source: &str) -> bool { + if source.contains("${") { + return false; + } + if !source.starts_with('/') { + return false; + } + SENSITIVE_HOST_PATHS.iter().any(|p| { + source == *p + || (*p != "/" && source.starts_with(&format!("{p}/"))) + || *p == "/" && source == "/" + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn grammar() -> Grammar { + Grammar::load_default().expect("the committed constraint inventory must load") + } + + // --- grammar projection ------------------------------------------------- + + #[test] + fn required_keys_come_from_the_inventorys_own_required_units() { + let g = grammar(); + assert_eq!(required_keys(&g, ConfigBranch::Image), vec!["image"]); + assert_eq!(required_keys(&g, ConfigBranch::Dockerfile), vec!["build"]); + assert_eq!( + required_keys(&g, ConfigBranch::Compose), + vec!["dockerComposeFile", "service", "workspaceFolder"], + "the Compose branch's three required keys are the grammar's, not a \ + hand-written list — a re-vendored schema changes them here automatically" + ); + } + + #[test] + fn optional_properties_exclude_the_required_ones_and_carry_declared_types() { + let g = grammar(); + let optional = optional_properties(&g, ConfigBranch::Image); + assert!( + !optional.iter().any(|p| p.name == "image"), + "a required key is never offered as optional" + ); + assert!( + optional.len() > 20, + "got {} optional properties", + optional.len() + ); + + let names: Vec<&str> = optional.iter().map(|p| p.name.as_str()).collect(); + for expected in [ + "remoteUser", + "forwardPorts", + "features", + "runArgs", + "waitFor", + ] { + assert!( + names.contains(&expected), + "{expected} missing from the draw domain" + ); + } + + let wait_for = optional + .iter() + .find(|p| p.name == "waitFor") + .expect("waitFor is declared"); + assert_eq!( + wait_for.allowed_values, + vec![ + json!("initializeCommand"), + json!("onCreateCommand"), + json!("updateContentCommand"), + json!("postCreateCommand"), + json!("postStartCommand"), + ], + "an `enum` property draws from the schema's exact legal values" + ); + + let ports = optional + .iter() + .find(|p| p.name == "forwardPorts") + .expect("forwardPorts is declared"); + assert_eq!(ports.types, vec!["array".to_string()]); + } + + #[test] + fn properties_are_sorted_and_deduplicated_across_groups() { + let g = grammar(); + for branch in ConfigBranch::all() { + let optional = optional_properties(&g, *branch); + let mut names: Vec<&str> = optional.iter().map(|p| p.name.as_str()).collect(); + let ordered = names.clone(); + names.sort_unstable(); + assert_eq!( + ordered, + names, + "{}: draw domain must be sorted", + branch.as_str() + ); + let before = names.len(); + names.dedup(); + assert_eq!( + before, + names.len(), + "{}: a property appears twice across groups", + branch.as_str() + ); + } + } + + // --- T030: constrained generation --------------------------------------- + + #[test] + fn a_valid_draw_satisfies_its_branchs_required_set() { + let g = grammar(); + let mut generator = Generator::new(&g, 0x5EED_0001); + for branch in ConfigBranch::all() { + for _ in 0..20 { + let doc = generator.draw_document(*branch); + let object = doc.as_object().expect("a document is an object"); + for key in required_keys(&g, *branch) { + assert!( + object.contains_key(&key), + "{} draw is missing required key `{key}`: {doc}", + branch.as_str() + ); + } + } + } + } + + #[test] + fn a_dockerfile_draw_composes_the_nested_build_objects_own_required_set() { + let g = grammar(); + let mut generator = Generator::new(&g, 42); + let mut saw_nested_required = false; + for _ in 0..20 { + let doc = generator.draw_document(ConfigBranch::Dockerfile); + let build = &doc["build"]; + assert!(build.is_object(), "`build` is an object: {doc}"); + if build.get("dockerfile").is_some() { + saw_nested_required = true; + } + } + assert!( + saw_nested_required, + "the nested `build` object has its own `required` set, read from the same \ + inventory rather than hand-written" + ); + } + + #[test] + fn a_near_valid_candidate_records_the_required_key_it_removed() { + let g = grammar(); + let mut generator = Generator::new(&g, 0xBEEF); + let mut near_valid = 0usize; + for _ in 0..400 { + let candidate = generator.next_candidate(); + if candidate.kind != CandidateKind::NearValid { + continue; + } + near_valid += 1; + assert!( + !candidate.violated_required.is_empty(), + "a near-valid candidate that violates nothing is a lie about its kind" + ); + } + assert!( + near_valid > 0, + "the stream must reach the near-valid kind — it is where strictness \ + divergences live" + ); + } + + #[test] + fn every_candidate_is_a_parseable_object_with_operations_and_an_id() { + let g = grammar(); + let mut generator = Generator::new(&g, 7); + for _ in 0..300 { + let candidate = generator.next_candidate(); + assert!(candidate.document.is_object(), "{}", candidate.document); + let rendered = serde_json::to_string(&candidate.document).expect("serializes"); + let back: Value = serde_json::from_str(&rendered).expect("round-trips"); + assert_eq!(back, candidate.document); + + assert!(candidate.id.starts_with("cnd-")); + assert_eq!(candidate.id.len(), "cnd-".len() + 8); + assert_eq!( + candidate.id, + Candidate::derive_id(&candidate.document, &candidate.operations), + "the id must recompute from its substance" + ); + assert_eq!(candidate.operations.len(), 1); + assert_eq!(candidate.operations[0].subcommand, "read-configuration"); + } + } + + #[test] + fn the_candidate_id_distinguishes_documents_that_differ_only_in_order() { + // `mop-ordering-change` deliberately produces documents that differ only in the + // order of a declaration-ordered collection. A key-sorting canonical form would + // give those the same id, collapsing exactly the candidates that category exists + // to explore. + let ops = vec![Operation::read_configuration()]; + let a = json!({ "image": "alpine:3.19", "forwardPorts": [3000, 8080] }); + let b = json!({ "image": "alpine:3.19", "forwardPorts": [8080, 3000] }); + assert_ne!( + Candidate::derive_id(&a, &ops), + Candidate::derive_id(&b, &ops) + ); + + // The operations participate too: the same document under a different subcommand + // is a different candidate. + let other_ops = vec![Operation { + subcommand: "build".to_string(), + argv: vec!["build".to_string()], + }]; + assert_ne!( + Candidate::derive_id(&a, &ops), + Candidate::derive_id(&a, &other_ops) + ); + } + + #[test] + fn generation_never_emits_an_initialize_command() { + // Excluded by construction: it executes on the host before any container + // sandboxing (crates/core/src/trust.rs, SECURITY.md). + let g = grammar(); + let mut generator = Generator::new(&g, 0xABCD); + for branch in ConfigBranch::all() { + for _ in 0..60 { + let doc = generator.draw_document(*branch); + assert!( + doc.get("initializeCommand").is_none(), + "a drawn document must never carry a host-side hook: {doc}" + ); + } + } + } + + // --- SC-003 / FR-008a --------------------------------------------------- + + #[test] + fn the_stream_applies_every_mutation_category_within_a_short_prefix() { + // SC-003 is a property of the round-robin schedule, not a bet on a long run: any + // window of eleven consecutive candidates covers the catalogue. Asserting over a + // slightly longer prefix leaves room for the retry path without weakening the + // claim. + let g = grammar(); + let mut generator = Generator::new(&g, 0x1234_5678); + let mut counts = mutate::empty_application_counts(); + for _ in 0..33 { + for mutation in generator.next_candidate().mutations { + *counts + .get_mut(mutation.category.name()) + .expect("every category has a key") += 1; + } + } + assert!( + mutate::unapplied_categories(&counts).is_empty(), + "categories never applied in 33 candidates: {:?}", + mutate::unapplied_categories(&counts) + ); + } + + #[test] + fn the_seed_corpus_is_the_committed_fixtures_and_every_one_parses() { + let names = seed_fixture_names(); + assert_eq!(names.len(), 5); + assert!(names.contains(&"config-image-reference")); + assert!(names.contains(&"config-compose-multiservice")); + let corpus = seed_corpus(); + assert_eq!(corpus.len(), names.len()); + for (name, doc) in &corpus { + assert!(doc.is_object(), "{name} is not an object"); + } + } + + #[test] + fn candidates_name_the_fixture_or_branch_they_came_from() { + let g = grammar(); + let mut generator = Generator::new(&g, 99); + let mut fixtures = 0usize; + let mut draws = 0usize; + for _ in 0..200 { + let c = generator.next_candidate(); + match (c.fixture, c.branch) { + (Some(name), None) => { + assert!(seed_fixture_names().contains(&name)); + fixtures += 1; + } + (None, Some(_)) => draws += 1, + other => panic!("a candidate must name exactly one provenance, got {other:?}"), + } + } + assert!( + fixtures > 0 && draws > 0, + "both provenances must be reachable" + ); + } + + // --- SC-001 reproducibility -------------------------------------------- + + #[test] + fn the_same_seed_reproduces_the_identical_ordered_candidate_sequence() { + // FR-001 at the generator level. The end-to-end claim (SC-001) also covers the + // finding set, but if this fails nothing downstream can hold. + let g = grammar(); + let mut a = Generator::new(&g, 0xFEED_FACE); + let mut b = Generator::new(&g, 0xFEED_FACE); + let left: Vec = (0..120).map(|_| a.next_candidate()).collect(); + let right: Vec = (0..120).map(|_| b.next_candidate()).collect(); + assert_eq!(left, right); + + let mut other = Generator::new(&g, 0xFEED_FACF); + let different: Vec = (0..120).map(|_| other.next_candidate()).collect(); + assert_ne!( + left.iter().map(|c| c.id.clone()).collect::>(), + different + .iter() + .map(|c| c.id.clone()) + .collect::>(), + "a different seed must not alias" + ); + } + + #[test] + fn the_generator_identity_names_the_stream_and_the_reduction_order() { + let identity = generator_identity(); + assert!(identity.contains("xoshiro256starstar")); + assert!(identity.contains("drop-optional-key")); + assert!(identity.ends_with("+generator/v1")); + } + + // --- FR-011 / FR-012 ---------------------------------------------------- + + #[test] + fn a_host_side_hook_is_unsafe_on_every_tier() { + let doc = json!({ "image": "alpine:3.19", "initializeCommand": "echo host" }); + for container_backed in [false, true] { + let reasons = unsafe_reasons(&doc, container_backed); + assert_eq!(reasons.len(), 1, "{reasons:?}"); + assert!(reasons[0].contains("initializeCommand")); + } + } + + #[test] + fn container_only_hazards_do_not_discard_a_configuration_only_candidate() { + // Flagging them on the configuration tier would discard candidates for a risk that + // tier does not take — a guard that over-refuses shrinks the explored space while + // reporting nothing about it. + let doc = json!({ + "image": "alpine:3.19", + "privileged": true, + "runArgs": ["--privileged"], + "mounts": ["source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"] + }); + assert!(unsafe_reasons(&doc, false).is_empty()); + let reasons = unsafe_reasons(&doc, true); + assert!(reasons.iter().any(|r| r.contains("privileged: true"))); + assert!(reasons.iter().any(|r| r.contains("--privileged"))); + assert!(reasons.iter().any(|r| r.contains("docker.sock"))); + } + + #[test] + fn a_workspace_rooted_mount_is_not_a_sensitive_host_path() { + let doc = json!({ + "image": "alpine:3.19", + "workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind", + "mounts": [{ "source": "${localWorkspaceFolder}/.cache", "target": "/cache", "type": "bind" }] + }); + assert!(unsafe_reasons(&doc, true).is_empty()); + } + + #[test] + fn unpinned_image_inputs_are_named() { + assert!(is_pinned_image("alpine:3.19")); + assert!(is_pinned_image( + "alpine@sha256:0000000000000000000000000000000000000000000000000000000000000000" + )); + assert!(is_pinned_image("localhost:5000/img:1.2")); + assert!(!is_pinned_image("alpine")); + assert!(!is_pinned_image("alpine:latest")); + assert!( + !is_pinned_image("localhost:5000/img"), + "a tag-less reference is `latest`" + ); + + assert_eq!( + unpinned_image_inputs(&json!({ "image": "alpine:latest" })), + vec!["alpine:latest".to_string()] + ); + assert!(unpinned_image_inputs(&json!({ "image": "alpine:3.19" })).is_empty()); + assert!(unpinned_image_inputs(&json!({ "dockerComposeFile": "c.yml" })).is_empty()); + } + + #[test] + fn a_non_object_document_is_unsafe_rather_than_silently_fine() { + let reasons = unsafe_reasons(&json!([1, 2, 3]), false); + assert_eq!(reasons.len(), 1); + assert!(reasons[0].contains("not a JSON object")); + } +} diff --git a/crates/conformance/src/discovery/grammar.rs b/crates/conformance/src/discovery/grammar.rs new file mode 100644 index 00000000..a5d3e749 --- /dev/null +++ b/crates/conformance/src/discovery/grammar.rs @@ -0,0 +1,388 @@ +//! The generation grammar: the committed constraint inventory, indexed +//! (025-exploratory-parity-discovery, research D1, T012/T013). +//! +//! Generation draws its grammar from `conformance/inventory/constraints.json` — the +//! machine-owned, fingerprint-verified extraction of the vendored pinned schemas — and +//! **not** from re-parsing `conformance/schemas//*.json`. Three properties come +//! free from that choice: +//! +//! 1. **The grammar pin is already a recorded revision.** `constraints.json` carries a +//! `revision` field that V14 validates against the registry schema pin, so FR-002's +//! `grammarVersion` element is already there and already guarded. +//! 2. **A schema pin bump automatically surfaces as a generation-input change.** +//! Re-vendoring regenerates the inventory, `inventory diff` enumerates the delta, and +//! every finding bound to the old revision is correctly invalidated with no separate +//! bookkeeping. +//! 3. **No second extraction path.** FR-015 forbids a second normalization definition; +//! the same argument applies one level up to schema interpretation. Two views of the +//! pinned schema surface that could disagree is the identical defect class. +//! +//! Hand-authoring a grammar was rejected outright: it would generate the shapes its +//! author thought of, which is exactly what the curated fixtures already do and exactly +//! the maintainer imagination this feature exists to escape. +//! +//! ## Annotations are excluded +//! +//! Of the inventory's units, the `annotation` kind (`description`, `title`, …) carries +//! no generative content — it constrains nothing and can be neither satisfied nor +//! violated. [`Grammar::load`] drops those and keeps the rest; the counts the unit tests +//! pin are exactly the non-annotation kinds. + +use std::collections::BTreeMap; +use std::path::Path; + +use crate::load::{LoadError, load_inventory}; +use crate::model::{ConstraintKind, ConstraintUnit}; + +/// The generation grammar: the non-annotation constraint units of one pinned inventory, +/// indexed for the two lookups generation and mutation actually perform — "what +/// constrains this schema pointer?" and "give me every unit of this kind". +/// +/// Every index holds *positions* into [`units`](Grammar::units) rather than clones, so +/// the same unit reached through either index is the same unit, and the memory cost of +/// indexing is a `usize` per entry rather than a second copy of the inventory. +#[derive(Debug, Clone)] +pub struct Grammar { + /// The inventory revision this grammar was built from (`rev-schema-`), recorded + /// verbatim as a campaign's `grammarVersion` (data-model.md § 4). + revision: String, + /// The retained (non-annotation) units, in committed inventory order. + units: Vec, + /// Schema pointer → the positions of every unit that constrains it, in inventory + /// order. `BTreeMap` because the pointer set is enumerated in reports and a + /// declaration-order-free container would make that output unstable. + by_pointer: BTreeMap>, + /// Constraint kind → the positions of every unit of that kind, in inventory order. + by_kind: BTreeMap<&'static str, Vec>, +} + +/// The stable wire spelling of a constraint kind — the same kebab-case spelling the +/// committed inventory uses, so a per-kind count in a test reads as the file does. +/// +/// Deliberately a free function rather than an inherent `ConstraintKind::as_str`: the +/// spelling is serde's (`rename_all = "kebab-case"`), and duplicating it as an inherent +/// method on the shared model would create a second definition that could drift from the +/// one the file actually round-trips through. +pub fn kind_name(kind: ConstraintKind) -> &'static str { + match kind { + ConstraintKind::PropertyExistence => "property-existence", + ConstraintKind::Required => "required", + ConstraintKind::Type => "type", + ConstraintKind::Enum => "enum", + ConstraintKind::Const => "const", + ConstraintKind::Default => "default", + ConstraintKind::UnionAlternative => "union-alternative", + ConstraintKind::AllOf => "all-of", + ConstraintKind::Conditional => "conditional", + ConstraintKind::AdditionalProperties => "additional-properties", + ConstraintKind::ArrayShape => "array-shape", + ConstraintKind::ValueShape => "value-shape", + ConstraintKind::Reference => "reference", + ConstraintKind::Annotation => "annotation", + ConstraintKind::UnmodeledKeyword => "unmodeled-keyword", + } +} + +impl Grammar { + /// Load the grammar from a committed constraint inventory. + /// + /// A missing inventory is a hard error rather than an empty grammar: a generator + /// with no grammar would produce nothing and report "found no differences", which is + /// the silent-vacuity failure mode this whole feature exists to avoid + /// (constitution IV). + pub fn load(inventory_file: &Path) -> Result { + let inventory = load_inventory(inventory_file)?.ok_or_else(|| LoadError::Root { + path: inventory_file.to_path_buf(), + cause: "constraint inventory not found — the generation grammar is the \ + committed inventory (research D1); regenerate it with `inventory generate`" + .to_string(), + })?; + Ok(Grammar::from_units(inventory.revision, inventory.units)) + } + + /// Load the grammar from the workspace's default inventory + /// (`conformance/inventory/constraints.json`). + pub fn load_default() -> Result { + Grammar::load(&crate::default_inventory_file()) + } + + /// Build a grammar from an inventory revision and its units, dropping annotations. + /// + /// Exposed so a test (or a future re-vendoring workflow) can build a grammar from + /// units it already holds without a round trip through the filesystem. + pub fn from_units(revision: String, units: Vec) -> Grammar { + let units: Vec = units + .into_iter() + .filter(|u| u.kind != ConstraintKind::Annotation) + .collect(); + + let mut by_pointer: BTreeMap> = BTreeMap::new(); + let mut by_kind: BTreeMap<&'static str, Vec> = BTreeMap::new(); + for (position, unit) in units.iter().enumerate() { + by_pointer + .entry(unit.pointer.clone()) + .or_default() + .push(position); + by_kind + .entry(kind_name(unit.kind)) + .or_default() + .push(position); + } + + Grammar { + revision, + units, + by_pointer, + by_kind, + } + } + + /// The inventory revision (`rev-schema-`) this grammar was built from — a + /// campaign's `grammarVersion`. + pub fn revision(&self) -> &str { + &self.revision + } + + /// Every retained (non-annotation) unit, in committed inventory order. + pub fn units(&self) -> &[ConstraintUnit] { + &self.units + } + + /// The total number of retained units. + pub fn len(&self) -> usize { + self.units.len() + } + + /// Whether the grammar is empty. Always `false` for the committed inventory — + /// [`Grammar::load`] refuses a missing file — but `clippy::len_without_is_empty` + /// asks for it and a caller checking before drawing is reasonable. + pub fn is_empty(&self) -> bool { + self.units.is_empty() + } + + /// Every unit constraining `pointer` (an RFC 6901 JSON Pointer into the pinned + /// schema), in inventory order. Empty when the pointer is unconstrained or unknown — + /// the two are the same fact for a generator, which simply has nothing to draw there. + pub fn at_pointer(&self, pointer: &str) -> Vec<&ConstraintUnit> { + self.by_pointer + .get(pointer) + .map(|positions| positions.iter().map(|&i| &self.units[i]).collect()) + .unwrap_or_default() + } + + /// Every unit of `kind` constraining `pointer`, in inventory order. + /// + /// The composite lookup, because that is the question generation actually asks: not + /// "what constrains this pointer" but "what *type* may this pointer hold" or "which + /// keys here are `required`". + pub fn at_pointer_of_kind(&self, pointer: &str, kind: ConstraintKind) -> Vec<&ConstraintUnit> { + self.by_pointer + .get(pointer) + .map(|positions| { + positions + .iter() + .map(|&i| &self.units[i]) + .filter(|u| u.kind == kind) + .collect() + }) + .unwrap_or_default() + } + + /// Every unit of `kind`, in inventory order. + pub fn of_kind(&self, kind: ConstraintKind) -> Vec<&ConstraintUnit> { + self.by_kind + .get(kind_name(kind)) + .map(|positions| positions.iter().map(|&i| &self.units[i]).collect()) + .unwrap_or_default() + } + + /// Every constrained schema pointer, sorted — the generator's draw domain. + pub fn pointers(&self) -> Vec<&str> { + self.by_pointer.keys().map(String::as_str).collect() + } + + /// Retained unit counts per kind, keyed by the committed wire spelling. + /// + /// Sorted (`BTreeMap`) so a report or a failure message renders identically on every + /// run; a re-vendored inventory therefore shows up as a readable diff rather than a + /// reshuffle. + pub fn kind_counts(&self) -> BTreeMap<&'static str, usize> { + self.by_kind + .iter() + .map(|(kind, positions)| (*kind, positions.len())) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn workspace_grammar() -> Grammar { + Grammar::load_default().expect("the committed constraint inventory must load") + } + + /// The per-kind unit counts of the inventory at `rev-schema-113500f4`. + /// + /// Pinned deliberately (T013): the grammar is a generation *input*, so a re-vendored + /// inventory that silently changes what can be generated would silently change what + /// campaigns explore and quietly invalidate every recorded finding. Making the counts + /// a test means a re-vendoring surfaces here — as a reviewed test change — instead of + /// as an unexplained shift in campaign output weeks later. + const EXPECTED_KIND_COUNTS: &[(&str, usize)] = &[ + ("additional-properties", 38), + ("all-of", 4), + ("array-shape", 41), + ("const", 3), + ("default", 9), + ("enum", 11), + ("property-existence", 117), + ("reference", 12), + ("required", 20), + ("type", 187), + ("union-alternative", 18), + ("value-shape", 9), + ]; + + #[test] + fn the_committed_inventory_loads_as_a_grammar() { + let grammar = workspace_grammar(); + assert_eq!( + grammar.revision(), + format!("rev-schema-{}", crate::CURRENT_SCHEMA_PIN), + "the grammar's revision is a campaign's `grammarVersion` and must equal the pin" + ); + assert!( + !grammar.is_empty(), + "an empty grammar generates nothing and would report `found no differences`" + ); + } + + #[test] + fn per_kind_unit_counts_match_the_pinned_inventory() { + let grammar = workspace_grammar(); + let counts = grammar.kind_counts(); + let expected: BTreeMap<&str, usize> = EXPECTED_KIND_COUNTS.iter().copied().collect(); + assert_eq!( + counts, expected, + "the generation grammar changed. If this is a deliberate re-vendoring, run \ + `inventory diff`, review the delta, and update EXPECTED_KIND_COUNTS in the same \ + commit — every finding bound to the old revision is invalidated by the change." + ); + } + + #[test] + fn the_generative_kinds_carry_the_documented_totals() { + // research D1's table, asserted directly: these six lines are the counts the + // design reasoned from, so they are worth stating in the vocabulary of the + // decision rather than only as part of the map above. + let grammar = workspace_grammar(); + assert_eq!(grammar.of_kind(ConstraintKind::Type).len(), 187); + assert_eq!( + grammar.of_kind(ConstraintKind::PropertyExistence).len(), + 117 + ); + assert_eq!(grammar.of_kind(ConstraintKind::ArrayShape).len(), 41); + assert_eq!(grammar.of_kind(ConstraintKind::Required).len(), 20); + assert_eq!(grammar.of_kind(ConstraintKind::UnionAlternative).len(), 18); + assert_eq!( + grammar.of_kind(ConstraintKind::Enum).len() + + grammar.of_kind(ConstraintKind::Const).len(), + 14, + "`enum` + `const` are the exact-legal-value kinds and the near-miss set one \ + edit away; research D1 counts them together" + ); + } + + #[test] + fn annotations_are_dropped_and_the_rest_is_retained() { + let grammar = workspace_grammar(); + assert!( + grammar.of_kind(ConstraintKind::Annotation).is_empty(), + "annotations constrain nothing and can be neither satisfied nor violated" + ); + let total: usize = EXPECTED_KIND_COUNTS.iter().map(|(_, n)| n).sum(); + assert_eq!( + grammar.len(), + total, + "the grammar is exactly the non-annotation units" + ); + assert_eq!( + grammar.len(), + 469, + "research D1 sizes the grammar at 469 non-annotation units of 609 total" + ); + } + + #[test] + fn lookup_by_pointer_returns_every_kind_at_that_pointer() { + let grammar = workspace_grammar(); + // A pointer taken from the committed inventory rather than invented, so the + // assertion is about the real indexing and not about a fixture. + let pointer = + "/definitions/dockerfileContainer/oneOf/0/properties/build/allOf/0/properties/context"; + let units = grammar.at_pointer(pointer); + assert!( + units.len() >= 2, + "expected the `property-existence` and `type` units at {pointer}, got {}", + units.len() + ); + assert!(units.iter().all(|u| u.pointer == pointer)); + + let types = grammar.at_pointer_of_kind(pointer, ConstraintKind::Type); + assert_eq!(types.len(), 1, "one `type` unit at {pointer}"); + assert_eq!(types[0].substance, serde_json::json!({ "type": "string" })); + + assert!( + grammar.at_pointer("/definitely/not/a/pointer").is_empty(), + "an unknown pointer yields nothing to draw, not a panic" + ); + assert!( + grammar + .at_pointer_of_kind(pointer, ConstraintKind::Annotation) + .is_empty(), + "annotations were dropped, so no pointer can reach one" + ); + } + + #[test] + fn pointers_are_sorted_and_index_positions_agree() { + let grammar = workspace_grammar(); + let pointers = grammar.pointers(); + let mut sorted = pointers.clone(); + sorted.sort_unstable(); + assert_eq!( + pointers, sorted, + "pointer enumeration must be deterministic" + ); + + // Every unit is reachable through BOTH indexes, and the two agree — the + // property that lets a caller mix the lookups without re-deriving anything. + for unit in grammar.units() { + assert!( + grammar + .at_pointer(&unit.pointer) + .iter() + .any(|u| u.id == unit.id), + "unit {} missing from the pointer index", + unit.id + ); + assert!( + grammar.of_kind(unit.kind).iter().any(|u| u.id == unit.id), + "unit {} missing from the kind index", + unit.id + ); + } + } + + #[test] + fn a_missing_inventory_fails_loudly() { + let err = Grammar::load(Path::new("/nonexistent/constraints.json")) + .expect_err("a missing inventory must not yield an empty grammar"); + let msg = err.to_string(); + assert!( + msg.contains("constraint inventory not found"), + "the diagnosis must name the cause, got: {msg}" + ); + } +} diff --git a/crates/conformance/src/discovery/metamorphic.rs b/crates/conformance/src/discovery/metamorphic.rs new file mode 100644 index 00000000..9d7ac519 --- /dev/null +++ b/crates/conformance/src/discovery/metamorphic.rs @@ -0,0 +1,287 @@ +//! Metamorphic relation (`mrl-`) records — `conformance/registry/metamorphic.json` +//! (025-exploratory-parity-discovery, data-model.md § 7, +//! contracts/metamorphic-catalogue.md, US6). +//! +//! Relations live **inside** the registry, unlike findings, because a relation is an +//! *assertion the project makes* — "reordering these keys must not change the result, +//! and here is the clause that says so" — and it references `clu-`/`bhv-` ids only the +//! registry loader can resolve (research D11). A finding, by contrast, is a *candidate* +//! for an assertion: machine-produced, unreviewed, possibly wrong, and structurally +//! unable to reach `certify`. +//! +//! ## The two effects, and why sensitivity is mandatory rather than a bonus +//! +//! | [`RelationEffect`] | Assertion | Catches | +//! |---|---|---| +//! | `invariance` | the transformation MUST NOT change the normalized result | a tool reading meaning that is not there | +//! | `sensitivity` | the transformation MUST change the normalized result | a tool ignoring meaning that *is* there | +//! +//! **A sensitivity relation is the one thing the differential structurally cannot +//! replace.** If deacon and the reference *both* wrongly ignore declaration order, the +//! differential comparison is clean and the defect is invisible to it — both sides agree, +//! and agreeing is exactly what the differential checks. A sensitivity relation asserts +//! the result *must* change, so consistent-wrongness fails it. That is why FR-043 mandates +//! both kinds rather than treating sensitivity as an optional extra. +//! +//! ## The ground requirement (FR-045) +//! +//! Every relation MUST name a [`ground`](MetamorphicRelation::ground) resolving to a +//! normative clause (`clu-`) or a recorded behavior (`bhv-`). Without one, a relation +//! records an author's intuition about what *ought* to be irrelevant — and an ungrounded +//! invariance relation that happens to be wrong does not fail, it **passes**, silently, +//! while asserting something the specification never said. A grounded one can be checked +//! by reading the clause. This mirrors the `ground` that 024 already requires on +//! applicability rules, and gets the same validation treatment +//! ([`crate::validate::check_metamorphic`], **V31**/**V32**). +//! +//! ## Ownership +//! +//! Hand-authored. No generator writes this file. The mandated family list +//! ([`MANDATED_RELATIONS`]) is named here rather than derived, for the same reason +//! `REQUIRED_SCENARIO_DIMENSIONS` is: *removing* a family must be a visible **V32** +//! failure rather than a quietly smaller relation set. + +use std::path::Path; + +use serde::{Deserialize, Serialize}; + +use crate::load::{LoadError, SchemaError, deserialize_located, read_file}; + +/// What a relation asserts about the transformation it declares +/// (contracts/metamorphic-catalogue.md, "The two effects"). +/// +/// A closed enum rather than a bare string so a misspelled effect is refused at **load** +/// time with a located diagnosis, strictly earlier than — and with the same outcome as — +/// the **V31** clause the contract states it under. A record whose effect nobody +/// recognises must never reach evaluation, where "unknown" would have to be resolved into +/// one of the two answers by a default. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum RelationEffect { + /// The transformation MUST NOT change the normalized result. + Invariance, + /// The transformation MUST change the normalized result. + Sensitivity, +} + +impl RelationEffect { + /// The stable wire spelling. + pub fn as_str(self) -> &'static str { + match self { + RelationEffect::Invariance => "invariance", + RelationEffect::Sensitivity => "sensitivity", + } + } +} + +/// One metamorphic relation (data-model.md § 7). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct MetamorphicRelation { + /// `mrl-`; unique across **all** registry id namespaces (V2). + pub id: String, + /// What is applied to the input, in one reviewable sentence. Unique across the + /// catalogue (**V31**): two records claiming the same transformation would be one + /// relation asserted twice, and a failure could not be attributed to either. + pub transformation: String, + /// Which of the two things the relation asserts. + pub effect: RelationEffect, + /// `clu-<…>` or `bhv-<…>`; REQUIRED and MUST resolve (**V31**, FR-045). + pub ground: String, + /// The observable channels the relation asserts over. Non-empty, each a declared + /// `chan-` (**V31**): a relation asserting over no channel observes nothing and can + /// never fail. + pub channels: Vec, + /// Why the ground supports the assertion — the argument a reviewer judges. Required + /// and non-filler (**V31**). + pub rationale: String, +} + +/// The `metamorphic.json` envelope — a `records` collection, matching every other +/// hand-authored registry file. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct MetamorphicFile { + /// Schema version of this file. + #[serde(default)] + pub schema_version: u32, + /// The relation records, in declaration order. + #[serde(default)] + pub records: Vec, +} + +/// The relation families FR-044 mandates (contracts/metamorphic-catalogue.md, "Mandated +/// families"). A family with no record is **V32**. +/// +/// Named rather than derived, so *removing* a family is a visible failure instead of a +/// quietly smaller relation set — the same reasoning as +/// [`crate::validate::REQUIRED_SCENARIO_DIMENSIONS`]. +pub const MANDATED_RELATIONS: &[&str] = &[ + "mrl-formatting-invariance", + "mrl-comment-invariance", + "mrl-key-order-invariance", + "mrl-path-relocation", + "mrl-lifecycle-equivalence", + "mrl-extends-flattening", + "mrl-declaration-order-sensitivity", +]; + +/// Load `metamorphic.json` from `path`. +/// +/// A missing file yields an empty list (a fixture registry that ships none is not an +/// error; **V32** is what refuses an incomplete set for a registry that has opted in). A +/// present-but-malformed file is a located [`LoadError::Schema`] — never silently empty +/// (constitution IV). +pub fn load_metamorphic(path: &Path) -> Result, LoadError> { + if !path.exists() { + return Ok(Vec::new()); + } + let raw = read_file(path).map_err(|e| LoadError::Schema(vec![e]))?; + let file: MetamorphicFile = + deserialize_located(path, &raw).map_err(|e| LoadError::Schema(vec![e]))?; + Ok(file.records) +} + +/// Collect duplicate-`id` relations as located schema errors, so two records can never +/// silently claim the same identity. +pub(crate) fn duplicate_id_errors( + path: &Path, + records: &[MetamorphicRelation], +) -> Vec { + let mut seen: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new(); + let mut out = Vec::new(); + for record in records { + if !seen.insert(record.id.as_str()) { + out.push(SchemaError { + file: path.to_path_buf(), + location: Some(record.id.clone()), + message: format!( + "duplicate metamorphic relation id `{}` — every lookup takes the first \ + match, so a duplicate would evaluate one record and silently ignore the \ + other", + record.id + ), + }); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + const GOOD: &str = r##"{ + "schemaVersion": 1, + "records": [ + { + "id": "mrl-key-order-invariance", + "transformation": "permute the key order within an unordered JSON object", + "effect": "invariance", + "ground": "clu-a1b2c3d4", + "channels": ["chan-structured-output"], + "rationale": "Object member order carries no meaning in JSON, and the configuration schema declares no ordered object. A result that changes under key permutation is reading order it must not read." + } + ] + }"##; + + #[test] + fn the_contract_example_record_loads() { + // contracts/metamorphic-catalogue.md § Record schema, verbatim. + let file: MetamorphicFile = serde_json::from_str(GOOD).expect("the contract example loads"); + let record = &file.records[0]; + assert_eq!(record.id, "mrl-key-order-invariance"); + assert_eq!(record.effect, RelationEffect::Invariance); + assert_eq!(record.ground, "clu-a1b2c3d4"); + assert_eq!(record.channels, vec!["chan-structured-output".to_string()]); + assert!(!record.rationale.is_empty()); + } + + #[test] + fn an_unknown_effect_is_refused_at_load_rather_than_defaulted() { + // The contract lists "unknown effect" under V31; the closed enum refuses it + // strictly earlier, which is the same outcome reached sooner. What must never + // happen is a record whose effect is unrecognised reaching evaluation, where + // "unknown" would have to be resolved into one of the two answers by a default. + let raw = GOOD.replace("\"invariance\"", "\"invariant-ish\""); + let err = serde_json::from_str::(&raw) + .expect_err("an unknown effect must not load"); + assert!( + err.to_string().contains("invariant-ish"), + "the diagnosis must name the offending spelling, got: {err}" + ); + } + + #[test] + fn unknown_fields_are_rejected() { + let raw = GOOD.replace("\"ground\":", "\"grounds\":"); + let err = serde_json::from_str::(&raw) + .expect_err("strict JSON must reject unknown fields"); + assert!(err.to_string().contains("grounds"), "got: {err}"); + } + + #[test] + fn a_record_missing_its_ground_does_not_load() { + // FR-045's floor, enforced at the shape that reads the file: `ground` is not + // `Option`, so a record without one is unrepresentable rather than merely invalid. + let raw = GOOD.replace(r#""ground": "clu-a1b2c3d4","#, ""); + let err = serde_json::from_str::(&raw) + .expect_err("a groundless relation must not load"); + assert!(err.to_string().contains("ground"), "got: {err}"); + } + + #[test] + fn both_effects_round_trip_through_their_wire_spellings() { + for effect in [RelationEffect::Invariance, RelationEffect::Sensitivity] { + let raw = serde_json::to_string(&effect).expect("serializes"); + assert_eq!(raw, format!("\"{}\"", effect.as_str())); + let back: RelationEffect = serde_json::from_str(&raw).expect("round-trips"); + assert_eq!(back, effect); + } + } + + #[test] + fn the_mandated_family_list_is_the_contract_table() { + // contracts/metamorphic-catalogue.md § Mandated families (FR-044): seven rows, + // exactly one of them a sensitivity relation. + assert_eq!(MANDATED_RELATIONS.len(), 7); + for id in MANDATED_RELATIONS { + assert!(id.starts_with("mrl-"), "{id} must be an `mrl-` id"); + } + assert!(MANDATED_RELATIONS.contains(&"mrl-declaration-order-sensitivity")); + let mut sorted = MANDATED_RELATIONS.to_vec(); + sorted.sort_unstable(); + sorted.dedup(); + assert_eq!(sorted.len(), MANDATED_RELATIONS.len(), "no duplicates"); + } + + #[test] + fn missing_file_loads_empty_and_malformed_file_fails_loud() { + let dir = tempfile::tempdir().expect("tempdir"); + assert!( + load_metamorphic(&dir.path().join("metamorphic.json")) + .expect("absent file is not an error") + .is_empty() + ); + let bad = dir.path().join("bad.json"); + std::fs::write(&bad, "{ not json").expect("write fixture"); + assert!( + load_metamorphic(&bad).is_err(), + "a malformed metamorphic.json must fail loud, never default to empty" + ); + let good = dir.path().join("metamorphic.json"); + std::fs::write(&good, GOOD).expect("write fixture"); + assert_eq!(load_metamorphic(&good).expect("loads").len(), 1); + } + + #[test] + fn duplicate_ids_are_reported() { + let file: MetamorphicFile = serde_json::from_str(GOOD).expect("loads"); + let mut records = file.records.clone(); + records.push(file.records[0].clone()); + let errors = duplicate_id_errors(Path::new("metamorphic.json"), &records); + assert_eq!(errors.len(), 1); + assert!(errors[0].message.contains("mrl-key-order-invariance")); + assert!(duplicate_id_errors(Path::new("metamorphic.json"), &file.records).is_empty()); + } +} diff --git a/crates/conformance/src/discovery/mod.rs b/crates/conformance/src/discovery/mod.rs new file mode 100644 index 00000000..487a3b0a --- /dev/null +++ b/crates/conformance/src/discovery/mod.rs @@ -0,0 +1,130 @@ +//! Exploratory parity discovery — the **hermetic** half +//! (025-exploratory-parity-discovery, plan.md § Project Structure). +//! +//! Everything in this module is pure data-to-data logic: it never invokes the pinned +//! oracle, never talks to Docker, and never touches the network. That is not a +//! convention to be careful about — it is what makes the hermeticity claim of FR-055 +//! cheap to hold, because the generator, the shrinker, the signature, and the queue are +//! unit-testable in the fast lane with no external dependency at all. The live half +//! (campaign driver, differential comparison, minimization predicate, candidate +//! assembly, corpus fetch, pipeline proof) lives in `parity_harness::discovery`. +//! +//! ## The two data roots +//! +//! | Root | Ownership | Reachable from `certify`? | +//! |---|---|---| +//! | `conformance/discovery/` | machine-produced + hand-triaged | **No** — a sibling of `registry/`; no loader path reaches it | +//! | `conformance/registry/metamorphic.json` | hand-authored | Yes — it is an assertion the project makes | +//! +//! The separation is structural, not conventional (research D6): [`crate::load`] +//! enumerates *named* subdirectories under `conformance/registry/` and has no wildcard +//! walk at the registry root, so a sibling of `registry/` has no code path that could +//! reach it. Exactly one reference crosses the boundary — `Finding.promotedTo → +//! case-` — and it points **out** of the discovery root into the registry. Nothing +//! in the registry points back, so following references from the registry can never +//! arrive at a finding. +//! +//! ## Violation classes are D-numbered, not V-numbered +//! +//! `discovery check` emits **D1–D5** over the discovery data root; `validate` emits +//! **V1–V32** over the registry. They are numbered separately on purpose: folding the +//! D-series into the V-series would imply the registry validator can see the queue, +//! which is precisely what research D6 says it must not (contracts/findings-queue.md). +//! +//! ## Module map +//! +//! | Module | Owns | +//! |---|---| +//! | [`rng`] | the in-repo deterministic PRNG (research D2) | +//! | [`grammar`] | the constraint inventory indexed as a generation grammar (research D1) | +//! | [`generate`] | constrained candidate generation | +//! | [`mutate`] | the eleven-category mutation operator catalogue | +//! | [`shrink`] | structural reduction; the reproduction predicate is a *parameter* (research D5) | +//! | [`signature`] | the normalized signature + value-shape class (research D3) | +//! | [`queue`] | the findings-queue model, strict loader, atomic writer, and D1–D5 | +//! | [`promote`] | the review-only promotion + tolerance skeletons (writes nothing, ever) | +//! | [`metamorphic`] | `mrl-` relation model + V31/V32 | +//! | [`corpus`] | the corpus manifest model + immutable-reference validation | +//! | [`report`] | byte-stable campaign + queue reports | + +/// The first 8 lowercase-hex chars of SHA-256 over `parts`, **length-prefixed**. +/// +/// The single hashing primitive for every discovery id (`sig-`, `fnd-`, `wit-`, `cmp-`, +/// `cnd-`, `cor-`), deliberately shared rather than re-derived per module: two truncation +/// conventions that could drift is the same defect class as two normalization paths. Only +/// the *truncation convention* is shared with `inventory::hash8` / `clause::hash8` — those +/// hash a schema pointer and a prose excerpt respectively, and coupling to either would +/// tie a discovery id's identity to an unrelated record's field set. +/// +/// ## Why length-prefixing rather than a separator byte +/// +/// The concatenation must be **injective** — `("ab", "c")` must never hash as +/// `("a", "bc")` — or two structurally distinct signatures could share an id, and two +/// distinct defects would silently merge into one finding. +/// +/// The other `hash8`s in this crate get injectivity from a `\u{1f}` separator plus the +/// argument that registry ids and dimension values are printable ASCII, so no input can +/// contain the separator. **That argument does not hold here.** A signature's `path` comes +/// verbatim from the diff and is ultimately built from user-controlled configuration keys, +/// so a generated candidate could contain any byte at all — including the separator, at +/// which point a hostile-or-merely-unlucky key would collapse two signatures into one. +/// Prefixing each part with its byte length makes the encoding injective unconditionally, +/// with no assumption about the input alphabet. +pub fn hash8(parts: &[&str]) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + for part in parts { + hasher.update((part.len() as u64).to_le_bytes()); + hasher.update(part.as_bytes()); + } + let digest = hasher.finalize(); + let mut hex = String::with_capacity(8); + for b in &digest[..4] { + use std::fmt::Write as _; + let _ = write!(hex, "{b:02x}"); + } + hex +} + +#[cfg(test)] +mod tests { + use super::hash8; + + #[test] + fn hashing_is_injective_regardless_of_the_input_alphabet() { + // The boundary case a separator byte would get wrong. + assert_ne!(hash8(&["ab", "c"]), hash8(&["a", "bc"])); + assert_ne!(hash8(&["a", ""]), hash8(&["", "a"])); + assert_ne!(hash8(&["a"]), hash8(&["a", ""])); + + // And the case the separator argument cannot cover: a part that CONTAINS the + // byte the other hash8s separate on. A signature's `path` is built from + // user-controlled configuration keys, so this is reachable input, not a + // thought experiment. + assert_ne!(hash8(&["a\u{1f}b"]), hash8(&["a", "b"])); + assert_ne!(hash8(&["a\u{1f}b", "c"]), hash8(&["a", "b\u{1f}c"])); + } + + #[test] + fn the_digest_is_eight_lowercase_hex_chars() { + let h = hash8(&["chan-stdout", "configuration.remoteUser"]); + assert_eq!(h.len(), 8); + assert!( + h.chars() + .all(|c| c.is_ascii_hexdigit() && !c.is_uppercase()) + ); + assert_eq!(h, hash8(&["chan-stdout", "configuration.remoteUser"])); + } +} + +pub mod corpus; +pub mod generate; +pub mod grammar; +pub mod metamorphic; +pub mod mutate; +pub mod promote; +pub mod queue; +pub mod report; +pub mod rng; +pub mod shrink; +pub mod signature; diff --git a/crates/conformance/src/discovery/mutate.rs b/crates/conformance/src/discovery/mutate.rs new file mode 100644 index 00000000..08de9c79 --- /dev/null +++ b/crates/conformance/src/discovery/mutate.rs @@ -0,0 +1,1440 @@ +//! The eleven-category mutation operator catalogue +//! (025-exploratory-parity-discovery, data-model.md § 5, T031/T032). +//! +//! The catalogue lives in code rather than as a data file because each operator is +//! executable logic; [`MUTATION_CATALOG_VERSION`] (one of the seven pinned-input-set +//! elements) pins its identity. Every application records its `mop-` on the +//! witness (FR-009), which is what lets a candidate name the operators that produced it +//! and what lets shrinking un-apply one operator as a reduction step (research D5). +//! +//! ## This module is the single source of the category key list +//! +//! `CampaignOutcome::mutation_applications` must carry **all eleven keys, always**, +//! including zeroes (FR-010): a category absent from the map is indistinguishable from a +//! category that was never applied, and FR-010 requires zero to be reported as an +//! explicit generation deficiency — which needs the key present. Every producer of that +//! map therefore starts from [`empty_application_counts`] rather than restating the +//! eleven names. A second list is a list that drifts, and the drift would be silent: the +//! map would simply stop mentioning a category nobody noticed had gone. +//! +//! ## Schema-adjacent, never byte-corrupted +//! +//! Every operator rewrites the **parsed** document and returns a document that still +//! serializes, still parses, and is still a JSON object. Corrupting bytes would produce +//! candidates that die at the document-syntax stage, which is exactly the budget waste +//! SC-002 caps at 10% — a campaign whose mutations produce malformed JSON explores the +//! parser rather than the tool. + +use indexmap::IndexMap; +use serde_json::{Map, Value}; + +use super::rng::Prng; + +/// The mutation **operator set**'s identity — one of the seven pinned-input-set elements +/// (data-model.md § 4). +/// +/// It names the operator set and nothing else. The pseudorandom stream and the reduction +/// catalogue's order live in `generatorVersion`, because folding either in here would +/// name them for something they are not: a deliberate change to reduction order would +/// look like a change to the mutation operators. +/// +/// Bump this whenever an operator is added, removed, or changes what it produces. +pub const MUTATION_CATALOG_VERSION: &str = "mutation-catalogue/v1"; + +/// The eleven mandated categories (FR-008, data-model.md § 5). +/// +/// A closed enum rather than strings so a twelfth category cannot appear by typo, and so +/// [`MutationCategory::all`] is exhaustive by construction rather than by review. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum MutationCategory { + /// Insert a key absent from the schema at a pointer. + UnknownField, + /// Replace a value with one of a different JSON type. + WrongType, + /// Replace a value with `null`. + NullValue, + /// Empty a collection or string in place. + EmptyValue, + /// Add a second configuration source (image + Dockerfile + Compose). + ConflictingSource, + /// Corrupt a Feature identifier's registry/path/tag shape. + InvalidFeatureId, + /// Introduce a cycle into an `extends` chain. + ExtendsCycle, + /// Nest, self-reference, or leave unterminated a `${…}` token. + SubstitutionEdge, + /// Switch between the permitted string/array/object lifecycle forms. + LifecycleShape, + /// Vary service count, `runServices`, override-file ordering. + ComposeCombination, + /// Permute a declaration-ordered collection. + OrderingChange, +} + +/// The number of mandated categories. Stated as a constant so a caller can assert the +/// arity without counting a slice, and so FR-008's "at minimum eleven" is checkable. +pub const CATEGORY_COUNT: usize = 11; + +impl MutationCategory { + /// Every category, in the catalogue's declaration order — which is also the key order + /// of [`empty_application_counts`], so a report renders the same way every run. + pub fn all() -> &'static [MutationCategory; CATEGORY_COUNT] { + &[ + MutationCategory::UnknownField, + MutationCategory::WrongType, + MutationCategory::NullValue, + MutationCategory::EmptyValue, + MutationCategory::ConflictingSource, + MutationCategory::InvalidFeatureId, + MutationCategory::ExtendsCycle, + MutationCategory::SubstitutionEdge, + MutationCategory::LifecycleShape, + MutationCategory::ComposeCombination, + MutationCategory::OrderingChange, + ] + } + + /// The category name, exactly as data-model.md § 5's table spells it. + pub fn name(self) -> &'static str { + match self { + MutationCategory::UnknownField => "unknown-field", + MutationCategory::WrongType => "wrong-type", + MutationCategory::NullValue => "null-value", + MutationCategory::EmptyValue => "empty-value", + MutationCategory::ConflictingSource => "conflicting-source", + MutationCategory::InvalidFeatureId => "invalid-feature-id", + MutationCategory::ExtendsCycle => "extends-cycle", + MutationCategory::SubstitutionEdge => "substitution-edge", + MutationCategory::LifecycleShape => "lifecycle-shape", + MutationCategory::ComposeCombination => "compose-combination", + MutationCategory::OrderingChange => "ordering-change", + } + } + + /// The `mop-` operator id recorded on a witness (FR-009, data-model.md § 1). + /// + /// Hand-assigned and stable rather than hashed: the id IS the declared name, so a + /// reviewer reading a witness sees which operator ran without a lookup. + pub fn operator_id(self) -> &'static str { + match self { + MutationCategory::UnknownField => "mop-unknown-field", + MutationCategory::WrongType => "mop-wrong-type", + MutationCategory::NullValue => "mop-null-value", + MutationCategory::EmptyValue => "mop-empty-value", + MutationCategory::ConflictingSource => "mop-conflicting-source", + MutationCategory::InvalidFeatureId => "mop-invalid-feature-id", + MutationCategory::ExtendsCycle => "mop-extends-cycle", + MutationCategory::SubstitutionEdge => "mop-substitution-edge", + MutationCategory::LifecycleShape => "mop-lifecycle-shape", + MutationCategory::ComposeCombination => "mop-compose-combination", + MutationCategory::OrderingChange => "mop-ordering-change", + } + } + + /// Parse a category name (not the `mop-` id), returning `None` on anything else — + /// never a default, which would silently attribute an application to the wrong + /// operator. + pub fn parse(name: &str) -> Option { + MutationCategory::all() + .iter() + .copied() + .find(|c| c.name() == name) + } + + /// Parse a `mop-` operator id. + pub fn parse_operator(id: &str) -> Option { + MutationCategory::all() + .iter() + .copied() + .find(|c| c.operator_id() == id) + } +} + +/// Applications per mutation category, in catalogue declaration order. +/// +/// A named alias so a caller in another crate can hold the map without naming `indexmap` +/// itself — which would make an ordering-critical container a dependency decision at every +/// call site rather than a property of the catalogue that owns it. +pub type ApplicationCounts = IndexMap; + +/// The eleven category names, in declaration order — **the** key list for +/// `CampaignOutcome::mutation_applications`. +pub fn category_names() -> Vec<&'static str> { + MutationCategory::all().iter().map(|c| c.name()).collect() +} + +/// A fresh application-count map carrying **all eleven keys at zero**, in catalogue +/// declaration order (FR-010). +/// +/// Every producer of `mutationApplications` starts here. Building the map by inserting +/// only the categories that fired is the defect FR-010 names: a category that never +/// applied would simply be missing, and "we never generated this shape" would be +/// indistinguishable from "we never looked". +pub fn empty_application_counts() -> IndexMap { + MutationCategory::all() + .iter() + .map(|c| (c.name().to_string(), 0u64)) + .collect() +} + +/// A category that never successfully applied, reported as an explicit generation +/// deficiency rather than by omission (FR-010, SC-003). +pub fn unapplied_categories(counts: &IndexMap) -> Vec<&'static str> { + MutationCategory::all() + .iter() + .filter(|c| counts.get(c.name()).copied().unwrap_or(0) == 0) + .map(|c| c.name()) + .collect() +} + +/// How to reverse one mutation: the root keys it touched, and what each held **before**. +/// +/// This is what makes the shrinker's `un-apply-mutation` step (data-model.md § 6, step 2) +/// an exact reversal rather than a guess at what an operator category probably did. +/// +/// ## Why it is a *local* edit rather than the pre-image document +/// +/// Reduction is greedy and cumulative: by the time `un-apply-mutation` is reached, earlier +/// steps have already removed keys, emptied collections, and minimized scalars. Restoring a +/// whole pre-image document would silently undo all of that, so the reversal has to name +/// only the sites the operator actually changed. +/// +/// ## Why it is derived by diffing rather than recorded per operator +/// +/// Every operator in the catalogue edits keys of the document **root**, so the pre- and +/// post-images fully determine the reversal. Recording it inside each of the eleven +/// operators would be eleven separate chances for the recorded reversal to disagree with +/// what the operator did — and a reversal that disagrees does not fail loudly, it quietly +/// produces a reduced input that no longer reproduces anything. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Reversal { + /// `(root key, value before the mutation)`. `None` means the key was **absent** + /// before, so reversing removes it. Sorted by key, so the reversal is deterministic. + pub keys: Vec<(String, Option)>, +} + +impl Reversal { + /// The reversal that turns `before` back out of `after`, at the document root. + /// + /// Both are expected to be objects (every candidate document is); a non-object on + /// either side yields an empty reversal, which reverses nothing rather than + /// pretending to. + fn between(before: &Map, after: &Map) -> Reversal { + let mut names: Vec<&String> = before.keys().chain(after.keys()).collect(); + names.sort_unstable(); + names.dedup(); + let keys = names + .into_iter() + .filter(|name| before.get(*name) != after.get(*name)) + .map(|name| (name.clone(), before.get(name).cloned())) + .collect(); + Reversal { keys } + } + + /// Whether this reversal names any site at all. + pub fn is_empty(&self) -> bool { + self.keys.is_empty() + } + + /// Apply the reversal to `document`, restoring each named root key to what it held + /// before the mutation (removing it where it was absent). + /// + /// A non-object `document` is returned unchanged: there is no root key to restore, and + /// inventing an object around it would fabricate a document the campaign never saw. + pub fn apply(&self, document: &Value) -> Value { + let Value::Object(object) = document else { + return document.clone(); + }; + let mut out = object.clone(); + for (key, previous) in &self.keys { + match previous { + Some(value) => { + out.insert(key.clone(), value.clone()); + } + None => { + out.remove(key); + } + } + } + Value::Object(out) + } +} + +/// One recorded mutation application (FR-009 attribution). +#[derive(Debug, Clone, PartialEq)] +pub struct Mutation { + /// The category that fired. + pub category: MutationCategory, + /// `mop-` — what the witness records. + pub operator: String, + /// A short, human-readable description of *where* and *what*, so a reviewer reading + /// a candidate can tell two applications of the same operator apart. + pub detail: String, + /// How to undo it, for the shrinker's `un-apply-mutation` step. + pub reversal: Reversal, +} + +impl Mutation { + fn new(category: MutationCategory, detail: impl Into, reversal: Reversal) -> Mutation { + Mutation { + category, + operator: category.operator_id().to_string(), + detail: detail.into(), + reversal, + } + } +} + +/// A successfully applied mutation and the document it produced. +#[derive(Debug, Clone, PartialEq)] +pub struct Applied { + /// The mutated document — still an object, still serializable, still parseable. + pub document: Value, + /// What was applied. + pub mutation: Mutation, +} + +/// Apply one category to `document`, or return `None` when the operator has no target. +/// +/// `None` is a first-class answer, not a failure: `ordering-change` on a document with no +/// multi-element collection has nothing to reorder, and inventing a collection to permute +/// would record an ordering mutation that never happened. The campaign counts only +/// **successful** applications, which is what makes FR-010's zero-count report honest. +/// +/// Every branch is a pure function of `document` and the [`Prng`] draws it makes, so the +/// same seed and the same input always yield the same output — the property FR-001 rests +/// on. +pub fn apply(category: MutationCategory, document: &Value, prng: &mut Prng) -> Option { + let Value::Object(object) = document else { + // Every candidate is a devcontainer.json document, which is an object. A + // non-object has no property to mutate, so there is nothing honest to do. + return None; + }; + let (mutated, detail) = match category { + MutationCategory::UnknownField => unknown_field(object, prng)?, + MutationCategory::WrongType => wrong_type(object, prng)?, + MutationCategory::NullValue => null_value(object, prng)?, + MutationCategory::EmptyValue => empty_value(object, prng)?, + MutationCategory::ConflictingSource => conflicting_source(object, prng)?, + MutationCategory::InvalidFeatureId => invalid_feature_id(object, prng)?, + MutationCategory::ExtendsCycle => extends_cycle(object, prng)?, + MutationCategory::SubstitutionEdge => substitution_edge(object, prng)?, + MutationCategory::LifecycleShape => lifecycle_shape(object, prng)?, + MutationCategory::ComposeCombination => compose_combination(object, prng)?, + MutationCategory::OrderingChange => ordering_change(object, prng)?, + }; + // The reversal is derived from the pre- and post-images rather than reported by the + // operator, so it cannot disagree with what the operator actually did (see [`Reversal`]). + let reversal = Reversal::between(object, &mutated); + Some(Applied { + document: Value::Object(mutated), + mutation: Mutation::new(category, detail, reversal), + }) +} + +// --------------------------------------------------------------------------- +// The eleven operators +// --------------------------------------------------------------------------- + +/// Key names that are **not** properties of the pinned devcontainer schema. +/// +/// A fixed pool rather than a generated string so the mutation is reproducible from the +/// seed alone, and so an unknown-field finding names a key a reviewer can grep for. +const UNKNOWN_FIELD_NAMES: &[&str] = &[ + "deaconDiscoveryProbe", + "xUnrecognizedSetting", + "notASchemaProperty", + "vendorSpecificKey", +]; + +/// `mop-unknown-field` — insert a key absent from the schema. +/// +/// Grammar input: `additional-properties` / `property-existence` (data-model.md § 5) — +/// the schema's own statement of which keys may appear at a pointer, and therefore of +/// which keys are the near-valid one-edit-away set. +fn unknown_field( + object: &Map, + prng: &mut Prng, +) -> Option<(Map, String)> { + let free: Vec<&&str> = UNKNOWN_FIELD_NAMES + .iter() + .filter(|name| !object.contains_key(**name)) + .collect(); + let name = **prng.choose(&free)?; + let value = scalar_draw(prng); + let mut out = object.clone(); + out.insert(name.to_string(), value); + Some(( + out, + format!("inserted unknown key `{name}` at the document root"), + )) +} + +/// `mop-wrong-type` — replace a value with one of a different JSON type. +/// +/// Grammar input: `type`. The replacement is *type*-directed, not random: swapping a +/// string for another string would be a value change, and this category exists to probe +/// the boundary the `type` constraint draws. +fn wrong_type( + object: &Map, + prng: &mut Prng, +) -> Option<(Map, String)> { + let key = pick_key(object, prng, |_, _| true)?; + let current = object.get(&key)?; + let replacement = other_type(current); + let mut out = object.clone(); + let from = json_type(current); + let to = json_type(&replacement); + out.insert(key.clone(), replacement); + Some((out, format!("retyped `{key}` from {from} to {to}"))) +} + +/// `mop-null-value` — replace a value with `null`. +/// +/// Distinct from `empty-value` on purpose: the spec distinguishes an authored `null` from +/// an omission and from an empty collection, and this repository has already been bitten +/// by a normalization that conflated the three (023 T062's `prune`). +fn null_value( + object: &Map, + prng: &mut Prng, +) -> Option<(Map, String)> { + let key = pick_key(object, prng, |_, v| !v.is_null())?; + let mut out = object.clone(); + out.insert(key.clone(), Value::Null); + Some((out, format!("set `{key}` to null"))) +} + +/// `mop-empty-value` — empty a collection or string **in place**, preserving its type. +/// +/// Grammar input: `array-shape` / `type`. +fn empty_value( + object: &Map, + prng: &mut Prng, +) -> Option<(Map, String)> { + let key = pick_key(object, prng, |_, v| match v { + Value::Array(a) => !a.is_empty(), + Value::Object(o) => !o.is_empty(), + Value::String(s) => !s.is_empty(), + _ => false, + })?; + let emptied = match object.get(&key)? { + Value::Array(_) => Value::Array(Vec::new()), + Value::Object(_) => Value::Object(Map::new()), + _ => Value::String(String::new()), + }; + let mut out = object.clone(); + let shape = json_type(&emptied); + out.insert(key.clone(), emptied); + Some((out, format!("emptied `{key}` in place (still {shape})"))) +} + +/// The four mutually-exclusive configuration sources the schema's `oneOf` branches +/// declare (`imageContainer` / `dockerfileContainer` × 2 spellings / `composeContainer`). +const CONFIG_SOURCE_KEYS: &[&str] = &["image", "dockerFile", "build", "dockerComposeFile"]; + +/// `mop-conflicting-source` — add a *second* configuration source. +/// +/// Grammar input: `union-alternative`. The schema's `oneOf` says exactly one branch may +/// match; satisfying two is the near-valid input that asks both implementations what they +/// do when the union is violated — a question no curated fixture asks, because a fixture +/// author writes configurations that work. +fn conflicting_source( + object: &Map, + prng: &mut Prng, +) -> Option<(Map, String)> { + let present: Vec<&str> = CONFIG_SOURCE_KEYS + .iter() + .copied() + .filter(|k| object.contains_key(*k)) + .collect(); + let absent: Vec<&str> = CONFIG_SOURCE_KEYS + .iter() + .copied() + .filter(|k| !object.contains_key(*k)) + .collect(); + let added = *prng.choose(&absent)?; + let value = match added { + "image" => Value::String("alpine:3.19".to_string()), + "dockerFile" => Value::String("Dockerfile".to_string()), + "build" => serde_json::json!({ "dockerfile": "Dockerfile" }), + _ => Value::String("docker-compose.yml".to_string()), + }; + let mut out = object.clone(); + out.insert(added.to_string(), value); + let existing = if present.is_empty() { + "no other source".to_string() + } else { + present.join(" + ") + }; + Some((out, format!("added source `{added}` alongside {existing}"))) +} + +/// One way to corrupt a Feature identifier: a label for the witness, and the rewrite. +/// +/// A named type rather than an inline tuple so the corruption table reads as the catalogue +/// it is, and so a reader sees that the label and the rewrite are one record. +type FeatureIdCorruption = (&'static str, fn(&str) -> String); + +/// Corruptions of a Feature identifier's registry / path / tag shape. +const FEATURE_ID_CORRUPTIONS: &[FeatureIdCorruption] = &[ + ("dropped the tag", |id| { + id.rsplit_once(':') + .map(|(l, _)| l.to_string()) + .unwrap_or_else(|| id.to_string()) + }), + ("doubled a registry dot", |id| id.replacen('.', "..", 1)), + ("stripped the registry and path", |id| { + id.rsplit('/').next().unwrap_or(id).to_string() + }), + ("appended an empty tag", |id| format!("{id}:")), +]; + +/// `mop-invalid-feature-id` — corrupt a Feature identifier's registry/path/tag shape. +/// +/// Grammar input: `property-existence` under `features`. A Feature id is not a schema +/// *type* — the schema says only that `features` is an object — so its shape is enforced +/// by the resolver rather than the validator, which makes it a place the two +/// implementations can genuinely disagree without either violating the schema. +fn invalid_feature_id( + object: &Map, + prng: &mut Prng, +) -> Option<(Map, String)> { + let mut out = object.clone(); + let existing: Vec<(String, Value)> = match object.get("features") { + Some(Value::Object(f)) if !f.is_empty() => { + f.iter().map(|(k, v)| (k.clone(), v.clone())).collect() + } + _ => Vec::new(), + }; + + if existing.is_empty() { + // No feature to corrupt: introduce one whose id is already malformed. This is + // still an invalid-feature-id mutation — the operator's subject is the identifier, + // not a pre-existing entry. + let mut features = Map::new(); + features.insert("not a feature id".to_string(), serde_json::json!({})); + out.insert("features".to_string(), Value::Object(features)); + return Some(( + out, + "introduced `features` with the malformed id `not a feature id`".to_string(), + )); + } + + let index = prng.next_index(existing.len())?; + let (target, value) = &existing[index]; + let corruption_index = prng.next_index(FEATURE_ID_CORRUPTIONS.len())?; + let (label, corrupt) = FEATURE_ID_CORRUPTIONS[corruption_index]; + let corrupted = corrupt(target); + + let mut features = Map::new(); + for (k, v) in &existing { + if k == target { + features.insert(corrupted.clone(), value.clone()); + } else { + features.insert(k.clone(), v.clone()); + } + } + out.insert("features".to_string(), Value::Object(features)); + Some(( + out, + format!("{label} on feature id `{target}` → `{corrupted}`"), + )) +} + +/// `mop-extends-cycle` — introduce a cycle into an `extends` chain. +/// +/// Grammar input: `property-existence`. Both the self-reference and the two-hop cycle are +/// reachable: the self-reference is the shortest cycle a resolver must detect, and the +/// two-hop form is the shortest one a naive "is it me?" check misses. +fn extends_cycle( + object: &Map, + prng: &mut Prng, +) -> Option<(Map, String)> { + let (value, detail) = if prng.next_bool() { + ( + Value::String("./devcontainer.json".to_string()), + "self-referential `extends` (the shortest cycle)", + ) + } else { + ( + Value::Array(vec![ + Value::String("./devcontainer.json".to_string()), + Value::String("./devcontainer.json".to_string()), + ]), + "two-hop `extends` cycle through the document itself", + ) + }; + if object.get("extends") == Some(&value) { + // Already exactly this cycle: applying it again would record a mutation that + // changed nothing, and an application count that includes no-ops is not a count + // of anything. + return None; + } + let mut out = object.clone(); + out.insert("extends".to_string(), value); + Some((out, detail.to_string())) +} + +/// `${…}` edge cases: nesting, self-reference, and non-termination. +const SUBSTITUTION_EDGES: &[(&str, &str)] = &[ + ("nested", "${localEnv:${localEnv:DEACON_DISCOVERY}}"), + ("unterminated", "${containerWorkspaceFolder"), + ( + "self-referential", + "${localWorkspaceFolder}${localWorkspaceFolder}", + ), + ("empty token", "${}"), + ("unknown scope", "${notAScope:VALUE}"), +]; + +/// `mop-substitution-edge` — nest, self-reference, or leave unterminated a `${…}` token. +/// +/// Grammar input: `type` restricted to string-valued fields. Substitution is applied by +/// the consumer, not by the schema, so this is another place two conformant +/// implementations can disagree while both satisfy the pinned schema. +fn substitution_edge( + object: &Map, + prng: &mut Prng, +) -> Option<(Map, String)> { + let edge_index = prng.next_index(SUBSTITUTION_EDGES.len())?; + let (label, token) = SUBSTITUTION_EDGES[edge_index]; + let mut out = object.clone(); + match pick_key(object, prng, |_, v| v.is_string()) { + Some(key) => { + out.insert(key.clone(), Value::String(token.to_string())); + Some((out, format!("{label} substitution token in `{key}`"))) + } + None => { + // No string-valued field to carry the token: `name` is a string on every + // branch of the schema, so introducing it keeps the document schema-adjacent. + out.insert("name".to_string(), Value::String(token.to_string())); + Some((out, format!("{label} substitution token in a new `name`"))) + } + } +} + +/// The lifecycle hooks whose value may be a string, an array, or an object. +const LIFECYCLE_KEYS: &[&str] = &[ + "initializeCommand", + "onCreateCommand", + "updateContentCommand", + "postCreateCommand", + "postStartCommand", + "postAttachCommand", +]; + +/// `mop-lifecycle-shape` — switch between the permitted string/array/object forms. +/// +/// Grammar input: `union-alternative`. All three forms are legal, so this operator +/// produces a **valid** document whose *shape* differs — which is precisely the case +/// where a difference is a defect rather than a rejection, and precisely the shape family +/// this repository has already shipped a fix for (012-fix-lifecycle-formats). +fn lifecycle_shape( + object: &Map, + prng: &mut Prng, +) -> Option<(Map, String)> { + let present: Vec<&&str> = LIFECYCLE_KEYS + .iter() + .filter(|k| object.contains_key(**k)) + .collect(); + let mut out = object.clone(); + + let (key, current) = match prng.choose(&present) { + Some(k) => ((**k).to_string(), object.get(**k).cloned()), + None => { + let index = prng.next_index(LIFECYCLE_KEYS.len())?; + (LIFECYCLE_KEYS[index].to_string(), None) + } + }; + + let (rotated, to) = match current.as_ref() { + // string → array + Some(Value::String(s)) => (Value::Array(vec![Value::String(s.clone())]), "array"), + // array → object (one named command per element, declaration-ordered) + Some(Value::Array(items)) => { + let mut map = Map::new(); + for (i, item) in items.iter().enumerate() { + map.insert(format!("step{i}"), item.clone()); + } + if map.is_empty() { + map.insert("step0".to_string(), Value::String("true".to_string())); + } + (Value::Object(map), "object") + } + // object → string (join the named commands) + Some(Value::Object(map)) => { + let joined = map + .values() + .map(render_command) + .collect::>() + .join(" && "); + let joined = if joined.is_empty() { + "true".to_string() + } else { + joined + }; + (Value::String(joined), "string") + } + // absent (or a shape the schema does not permit): introduce the string form + _ => (Value::String("echo discovery".to_string()), "string"), + }; + + let from = current.as_ref().map(json_type).unwrap_or("absent"); + out.insert(key.clone(), rotated); + Some((out, format!("lifecycle `{key}` from {from} to {to}"))) +} + +/// `mop-compose-combination` — vary service count, `runServices`, override-file ordering. +/// +/// Grammar input: `union-alternative` + `array-shape`. `dockerComposeFile` accepts both a +/// string and an array of override files whose **order** is significant, so this operator +/// covers a shape whose semantics live entirely outside the schema. +fn compose_combination( + object: &Map, + prng: &mut Prng, +) -> Option<(Map, String)> { + let mut out = object.clone(); + match object.get("dockerComposeFile") { + Some(Value::String(single)) => { + // string → ordered override list. The order is the semantics: later files + // override earlier ones. + out.insert( + "dockerComposeFile".to_string(), + Value::Array(vec![ + Value::String(single.clone()), + Value::String("docker-compose.override.yml".to_string()), + ]), + ); + Some(( + out, + "`dockerComposeFile` from a single file to an ordered override list".to_string(), + )) + } + Some(Value::Array(files)) if files.len() >= 2 => { + let mut reversed = files.clone(); + reversed.reverse(); + out.insert("dockerComposeFile".to_string(), Value::Array(reversed)); + Some(( + out, + "reversed the `dockerComposeFile` override order".to_string(), + )) + } + Some(Value::Array(files)) => { + let mut extended = files.clone(); + extended.push(Value::String("docker-compose.override.yml".to_string())); + out.insert("dockerComposeFile".to_string(), Value::Array(extended)); + Some((out, "appended a `dockerComposeFile` override".to_string())) + } + _ => { + // No Compose source: introduce one, with the two keys `composeContainer` + // additionally requires so the result stays schema-adjacent rather than + // becoming an obviously-incomplete branch. + out.insert( + "dockerComposeFile".to_string(), + Value::String("docker-compose.yml".to_string()), + ); + out.insert("service".to_string(), Value::String("app".to_string())); + if !out.contains_key("workspaceFolder") { + out.insert( + "workspaceFolder".to_string(), + Value::String("/workspace".to_string()), + ); + } + let services = if prng.next_bool() { + Value::Array(vec![Value::String("db".to_string())]) + } else { + Value::Array(vec![ + Value::String("db".to_string()), + Value::String("cache".to_string()), + ]) + }; + let count = services.as_array().map(Vec::len).unwrap_or(0); + out.insert("runServices".to_string(), services); + Some(( + out, + format!("introduced a Compose source with {count} `runServices`"), + )) + } + } +} + +/// `mop-ordering-change` — permute a declaration-ordered collection. +/// +/// Grammar input: `array-shape`. Returns `None` when the document holds no collection with +/// at least two members: there is no order to change, and manufacturing one would record +/// an ordering mutation that never happened. That honesty is what makes FR-010's +/// zero-count report mean something. +/// +/// The permutation is guaranteed to *differ* from the input — a shuffle that happens to +/// return the identity is rotated — because a mutation that changed nothing would inflate +/// the application count without exploring anything. +fn ordering_change( + object: &Map, + prng: &mut Prng, +) -> Option<(Map, String)> { + let mut candidates: Vec = object + .iter() + .filter(|(_, v)| matches!(v, Value::Array(a) if a.len() >= 2)) + .map(|(k, _)| k.clone()) + .collect(); + candidates.sort_unstable(); + let key = prng.choose(&candidates)?.clone(); + let Some(Value::Array(items)) = object.get(&key) else { + return None; + }; + + let mut permuted = items.clone(); + prng.shuffle(&mut permuted); + if permuted == *items { + permuted.rotate_left(1); + } + + let mut out = object.clone(); + let len = permuted.len(); + out.insert(key.clone(), Value::Array(permuted)); + Some((out, format!("permuted the {len} elements of `{key}`"))) +} + +// --------------------------------------------------------------------------- +// Shared helpers +// --------------------------------------------------------------------------- + +/// Pick a key of `object` satisfying `predicate`, drawn uniformly from the **sorted** +/// candidate list. +/// +/// Sorting matters for reproducibility: `serde_json` preserves insertion order in this +/// crate, so two documents that are `==` as maps can enumerate their keys differently, and +/// drawing from the enumeration order would make the same seed pick different targets for +/// documents the comparison calls identical. +fn pick_key( + object: &Map, + prng: &mut Prng, + predicate: impl Fn(&str, &Value) -> bool, +) -> Option { + let mut keys: Vec<&String> = object + .iter() + .filter(|(k, v)| predicate(k.as_str(), v)) + .map(|(k, _)| k) + .collect(); + keys.sort_unstable(); + prng.choose(&keys).map(|k| (*k).clone()) +} + +/// A value of a **different** JSON type than `current`, chosen deterministically. +fn other_type(current: &Value) -> Value { + match current { + Value::String(_) => Value::Number(42.into()), + Value::Number(_) => Value::String("42".to_string()), + Value::Bool(b) => Value::String(b.to_string()), + Value::Array(_) => Value::String("joined,as,a,string".to_string()), + Value::Object(_) => Value::Array(vec![Value::String("was-an-object".to_string())]), + Value::Null => Value::Number(0.into()), + } +} + +/// A scalar draw for the unknown-field operator, covering every scalar type so an unknown +/// key is not always a string. +fn scalar_draw(prng: &mut Prng) -> Value { + match prng.next_bounded(4).unwrap_or(0) { + 0 => Value::String("discovery".to_string()), + 1 => Value::Number(7.into()), + 2 => Value::Bool(true), + _ => Value::Null, + } +} + +/// The JSON type name, for readable mutation details. +fn json_type(v: &Value) -> &'static str { + match v { + Value::Null => "null", + Value::Bool(_) => "boolean", + Value::Number(_) => "number", + Value::String(_) => "string", + Value::Array(_) => "array", + Value::Object(_) => "object", + } +} + +/// Render a lifecycle command element as a shell-ish string for the object → string +/// rotation. +fn render_command(v: &Value) -> String { + match v { + Value::String(s) => s.clone(), + Value::Array(items) => items + .iter() + .map(render_command) + .collect::>() + .join(" "), + other => other.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + /// A document rich enough that every operator has a target: a config source, a + /// feature, a lifecycle hook, a string field, and two multi-element arrays. + fn rich() -> Value { + json!({ + "name": "Discovery Seed", + "image": "alpine:3.19", + "features": { "ghcr.io/devcontainers/features/git:1": { "version": "os-provided" } }, + "forwardPorts": [3000, 8080], + "runArgs": ["--init", "--rm"], + "containerEnv": { "A": "1", "B": "2" }, + "postCreateCommand": "echo hello", + "remoteUser": "vscode" + }) + } + + fn apply_ok(category: MutationCategory, doc: &Value, seed: u64) -> Applied { + let mut prng = Prng::from_seed(seed); + apply(category, doc, &mut prng) + .unwrap_or_else(|| panic!("{} must apply to the rich document", category.name())) + } + + // --- catalogue identity ------------------------------------------------- + + #[test] + fn the_catalogue_declares_exactly_eleven_categories() { + // FR-008 mandates eleven. Asserting the arity here means adding a twelfth is a + // deliberate act that also has to update `MUTATION_CATALOG_VERSION`, rather than + // a quiet append nobody notices in the outcome map. + assert_eq!(MutationCategory::all().len(), CATEGORY_COUNT); + assert_eq!(CATEGORY_COUNT, 11); + assert_eq!(category_names().len(), 11); + } + + #[test] + fn names_and_operator_ids_are_unique_and_round_trip() { + let mut names: Vec<&str> = category_names(); + let mut ids: Vec<&str> = MutationCategory::all() + .iter() + .map(|c| c.operator_id()) + .collect(); + names.sort_unstable(); + ids.sort_unstable(); + let unique_names = { + let mut d = names.clone(); + d.dedup(); + d.len() + }; + let unique_ids = { + let mut d = ids.clone(); + d.dedup(); + d.len() + }; + assert_eq!(unique_names, 11, "two categories share a name"); + assert_eq!(unique_ids, 11, "two categories share a `mop-` id"); + + for category in MutationCategory::all() { + assert_eq!(MutationCategory::parse(category.name()), Some(*category)); + assert_eq!( + MutationCategory::parse_operator(category.operator_id()), + Some(*category) + ); + assert_eq!( + category.operator_id(), + format!("mop-{}", category.name()), + "the operator id is the declared name, prefixed — never an independent \ + spelling that could drift from it" + ); + } + assert_eq!(MutationCategory::parse("not-a-category"), None); + assert_eq!(MutationCategory::parse_operator("wrong-type"), None); + } + + #[test] + fn the_empty_count_map_carries_every_key_at_zero_in_declaration_order() { + // FR-010: a category absent from the map is indistinguishable from a category + // that was never applied. This map is the single source every producer starts + // from, so the property is established once rather than re-established per caller. + let counts = empty_application_counts(); + assert_eq!(counts.len(), 11); + let keys: Vec<&str> = counts.keys().map(String::as_str).collect(); + assert_eq!( + keys, + category_names(), + "declaration order, not sorted order" + ); + assert!(counts.values().all(|&n| n == 0)); + assert_eq!(unapplied_categories(&counts).len(), 11); + + let mut applied = counts.clone(); + for name in category_names() { + applied.insert(name.to_string(), 1); + } + assert!(unapplied_categories(&applied).is_empty()); + + let mut partial = counts; + partial.insert("wrong-type".to_string(), 3); + assert_eq!(unapplied_categories(&partial).len(), 10); + assert!(!unapplied_categories(&partial).contains(&"wrong-type")); + } + + // --- T032: per-operator behavior --------------------------------------- + + #[test] + fn every_operator_applies_to_a_rich_document_and_stays_schema_adjacent() { + // The whole point of structural mutation (research D5's argument, applied at + // generation time): a mutated candidate must still PARSE, or the campaign spends + // its budget on the document-syntax stage instead of on configuration + // resolution — the pathology SC-002 caps at 10%. + let doc = rich(); + for category in MutationCategory::all() { + let applied = apply_ok(*category, &doc, 0x5EED_0001); + assert!( + applied.document.is_object(), + "{} produced a non-object document", + category.name() + ); + let rendered = serde_json::to_string(&applied.document).unwrap_or_else(|e| { + panic!("{} produced unserializable JSON: {e}", category.name()) + }); + let reparsed: Value = serde_json::from_str(&rendered).unwrap_or_else(|e| { + panic!( + "{} produced JSON that does not round-trip: {e}", + category.name() + ) + }); + assert_eq!(reparsed, applied.document); + assert_ne!( + applied.document, + doc, + "{} changed nothing — an application count that includes no-ops counts \ + nothing", + category.name() + ); + assert_eq!(applied.mutation.category, *category); + assert_eq!(applied.mutation.operator, category.operator_id()); + assert!( + !applied.mutation.detail.is_empty(), + "{} recorded no detail; a witness must name what was applied where", + category.name() + ); + } + } + + #[test] + fn unknown_field_inserts_a_key_the_schema_does_not_declare() { + let doc = rich(); + let applied = apply_ok(MutationCategory::UnknownField, &doc, 11); + let out = applied.document.as_object().expect("object"); + let added: Vec<&String> = out + .keys() + .filter(|k| !doc.as_object().expect("object").contains_key(*k)) + .collect(); + assert_eq!(added.len(), 1, "exactly one key added"); + assert!( + UNKNOWN_FIELD_NAMES.contains(&added[0].as_str()), + "the inserted key must come from the declared pool so a finding names a key \ + a reviewer can grep for" + ); + } + + #[test] + fn wrong_type_changes_the_json_type_and_nothing_else() { + let doc = rich(); + let applied = apply_ok(MutationCategory::WrongType, &doc, 22); + let before = doc.as_object().expect("object"); + let after = applied.document.as_object().expect("object"); + assert_eq!(before.len(), after.len(), "no key added or removed"); + let changed: Vec<&String> = after + .keys() + .filter(|k| before.get(*k) != after.get(*k)) + .collect(); + assert_eq!(changed.len(), 1); + let key = changed[0]; + assert_ne!( + json_type(&before[key]), + json_type(&after[key]), + "a wrong-TYPE mutation that keeps the type is a value change wearing the \ + wrong operator's name" + ); + } + + #[test] + fn null_value_and_empty_value_are_not_the_same_mutation() { + // The spec distinguishes an authored `null` from an authored empty collection + // from an omission; conflating them is the defect 023 T062 removed from the + // normalizer, and re-introducing it in the generator would be the same mistake + // one stage earlier. + let doc = rich(); + let nulled = apply_ok(MutationCategory::NullValue, &doc, 33).document; + let emptied = apply_ok(MutationCategory::EmptyValue, &doc, 33).document; + + let nulled_keys: Vec<&String> = nulled + .as_object() + .expect("object") + .iter() + .filter(|(_, v)| v.is_null()) + .map(|(k, _)| k) + .collect(); + assert_eq!(nulled_keys.len(), 1, "exactly one key became null"); + + let before = doc.as_object().expect("object"); + let after = emptied.as_object().expect("object"); + let changed: Vec<&String> = after + .keys() + .filter(|k| before.get(*k) != after.get(*k)) + .collect(); + assert_eq!(changed.len(), 1); + let key = changed[0]; + assert!(!after[key].is_null(), "empty-value must not produce a null"); + assert_eq!( + json_type(&before[key]), + json_type(&after[key]), + "emptied IN PLACE: the type is preserved" + ); + let is_empty = match &after[key] { + Value::Array(a) => a.is_empty(), + Value::Object(o) => o.is_empty(), + Value::String(s) => s.is_empty(), + other => panic!("unexpected emptied shape {other}"), + }; + assert!(is_empty); + } + + #[test] + fn conflicting_source_leaves_two_sources_declared() { + let doc = rich(); // carries `image` + let applied = apply_ok(MutationCategory::ConflictingSource, &doc, 44); + let out = applied.document.as_object().expect("object"); + let sources = CONFIG_SOURCE_KEYS + .iter() + .filter(|k| out.contains_key(**k)) + .count(); + assert!( + sources >= 2, + "the schema's oneOf permits exactly one source; violating it is the whole \ + point of this operator, and it needs at least two present to do so" + ); + } + + #[test] + fn invalid_feature_id_corrupts_the_identifier_and_keeps_the_options() { + let doc = rich(); + let applied = apply_ok(MutationCategory::InvalidFeatureId, &doc, 55); + let features = applied.document["features"].as_object().expect("features"); + assert_eq!(features.len(), 1, "the entry is corrupted, not duplicated"); + let (id, options) = features.iter().next().expect("one entry"); + assert_ne!(id, "ghcr.io/devcontainers/features/git:1"); + assert_eq!( + options, + &json!({ "version": "os-provided" }), + "the identifier is the subject; the options ride along unchanged" + ); + + // With no `features` at all, the operator introduces a malformed one rather than + // declining: the identifier is its subject whether or not one already exists. + let bare = json!({ "image": "alpine:3.19" }); + let applied = apply_ok(MutationCategory::InvalidFeatureId, &bare, 55); + assert!(applied.document["features"].is_object()); + } + + #[test] + fn extends_cycle_produces_a_cycle_and_declines_to_re_apply_the_same_one() { + let doc = rich(); + let applied = apply_ok(MutationCategory::ExtendsCycle, &doc, 66); + let extends = &applied.document["extends"]; + let refers_to_self = match extends { + Value::String(s) => s == "./devcontainer.json", + Value::Array(items) => items.iter().all(|v| v == "./devcontainer.json"), + other => panic!("unexpected extends shape {other}"), + }; + assert!(refers_to_self); + + // Re-applying the identical cycle changes nothing, so the operator declines + // rather than inflating the application count with a no-op. + let mut prng = Prng::from_seed(66); + assert!( + apply(MutationCategory::ExtendsCycle, &applied.document, &mut prng).is_none(), + "an operator that changes nothing must not report an application" + ); + } + + #[test] + fn substitution_edge_writes_a_declared_edge_token() { + let doc = rich(); + let applied = apply_ok(MutationCategory::SubstitutionEdge, &doc, 77); + let out = applied.document.as_object().expect("object"); + let tokens: Vec<&str> = SUBSTITUTION_EDGES.iter().map(|(_, t)| *t).collect(); + assert!( + out.values() + .any(|v| v.as_str().is_some_and(|s| tokens.contains(&s))), + "the mutated document must carry one of the declared edge tokens" + ); + + // A document with no string field still gets one: `name` is a string on every + // branch of the schema, so the result stays schema-adjacent. + let no_strings = json!({ "forwardPorts": [1, 2] }); + let applied = apply_ok(MutationCategory::SubstitutionEdge, &no_strings, 77); + assert!(applied.document["name"].is_string()); + } + + #[test] + fn lifecycle_shape_rotates_string_to_array_to_object_to_string() { + let string_form = json!({ "image": "alpine:3.19", "postCreateCommand": "echo hi" }); + let as_array = apply_ok(MutationCategory::LifecycleShape, &string_form, 88).document; + assert!( + as_array["postCreateCommand"].is_array(), + "string rotates to array" + ); + + let as_object = apply_ok(MutationCategory::LifecycleShape, &as_array, 88).document; + assert!( + as_object["postCreateCommand"].is_object(), + "array rotates to object" + ); + + let back_to_string = apply_ok(MutationCategory::LifecycleShape, &as_object, 88).document; + assert!( + back_to_string["postCreateCommand"].is_string(), + "object rotates back to string — the three forms are a cycle, so a campaign \ + reaches all of them" + ); + + // All three forms are legal, so every rotation is a VALID document. That is what + // makes this category able to surface a defect rather than a rejection. + for doc in [&as_array, &as_object, &back_to_string] { + assert!(doc.is_object()); + } + } + + #[test] + fn compose_combination_varies_the_override_list_and_its_order() { + let single = json!({ + "dockerComposeFile": "docker-compose.yml", + "service": "app", + "workspaceFolder": "/workspace" + }); + let as_list = apply_ok(MutationCategory::ComposeCombination, &single, 99).document; + let files = as_list["dockerComposeFile"].as_array().expect("array"); + assert_eq!( + files.len(), + 2, + "a single file becomes an ordered override list" + ); + + let reversed = apply_ok(MutationCategory::ComposeCombination, &as_list, 99).document; + let reversed_files = reversed["dockerComposeFile"].as_array().expect("array"); + assert_eq!( + reversed_files, + &files.iter().rev().cloned().collect::>(), + "override ORDER is the semantics — later files override earlier ones — so \ + reversing it is a real mutation, not a cosmetic one" + ); + + // A non-Compose document gains a complete Compose branch, not a half of one. + let image = json!({ "image": "alpine:3.19" }); + let composed = apply_ok(MutationCategory::ComposeCombination, &image, 99).document; + assert!(composed["dockerComposeFile"].is_string()); + assert!(composed["service"].is_string()); + assert!(composed["workspaceFolder"].is_string()); + assert!(composed["runServices"].is_array()); + } + + #[test] + fn ordering_change_permutes_a_collection_and_never_returns_the_identity() { + let doc = rich(); + let applied = apply_ok(MutationCategory::OrderingChange, &doc, 1234); + let before = doc.as_object().expect("object"); + let after = applied.document.as_object().expect("object"); + let changed: Vec<&String> = after + .keys() + .filter(|k| before.get(*k) != after.get(*k)) + .collect(); + assert_eq!(changed.len(), 1); + let key = changed[0]; + + let mut before_sorted: Vec = before[key] + .as_array() + .expect("array") + .iter() + .map(|v| v.to_string()) + .collect(); + let mut after_sorted: Vec = after[key] + .as_array() + .expect("array") + .iter() + .map(|v| v.to_string()) + .collect(); + before_sorted.sort(); + after_sorted.sort(); + assert_eq!( + before_sorted, after_sorted, + "a permutation, not a rewrite: the multiset must be preserved" + ); + assert_ne!( + before[key], after[key], + "a shuffle that returned the identity is rotated, because a mutation that \ + changed nothing would inflate the count without exploring anything" + ); + } + + #[test] + fn ordering_change_declines_when_there_is_no_order_to_change() { + // The honesty that makes FR-010's zero-count report mean something: rather than + // manufacture a collection to permute, the operator reports that it did not apply. + let flat = json!({ "image": "alpine:3.19", "name": "flat", "forwardPorts": [3000] }); + let mut prng = Prng::from_seed(7); + assert!(apply(MutationCategory::OrderingChange, &flat, &mut prng).is_none()); + } + + #[test] + fn a_non_object_document_has_nothing_to_mutate() { + let mut prng = Prng::from_seed(1); + for doc in [json!([1, 2, 3]), json!("string"), json!(null)] { + for category in MutationCategory::all() { + assert!( + apply(*category, &doc, &mut prng).is_none(), + "{} claimed to mutate a non-object", + category.name() + ); + } + } + } + + // --- reproducibility ---------------------------------------------------- + + #[test] + fn the_same_seed_and_document_reproduce_the_same_mutation() { + // FR-001 in miniature at the operator level: without this, a recorded seed could + // not reproduce a candidate even with an identical generator. + let doc = rich(); + for category in MutationCategory::all() { + let mut a = Prng::from_seed(0xC0FF_EE01); + let mut b = Prng::from_seed(0xC0FF_EE01); + assert_eq!( + apply(*category, &doc, &mut a), + apply(*category, &doc, &mut b), + "{} is not reproducible from its seed", + category.name() + ); + } + } + + #[test] + fn target_selection_does_not_depend_on_key_insertion_order() { + // `serde_json` preserves insertion order in this crate, so two documents that are + // `==` as maps can enumerate their keys differently. Drawing from the enumeration + // order would make the same seed pick different targets for documents the + // comparison itself calls identical — a reproducibility hole with no symptom + // until a finding stopped reproducing for no visible reason. + let a = json!({ "image": "alpine:3.19", "name": "x", "remoteUser": "vscode" }); + let b = json!({ "remoteUser": "vscode", "name": "x", "image": "alpine:3.19" }); + assert_eq!(a, b, "serde_json compares objects as maps"); + + for category in [ + MutationCategory::WrongType, + MutationCategory::NullValue, + MutationCategory::SubstitutionEdge, + ] { + let left = apply_ok(category, &a, 0xDEAD_BEEF); + let right = apply_ok(category, &b, 0xDEAD_BEEF); + assert_eq!( + left.mutation.detail, + right.mutation.detail, + "{} picked a different target for two equal documents", + category.name() + ); + assert_eq!(left.document, right.document); + } + } + + // ----------------------------------------------------------------------- + // The reversal record (US2, data-model.md § 6 step 2) + // ----------------------------------------------------------------------- + + #[test] + fn every_operator_records_a_reversal_that_exactly_undoes_it() { + // The property the shrinker's `un-apply-mutation` step rests on. Asserted for + // every category rather than for a representative few: a wrong reversal does not + // fail loudly, it quietly produces a reduced input that no longer reproduces + // anything, and the finding is then recorded against an input nobody can + // re-examine. + let base = rich(); + for category in *MutationCategory::all() { + let applied = apply_ok(category, &base, 0x5EED_0041); + assert_ne!( + applied.document, + base, + "{} recorded an application that changed nothing", + category.name() + ); + assert!( + !applied.mutation.reversal.is_empty(), + "{} changed the document but named no site to reverse", + category.name() + ); + assert_eq!( + applied.mutation.reversal.apply(&applied.document), + base, + "{} does not reverse exactly", + category.name() + ); + } + } + + #[test] + fn a_reversal_removes_a_key_the_mutation_introduced() { + // The `None` arm: a key absent before the mutation must be REMOVED by the + // reversal, never restored as `null`. The spec distinguishes an authored `null` + // from an omission, and conflating them here would reverse an `unknown-field` + // insertion into an authored null field the campaign never produced. + let base = json!({ "image": "alpine:3.19" }); + let applied = apply_ok(MutationCategory::UnknownField, &base, 0x5EED_0042); + assert_eq!( + applied.document.as_object().map(Map::len), + Some(2), + "one key was inserted" + ); + let reversed = applied.mutation.reversal.apply(&applied.document); + assert_eq!(reversed, base); + assert_eq!( + reversed.as_object().map(Map::len), + Some(1), + "the introduced key is removed, never left behind as `null`" + ); + } + + #[test] + fn a_reversal_is_a_local_edit_and_leaves_unrelated_reductions_alone() { + // Why the reversal names sites rather than carrying the whole pre-image: by the + // time `un-apply-mutation` runs, earlier catalogue steps have already reduced + // other parts of the document, and restoring a whole pre-image would silently + // undo all of it. + let base = rich(); + let applied = apply_ok(MutationCategory::NullValue, &base, 0x5EED_0043); + + // Simulate an earlier reduction: drop an unrelated optional key. + let mut reduced = applied.document.as_object().expect("object").clone(); + reduced.remove("runArgs"); + let reduced = Value::Object(reduced); + + let reversed = applied.mutation.reversal.apply(&reduced); + assert!( + reversed.get("runArgs").is_none(), + "un-applying a mutation must not resurrect a key an earlier reduction dropped" + ); + // …and it did still reverse its own site. + let key = applied + .mutation + .reversal + .keys + .first() + .map(|(k, _)| k.clone()) + .expect("one site"); + assert_eq!(reversed.get(&key), base.get(&key)); + } + + #[test] + fn a_reversal_of_a_non_object_document_changes_nothing() { + let reversal = Reversal { + keys: vec![("image".to_string(), None)], + }; + assert_eq!(reversal.apply(&json!([1, 2])), json!([1, 2])); + } +} diff --git a/crates/conformance/src/discovery/promote.rs b/crates/conformance/src/discovery/promote.rs new file mode 100644 index 00000000..1074a244 --- /dev/null +++ b/crates/conformance/src/discovery/promote.rs @@ -0,0 +1,650 @@ +//! Review-only promotion: the skeletons `discovery scaffold` prints, and the rules that +//! keep promotion a **human** act (025-exploratory-parity-discovery, US5, +//! FR-036 – FR-042, contracts/discovery-cli.md § `discovery scaffold`). +//! +//! ## Nothing here writes anything +//! +//! Every function in this module is a pure `Finding → JSON` transformation. There is no +//! path, no file handle, and no writer — not as a discipline someone must remember, but +//! because the module never takes one. `discovery scaffold` prints the result to +//! **stdout** and stops; the reviewer copies what they judge correct into the registry by +//! hand. That is what makes FR-036 hold: a stochastic process cannot author the record it +//! is tested against, because there is no code path from a finding to a registry write. +//! +//! ## Why every field carries a sentinel the loader rejects +//! +//! A skeleton whose fields were plausible defaults would invite committing it unread, and +//! the one thing a finding cannot tell you is the thing a behavior record must state: a +//! difference says *what* differs, never whether deacon is wrong, the reference is wrong, +//! or the specification is silent. So each field a human must decide carries +//! [`UNREVIEWED`], which the registry loader rejects +//! ([`crate::residual::UNREVIEWED_SENTINEL`]) — the same discipline `inventory scaffold`, +//! `clause scaffold`, `coverage scaffold`, and `migration scaffold` already use. +//! +//! ## The one rule that is a rule rather than a sentinel (FR-041) +//! +//! A tolerance must be **scoped**. [`reject_blanket_observable_path`] is the single +//! definition of that rule: an `observablePath` that names a bare channel is a global +//! ignore list wearing a waiver id, and the registry rejects one at load (V19). Emitting +//! one here and letting the loader catch it later would be worse than refusing, because +//! the intermediate state is a reviewer holding a document that looks authored. + +use serde_json::{Value, json}; + +use crate::model::{CHAN_EXIT_CODE, Expect, Scope}; +use crate::residual::UNREVIEWED_SENTINEL; + +use super::queue::{Finding, ObservedValues}; +use super::signature::Signature; + +/// The sentinel every field a human must decide carries. +/// +/// Re-exported from [`crate::residual`] rather than redeclared: two spellings of "not yet +/// reviewed" is exactly the drift that would let one scaffold's output slip past the +/// loader rejection the other relies on. +pub const UNREVIEWED: &str = UNREVIEWED_SENTINEL; + +/// The three disposition axes a behavior record must carry (FR-037/FR-038). +/// +/// Named here as the *promotion pre-flight's* view of them and cross-checked against +/// [`crate::model::BehaviorUnit`]'s own serialization by this module's +/// `the_axis_names_match_the_behavior_record` test, so renaming an axis on the model fails +/// this module rather than silently making the pre-flight check a field that no longer +/// exists. +pub const BEHAVIOR_DISPOSITION_AXES: [&str; 3] = ["spec", "reference", "decision"]; + +/// Why a promotion skeleton could not be produced, or why a proposed promotion is not yet +/// a promotion. +/// +/// Every variant names the *remedy*, not only the rule: a reviewer meeting one of these +/// is mid-task, and "what is missing" is the only useful thing to say. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum PromotionError { + /// FR-035: `normalizer-defect` / `fixture-defect` describe a defect in the discovery + /// or comparison machinery, not a behavior of either implementation. + #[error( + "finding `{finding}` is classified `{classification}`, which is not promotable \ + (FR-035): it describes a defect in the discovery machinery, not a behavior of \ + either implementation. Remedy: fix the normalizer or the generator instead — no \ + registry case can legitimately carry it." + )] + NonPromotable { + finding: String, + classification: &'static str, + }, + + /// FR-041: a tolerance whose `observablePath` names a bare channel tolerates + /// everything on that channel forever. + #[error( + "tolerance for `{finding}` would be UNSCOPED: `{path}` names a bare channel, which \ + is a global ignore list wearing a waiver id. Remedy: scope it to the observable \ + path the difference actually occurs at — the registry rejects a bare-channel \ + scope at load (V19), and a blanket tolerance silences differences nobody reviewed." + )] + UnscopedTolerance { finding: String, path: String }, + + /// FR-037: the promotion names no behavior, or still carries the scaffold sentinel. + #[error( + "promotion of `{finding}` carries no behavior identity ({detail}). Remedy: name an \ + existing `bhv-` record, or author a new one with all three disposition axes — a \ + promotion with no stable identity cannot be found again, which is the whole point \ + of promoting it." + )] + MissingBehaviorIdentity { finding: String, detail: String }, + + /// FR-038: a disposition axis is absent, blank, or still the scaffold sentinel. + #[error( + "promotion of `{finding}` has no `{axis}` disposition ({detail}). Remedy: state it. \ + A finding tells you what DIFFERS; it never tells you whether deacon is wrong, the \ + reference is wrong, or the specification is silent — that judgement is the review, \ + and a record missing it claims an evidence-backed status nobody established." + )] + MissingDisposition { + finding: String, + axis: &'static str, + detail: String, + }, + + /// FR-042 / **D3**, checked *before* the write rather than only after it. + #[error( + "promotion of `{finding}` names case `{case}`, which is not declared in the \ + registry. Remedy: commit the case first, then point `promotedTo` at its real id — \ + a promotion the registry cannot back reads as covered while nothing executes it." + )] + UnresolvedCase { finding: String, case: String }, +} + +/// Reject an `observablePath` that is not scoped **within** a channel (FR-041, V19). +/// +/// The single definition of the rule, called by [`observable_path`] and driven directly by +/// the guard that asserts a blanket scope is refused. A second copy would be the one that +/// eventually accepted a bare channel. +/// +/// Scoped means: a channel id, a `.`, and a non-empty remainder. `chan-structured-output` +/// is refused; `chan-structured-output.configuration` is accepted, and it legitimately +/// covers everything beneath it — the harness's tolerance index matches a scoped path plus +/// its subtree, which is a decision a reviewer made about one field, not a standing licence +/// over a channel. +pub fn reject_blanket_observable_path(finding: &str, path: &str) -> Result<(), PromotionError> { + let scoped = path + .split_once('.') + .is_some_and(|(channel, rest)| !channel.trim().is_empty() && !rest.trim().is_empty()); + if scoped { + return Ok(()); + } + Err(PromotionError::UnscopedTolerance { + finding: finding.to_string(), + path: path.to_string(), + }) +} + +/// The scoped `observablePath` for a signature: `.`. +/// +/// Fails when the signature carries no observable path at all, which would produce a bare +/// channel — the blanket tolerance FR-041 forbids. +pub fn observable_path(finding: &str, signature: &Signature) -> Result { + let path = format!("{}.{}", signature.channel, signature.path); + reject_blanket_observable_path(finding, &path)?; + Ok(path) +} + +/// Refuse a finding whose classification can never lead to a registry record (FR-035). +fn require_promotable(finding: &Finding) -> Result<(), PromotionError> { + match finding.classification { + Some(classification) if !classification.is_promotable() => { + Err(PromotionError::NonPromotable { + finding: finding.id.clone(), + classification: classification.as_str(), + }) + } + _ => Ok(()), + } +} + +/// The behavior + case + fixture skeleton `discovery scaffold` prints (FR-037/FR-039). +/// +/// The **fixture** is the one part that is not a sentinel: it is the witness's own reduced +/// input, verbatim. That asymmetry is the point — the machine knows what input reproduces +/// the difference and the human does not, whereas the human knows what the difference +/// *means* and the machine does not. Filling in the half a finding can actually support, +/// and refusing to guess the other half, is what keeps the reviewer's job deciding rather +/// than verifying. +pub fn promotion_skeleton(finding: &Finding) -> Result { + require_promotable(finding)?; + let signature = &finding.signature; + // A finding always carries at least one witness: the strict loader rejects an empty + // list and **D1** rejects a programmatically-built one, so the first is always there. + let Some(witness) = finding.witnesses.first() else { + return Err(PromotionError::MissingBehaviorIdentity { + finding: finding.id.clone(), + detail: "the finding carries no witness, so there is no input to promote".to_string(), + }); + }; + + Ok(json!({ + "behavior": { + "id": format!("bhv-{UNREVIEWED}"), + "summary": UNREVIEWED, + "spec": UNREVIEWED, + "reference": UNREVIEWED, + "decision": UNREVIEWED, + "notes": format!( + "Promoted from discovery finding {} (signature {}, channel {}, path {}, \ + {} / {}).", + finding.id, + signature.id, + signature.channel, + signature.path, + signature.kind.as_str(), + signature.value_shape_class.as_str(), + ), + }, + "case": { + "id": format!("case-{UNREVIEWED}"), + "behaviors": [format!("bhv-{UNREVIEWED}")], + "oracleType": UNREVIEWED, + "scenarioContext": UNREVIEWED, + "fixtures": [format!("fx-{UNREVIEWED}")], + }, + "fixture": { + "id": format!("fx-{UNREVIEWED}"), + "minimalInput": witness.minimal_input, + "isMinimal": witness.is_minimal, + "mutationOperators": witness.mutation_operators, + }, + })) +} + +/// The scoped waiver + the `allowedDifferences` entry that references it, for a finding a +/// reviewer chooses to **tolerate** rather than fix (FR-041, `--tolerate`). +/// +/// Two records rather than one, because a tolerance in this project is two records: the +/// `wvr-` carries the rationale and the expiry that make it self-invalidating, and the +/// case's scoped `allowedDifferences` entry is what the runner consults. Emitting only the +/// waiver would leave a reviewer with a record nothing reads; emitting only the allowed +/// difference would leave a tolerance with no expiry and no rationale — an unbacked +/// silence, which is the shape V19 exists to refuse. +/// +/// `expect` is the one derived field: it restates **what was observed**, which the witness +/// already carries, so the reviewer is not asked to re-read the evidence to retype it. +/// Everything a reviewer must *decide* — the behavior, the rationale, the expiry, the +/// binary/fixture the scope names — is a sentinel. +pub fn tolerance_skeleton(finding: &Finding) -> Result { + require_promotable(finding)?; + let observable_path = observable_path(&finding.id, &finding.signature)?; + let waiver_id = format!("wvr-{UNREVIEWED}"); + let behavior_id = format!("bhv-{UNREVIEWED}"); + + // Scoped to the observable path the difference occurs at. The harness's tolerance + // index reads `field` and matches it against a signature's path, so this is the field + // that makes the waiver do anything at all — and it is the one field here taken from + // the evidence rather than left to the reviewer, for the same reason `expect` is. + let scope = Scope::StateField { + binary: UNREVIEWED.to_string(), + fixture: UNREVIEWED.to_string(), + field: finding.signature.path.clone(), + }; + + Ok(json!({ + "waiver": { + "id": waiver_id, + "behaviors": [behavior_id], + "scope": scope, + "expect": observed_expectation(finding), + "rationale": UNREVIEWED, + "added": UNREVIEWED, + "expires": UNREVIEWED, + }, + "allowedDifference": { + "behavior": behavior_id, + // Never empty: an empty context reads as "everywhere", and a tolerance whose + // validity nobody bounded is the blanket tolerance in a different field. + "context": [UNREVIEWED], + "observablePath": observable_path, + "rationale": UNREVIEWED, + "waiverId": waiver_id, + }, + "note": format!( + "Tolerating finding {} means recording that this difference is ACCEPTABLE, not \ + that it stopped mattering. The waiver is self-invalidating: it fails as stale \ + the moment the difference stops reproducing. Fill in `rationale`, `expires`, \ + the behavior, and the binary/fixture the scope names — the `{UNREVIEWED}` \ + sentinels are rejected by the registry loader, so this cannot be committed \ + unedited.", + finding.id + ), + })) +} + +/// The `expect` a waiver would characterize, read from the finding's first witness. +/// +/// An accept/reject divergence is a **strictness** claim and gets the directional variant; +/// everything else is the concrete value difference. An observed absence renders as JSON +/// `null` — the same conflation [`ObservedValues`] already documents, harmless here +/// because the signature's own value-shape class carries the presence distinction and the +/// reviewer is editing this record anyway. +fn observed_expectation(finding: &Finding) -> Expect { + let observed = finding + .witnesses + .first() + .map(|w| w.observed_values.clone()) + .unwrap_or_default(); + let ObservedValues { deacon, reference } = observed; + + if finding.signature.channel == CHAN_EXIT_CODE { + match deacon.as_ref().and_then(Value::as_str) { + Some("rejected") => return Expect::DeaconStricter { signal: None }, + Some("accepted") => return Expect::ReferenceStricter { signal: None }, + _ => {} + } + } + Expect::FieldDivergence { + ours: deacon.unwrap_or(Value::Null), + reference: reference.unwrap_or(Value::Null), + } +} + +/// Pre-flight a **proposed** promotion: the behavior record a reviewer authored plus the +/// case id the finding would name (FR-037/FR-038/FR-042). +/// +/// Reports **every** problem in one pass rather than the first, matching `validate` and +/// `discovery check`: a reviewer fixing a batch should not have to rediscover the next +/// objection by re-running. +/// +/// This runs *before* the write; **D3** runs over committed data afterwards. Both exist on +/// purpose: the pre-flight is what tells a reviewer what is missing while they can still +/// act on it cheaply, and D3 is what keeps the claim true afterwards — a case deleted or +/// renamed later produces the same unresolvable promotion, and nothing in the registry can +/// notice, because the registry never reads the queue. +/// +/// A field that is absent, blank, or still carrying [`UNREVIEWED`] is treated identically: +/// all three mean "nobody decided this", and distinguishing them would only let a +/// scaffolded record slip through by having been *printed* rather than left out. +pub fn validate_promotion( + finding: &Finding, + behavior: &Value, + case_id: Option<&str>, + declared_cases: &[&str], +) -> Vec { + let mut errors = Vec::new(); + + if let Err(e) = require_promotable(finding) { + errors.push(e); + } + + match decided(behavior, "id") { + Ok(id) if id.starts_with("bhv-") => {} + Ok(id) => errors.push(PromotionError::MissingBehaviorIdentity { + finding: finding.id.clone(), + detail: format!("`id` is `{id}`, which is not a `bhv-` record id"), + }), + Err(detail) => errors.push(PromotionError::MissingBehaviorIdentity { + finding: finding.id.clone(), + detail, + }), + } + + for axis in BEHAVIOR_DISPOSITION_AXES { + if let Err(detail) = decided(behavior, axis) { + errors.push(PromotionError::MissingDisposition { + finding: finding.id.clone(), + axis, + detail, + }); + } + } + + match case_id { + None => errors.push(PromotionError::UnresolvedCase { + finding: finding.id.clone(), + case: "".to_string(), + }), + Some(case) if !declared_cases.contains(&case) => { + errors.push(PromotionError::UnresolvedCase { + finding: finding.id.clone(), + case: case.to_string(), + }); + } + Some(_) => {} + } + + errors +} + +/// The value of `field` when a human actually decided it, else why it does not count. +fn decided<'a>(record: &'a Value, field: &str) -> Result<&'a str, String> { + let Some(value) = record.get(field) else { + return Err(format!("`{field}` is absent")); + }; + let Some(text) = value.as_str() else { + return Err(format!( + "`{field}` is {value}, which is not a decided string" + )); + }; + let trimmed = text.trim(); + if trimmed.is_empty() { + return Err(format!("`{field}` is blank")); + } + // `contains`, not `==`: the scaffold prefixes ids with their record kind + // (`bhv-UNREVIEWED`), so an equality check would let the one field that must be a + // stable IDENTITY pass by virtue of having been decorated. + if trimmed.contains(UNREVIEWED) { + return Err(format!( + "`{field}` is `{trimmed}`, which still carries the `{UNREVIEWED}` scaffold \ + sentinel" + )); + } + Ok(trimmed) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::discovery::queue::{Classification, FindingState, Witness}; + use crate::discovery::signature::{Divergence, DivergenceKind}; + use crate::model::CHAN_STRUCTURED_OUTPUT; + + fn signature(channel: &str, path: &str) -> Signature { + let d = json!("vscode"); + let r = json!("root"); + Signature::derive( + channel, + &Divergence { + kind: DivergenceKind::Value, + path, + deacon: Some(&d), + reference: Some(&r), + }, + ) + } + + fn finding(signature: Signature) -> Finding { + let campaign = "cmp-11111111"; + let witness = Witness { + id: Witness::derived_id(campaign, "cnd-11111111"), + campaign_id: campaign.to_string(), + candidate_id: "cnd-11111111".to_string(), + minimal_input: json!({ "image": "alpine:3.18" }), + is_minimal: true, + reduction_steps: vec!["drop-optional-key".to_string()], + observed_values: ObservedValues { + deacon: Some(json!("vscode")), + reference: Some(json!("root")), + }, + mutation_operators: vec!["mop-wrong-type".to_string()], + }; + Finding::newly_admitted(signature, witness, campaign) + } + + #[test] + fn the_promotion_skeleton_carries_the_sentinel_everywhere_a_human_must_decide() { + let f = finding(signature( + CHAN_STRUCTURED_OUTPUT, + "configuration.remoteUser", + )); + let doc = promotion_skeleton(&f).expect("an untriaged finding scaffolds"); + for axis in BEHAVIOR_DISPOSITION_AXES { + assert_eq!( + doc["behavior"][axis], + json!(UNREVIEWED), + "`{axis}` must be a sentinel: a finding says what DIFFERS, never whose \ + behavior is right" + ); + } + assert_eq!(doc["behavior"]["id"], json!("bhv-UNREVIEWED")); + assert_eq!(doc["case"]["scenarioContext"], json!(UNREVIEWED)); + // The one part taken from the evidence: the input that reproduces it. + assert_eq!( + doc["fixture"]["minimalInput"], + json!({"image":"alpine:3.18"}) + ); + assert_eq!(doc["fixture"]["isMinimal"], json!(true)); + } + + #[test] + fn a_non_promotable_classification_is_refused_by_both_skeletons() { + for non_promotable in [ + Classification::NormalizerDefect, + Classification::FixtureDefect, + ] { + let mut f = finding(signature(CHAN_STRUCTURED_OUTPUT, "configuration.image")); + f.state = FindingState::Triaged; + f.classification = Some(non_promotable); + for result in [promotion_skeleton(&f), tolerance_skeleton(&f)] { + assert!( + matches!(result, Err(PromotionError::NonPromotable { .. })), + "{} must be refused by every promotion path (FR-035)", + non_promotable.as_str() + ); + } + } + } + + #[test] + fn a_tolerance_is_scoped_within_its_channel_never_to_the_channel() { + let f = finding(signature( + CHAN_STRUCTURED_OUTPUT, + "configuration.remoteUser", + )); + let doc = tolerance_skeleton(&f).expect("scaffolds"); + assert_eq!( + doc["allowedDifference"]["observablePath"], + json!("chan-structured-output.configuration.remoteUser") + ); + assert_eq!(doc["allowedDifference"]["waiverId"], doc["waiver"]["id"]); + assert_eq!( + doc["waiver"]["scope"]["field"], + json!("configuration.remoteUser"), + "the scope names the observable path, which is what makes the waiver do \ + anything at all" + ); + for sentinel in [ + &doc["waiver"]["rationale"], + &doc["waiver"]["expires"], + &doc["waiver"]["added"], + &doc["allowedDifference"]["rationale"], + ] { + assert_eq!(*sentinel, json!(UNREVIEWED)); + } + } + + #[test] + fn a_bare_channel_observable_path_is_refused() { + // The FR-041 rule, driven directly. A bare channel tolerates everything on that + // channel forever, which is a global ignore list wearing a waiver id. + for blanket in [ + "chan-structured-output", + "chan-structured-output.", + ".configuration", + "", + ] { + assert!( + matches!( + reject_blanket_observable_path("fnd-x", blanket), + Err(PromotionError::UnscopedTolerance { .. }) + ), + "{blanket:?} must be refused" + ); + } + reject_blanket_observable_path("fnd-x", "chan-structured-output.configuration") + .expect("a scoped path is accepted"); + } + + #[test] + fn a_signature_with_no_path_cannot_be_tolerated_at_all() { + let f = finding(signature(CHAN_STRUCTURED_OUTPUT, "")); + assert!(matches!( + tolerance_skeleton(&f), + Err(PromotionError::UnscopedTolerance { .. }) + )); + } + + #[test] + fn an_outcome_divergence_becomes_a_directional_strictness_expectation() { + let mut f = finding(signature(CHAN_EXIT_CODE, "outcome")); + f.witnesses[0].observed_values = ObservedValues { + deacon: Some(json!("rejected")), + reference: Some(json!("accepted")), + }; + let doc = tolerance_skeleton(&f).expect("scaffolds"); + assert_eq!(doc["waiver"]["expect"]["kind"], json!("deacon-stricter")); + + f.witnesses[0].observed_values = ObservedValues { + deacon: Some(json!("accepted")), + reference: Some(json!("rejected")), + }; + let doc = tolerance_skeleton(&f).expect("scaffolds"); + assert_eq!(doc["waiver"]["expect"]["kind"], json!("reference-stricter")); + } + + #[test] + fn a_value_divergence_becomes_a_field_divergence_expectation() { + let f = finding(signature( + CHAN_STRUCTURED_OUTPUT, + "configuration.remoteUser", + )); + let doc = tolerance_skeleton(&f).expect("scaffolds"); + assert_eq!(doc["waiver"]["expect"]["kind"], json!("field-divergence")); + assert_eq!(doc["waiver"]["expect"]["ours"], json!("vscode")); + assert_eq!(doc["waiver"]["expect"]["reference"], json!("root")); + } + + #[test] + fn the_scaffolded_behavior_fails_the_promotion_pre_flight_on_every_axis() { + // The scaffold must never be committable unedited: its own output is exactly what + // FR-038 rejects, and it says so by axis rather than as one lump. + let f = finding(signature( + CHAN_STRUCTURED_OUTPUT, + "configuration.remoteUser", + )); + let doc = promotion_skeleton(&f).expect("scaffolds"); + let errors = validate_promotion(&f, &doc["behavior"], None, &[]); + + for axis in BEHAVIOR_DISPOSITION_AXES { + assert!( + errors.iter().any(|e| matches!( + e, + PromotionError::MissingDisposition { axis: a, .. } if *a == axis + )), + "`{axis}` must be reported BY NAME: {errors:?}" + ); + } + assert!( + errors + .iter() + .any(|e| matches!(e, PromotionError::MissingBehaviorIdentity { .. })), + "the sentinel id is not an identity: {errors:?}" + ); + assert!( + errors + .iter() + .any(|e| matches!(e, PromotionError::UnresolvedCase { .. })), + "a promotion naming no case names nothing: {errors:?}" + ); + } + + #[test] + fn a_fully_decided_promotion_naming_a_declared_case_passes_the_pre_flight() { + let mut f = finding(signature( + CHAN_STRUCTURED_OUTPUT, + "configuration.remoteUser", + )); + f.state = FindingState::Triaged; + f.classification = Some(Classification::DeaconRegression); + let behavior = json!({ + "id": "bhv-readconfig-remote-user", + "spec": "conformant", + "reference": "divergent", + "decision": "follow-spec", + }); + assert_eq!( + validate_promotion(&f, &behavior, Some("case-real"), &["case-real"]), + Vec::new() + ); + } + + /// Cross-check against the model, so renaming an axis on [`crate::model::BehaviorUnit`] + /// fails here rather than leaving the pre-flight checking a field that no longer exists. + #[test] + fn the_axis_names_match_the_behavior_record() { + use crate::model::{BehaviorUnit, Decision, ReferenceStatus, SpecStatus}; + let unit = BehaviorUnit { + id: "bhv-x".to_string(), + area: "read-configuration".to_string(), + statement: "s".to_string(), + applicability: Vec::new(), + spec: SpecStatus::Conformant, + reference: ReferenceStatus::Aligned, + decision: Decision::FollowSpec, + notes: None, + }; + let rendered = serde_json::to_value(&unit).expect("a behavior serializes"); + for axis in BEHAVIOR_DISPOSITION_AXES { + assert!( + rendered.get(axis).is_some(), + "`{axis}` is not a field of the behavior record any more; the promotion \ + pre-flight would be checking something that does not exist" + ); + } + } +} diff --git a/crates/conformance/src/discovery/queue.rs b/crates/conformance/src/discovery/queue.rs new file mode 100644 index 00000000..f7675bda --- /dev/null +++ b/crates/conformance/src/discovery/queue.rs @@ -0,0 +1,3176 @@ +//! The findings queue — records, strict loader, atomic writer, signature-keyed upsert, +//! and the **D1**/**D5** violation classes +//! (025-exploratory-parity-discovery, data-model.md §§ 3–4, +//! contracts/findings-queue.md, T016–T021). +//! +//! ## Where this lives, and why it matters +//! +//! `conformance/discovery/` is a **sibling** of `conformance/registry/`, not a child. +//! [`crate::load::Registry::load`] enumerates *named* subdirectories under the registry +//! root and has no wildcard walk, so nothing here can be reached by the registry loader +//! or, transitively, by `certify`. The guarantee that an unreviewed finding can never +//! influence a release gate is therefore a property of the directory layout rather than +//! a rule someone must remember — which matters precisely because the failure mode is +//! silent: a finding quietly joining the certification denominator (research D6). +//! +//! Exactly one reference crosses the boundary, and it points **out**: +//! [`Finding::promoted_to`] → `case-`. Nothing in the registry points back. +//! +//! ## Duplicate findings are unrepresentable +//! +//! A finding's `id` is *derived* from its signature ([`Signature::finding_id`]), so +//! FR-030's "two findings with the same signature are one finding" is a property of the +//! identity function rather than an invariant the merge logic has to maintain — and can +//! therefore violate. [`upsert_finding`] appends a witness to the existing record; it +//! cannot create a second one. + +use indexmap::IndexMap; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::Value; +use std::path::{Path, PathBuf}; + +use crate::DiscoveryError; +use crate::load::{LoadError, SchemaError, deserialize_located, read_file}; +use crate::model::{RevisionKind, SourceRevision}; + +use super::signature::Signature; + +// --------------------------------------------------------------------------- +// T016 — Finding, Witness, FindingState, Classification (data-model.md § 3) +// --------------------------------------------------------------------------- + +/// The closed classification set (FR-028). Exactly one, once a finding is triaged. +/// +/// The last two are **non-promotable** (FR-035) because they describe a defect in the +/// discovery machinery, not a behavior of either implementation: resolving them changes +/// the normalizer or the generator, and promoting one would record a claim about deacon +/// or the reference that the evidence does not support. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Classification { + /// deacon is wrong; the reference and/or the spec is right. Promotes to a behavior + /// plus a fix. + DeaconRegression, + /// The reference diverges from the spec; deacon is right. Promotes to a behavior + /// plus a waiver. + ReferenceQuirk, + /// The spec does not decide the question. Promotes to a behavior with + /// `spec: unspecified`. + SpecAmbiguity, + /// deacon lacks the capability entirely. Promotes to a `gap-` record. + UnsupportedBehavior, + /// The comparison machinery manufactured or hid the difference. **Not promotable.** + NormalizerDefect, + /// The generated input was invalid in a way the harness cannot express. + /// **Not promotable.** + FixtureDefect, +} + +impl Classification { + /// The stable wire spelling. + pub fn as_str(self) -> &'static str { + match self { + Classification::DeaconRegression => "deacon-regression", + Classification::ReferenceQuirk => "reference-quirk", + Classification::SpecAmbiguity => "spec-ambiguity", + Classification::UnsupportedBehavior => "unsupported-behavior", + Classification::NormalizerDefect => "normalizer-defect", + Classification::FixtureDefect => "fixture-defect", + } + } + + /// Whether a finding carrying this classification may ever reach + /// [`FindingState::Promoted`] (FR-035). + pub fn is_promotable(self) -> bool { + !matches!( + self, + Classification::NormalizerDefect | Classification::FixtureDefect + ) + } + + /// Every classification, in declaration order — the CLI's `--classification` value + /// set and the report's bucket order. + pub fn all() -> &'static [Classification] { + &[ + Classification::DeaconRegression, + Classification::ReferenceQuirk, + Classification::SpecAmbiguity, + Classification::UnsupportedBehavior, + Classification::NormalizerDefect, + Classification::FixtureDefect, + ] + } + + /// Parse a wire spelling, returning `None` on anything else — never a default. + pub fn parse(s: &str) -> Option { + Classification::all() + .iter() + .copied() + .find(|c| c.as_str() == s) + } +} + +/// The finding lifecycle (data-model.md § 3's state machine). +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum FindingState { + /// Admitted by a campaign, nobody has looked yet. **Counted, never implicit** + /// (FR-029) — "not yet looked at" must never read as "nothing found". + Untriaged, + /// A reviewer assigned a classification. + Triaged, + /// A reviewer split a signature-merged finding whose witnesses had different causes. + /// The parent becomes an inert ancestor: it keeps its witnesses as historical record, + /// stops accepting new ones, and surrenders classification to its children — a split + /// exists precisely because one classification could not describe them all + /// (invariant Q10). + Split, + /// Carried by a real registry case. Terminal; `promotedTo` is set and must resolve. + Promoted, + /// Stopped reproducing. **A state, not a deletion** (FR-033): the disappearance is + /// information — it may mean a fix landed, or it may mean the generator stopped + /// reaching the input, and deleting the record destroys the ability to tell those + /// apart. A later campaign that reproduces it moves it back to `triaged`, keeping + /// its classification. + NoLongerReproducing, +} + +impl FindingState { + /// The stable wire spelling. + pub fn as_str(self) -> &'static str { + match self { + FindingState::Untriaged => "untriaged", + FindingState::Triaged => "triaged", + FindingState::Split => "split", + FindingState::Promoted => "promoted", + FindingState::NoLongerReproducing => "no-longer-reproducing", + } + } + + /// Every state, in lifecycle order — the report's bucket order. + pub fn all() -> &'static [FindingState] { + &[ + FindingState::Untriaged, + FindingState::Triaged, + FindingState::Split, + FindingState::Promoted, + FindingState::NoLongerReproducing, + ] + } + + /// Whether `self → next` is a transition data-model.md § 3's state machine declares + /// (**T069**). + /// + /// The permitted set is exactly the arrows in that diagram, and the three arrows it + /// does **not** draw are each absent for a reason worth stating, because each looks + /// plausible enough that someone will eventually add it: + /// + /// - **`untriaged → no-longer-reproducing`** is absent. `no-longer-reproducing` + /// invalidates a *judgement*, and an untriaged finding has none — there is nothing + /// for the disappearance to invalidate. Worse, the state requires a classification + /// (**D2**), so allowing the arrow would manufacture a violation out of a campaign + /// merely not re-observing something nobody had looked at. An untriaged finding that + /// stops reproducing simply stays untriaged, which is the truth: it is still "not + /// yet looked at". + /// - **`no-longer-reproducing → promoted`** is absent. Promotion authors a case + /// asserting a difference the implementations exhibit *now*; a finding whose + /// difference stopped reproducing has no current evidence, so promoting it would put + /// a case in the registry that the next run cannot reproduce. Reviving it first (a + /// campaign observing it again) is exactly the evidence promotion needs. + /// - **`untriaged → split`** is absent. A split exists because one *classification* + /// could not describe several witnesses; before anyone has classified anything there + /// is no judgement to separate. + /// + /// `promoted` and `split` are terminal. Re-classifying a finding *within* a state is + /// not a transition at all and does not consult this function — see + /// [`Finding::triage`]. + pub fn may_transition_to(self, next: FindingState) -> bool { + matches!( + (self, next), + (FindingState::Untriaged, FindingState::Triaged) + | (FindingState::Triaged, FindingState::Split) + | (FindingState::Triaged, FindingState::Promoted) + | (FindingState::Triaged, FindingState::NoLongerReproducing) + | (FindingState::NoLongerReproducing, FindingState::Triaged) + ) + } +} + +/// Why a state change or a split was refused (**T069**/**T070**). +/// +/// A rejected transition is an ordinary, expected outcome — a reviewer asking to promote +/// a `normalizer-defect`, or to split a finding with a single witness — so it is a typed +/// error rather than a panic, and every variant names the remedy rather than only the +/// rule. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum TransitionError { + /// The state machine declares no arrow from `from` to `to`. + #[error( + "finding `{finding}` cannot move from `{from}` to `{to}`: the lifecycle declares \ + no such transition. Remedy: reach `{to}` through the declared path, or leave the \ + finding where it is — a state reached out of order asserts an event that did not \ + happen." + )] + NotPermitted { + finding: String, + from: &'static str, + to: &'static str, + }, + + /// A state that requires a classification was requested for a finding without one. + #[error( + "finding `{finding}` has no classification, and state `{to}` requires exactly one \ + (FR-028). Remedy: `discovery triage {finding} --classification ` first." + )] + MissingClassification { finding: String, to: &'static str }, + + /// Promotion of a `normalizer-defect` / `fixture-defect` finding (FR-035). + #[error( + "finding `{finding}` is classified `{classification}`, which is not promotable \ + (FR-035): it describes a defect in the discovery or comparison machinery, not a \ + behavior of either implementation. Remedy: fix the normalizer or the generator — \ + promoting it would record a claim about deacon or the reference that the evidence \ + does not support." + )] + NonPromotable { + finding: String, + classification: &'static str, + }, + + /// A split was asked of a finding with fewer than two witnesses. + #[error( + "finding `{finding}` carries {witnesses} witness(es); a split separates witnesses \ + whose causes differ, so there is nothing to separate. Remedy: wait for a second \ + witness, or triage the finding as it stands." + )] + NothingToSplit { finding: String, witnesses: usize }, + + /// The named finding is not in the queue. + #[error("no finding `{finding}` in the queue")] + UnknownFinding { finding: String }, +} + +/// The two sides' concrete observed values for one witness. +/// +/// **Evidence, not identity.** Concrete values never enter the signature — including +/// them would split one defect across every generated value and make deduplication do +/// nothing (research D3) — so they live here, where a reviewer can read them. +/// +/// A `null` in either field means *the side produced nothing at this path*. An observed +/// JSON `null` renders identically, which is a deliberate and harmless conflation: the +/// presence distinction is carried by the signature's `valueShapeClass` +/// (`present-absent` versus `type-changed`), so nothing that matters is lost here. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ObservedValues { + /// What deacon produced at the signature's path. + #[serde(default)] + pub deacon: Option, + /// What the reference produced at the signature's path. + #[serde(default)] + pub reference: Option, +} + +/// One observation of a finding (data-model.md § 3). +/// +/// Witnesses are retained per finding (FR-032) so a merge can be reviewed and, if the +/// reviewer disagrees, reversed by a split. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Witness { + /// `wit-` over `campaignId ‖ candidateId`. + pub id: String, + /// The campaign that observed it — must resolve in `campaigns.json` (**D1**). + pub campaign_id: String, + /// The candidate input that produced it (`cnd-`). + pub candidate_id: String, + /// The reduced fixture. + pub minimal_input: Value, + /// `false` when the shrink budget was exhausted (FR-022) — a partially reduced input + /// is **never** silently presented as minimal. + pub is_minimal: bool, + /// The ordered catalogue step names applied during reduction. + #[serde(default)] + pub reduction_steps: Vec, + /// The concrete values each side produced. + #[serde(default)] + pub observed_values: ObservedValues, + /// The `mop-` operators that produced the candidate (FR-009 attribution). + #[serde(default)] + pub mutation_operators: Vec, +} + +impl Witness { + /// Derive the `wit-` id from `campaignId ‖ candidateId`. + pub fn derived_id(campaign_id: &str, candidate_id: &str) -> String { + format!("wit-{}", super::hash8(&[campaign_id, candidate_id])) + } +} + +/// One distinct signature and everything known about it (data-model.md § 3). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Finding { + /// `fnd-`, derived from `signature.id` — 1:1 with the signature, which is + /// what makes a duplicate finding unrepresentable (data-model.md § 1). + pub id: String, + /// The deduplication key. + pub signature: Signature, + /// At least one, declaration-ordered by first observation (**D1** otherwise). + #[serde(deserialize_with = "non_empty_witnesses")] + pub witnesses: Vec, + /// `null` while untriaged — the visible bucket (FR-029). + #[serde(default)] + pub classification: Option, + /// The lifecycle state. + pub state: FindingState, + /// The campaign that first admitted it — must resolve (**D1**). + pub first_observed: String, + /// The most recent campaign that reproduced it — must resolve (**D1**). + pub last_observed: String, + /// The registry case carrying it; set only in state `promoted` (**D3**). + #[serde(default)] + pub promoted_to: Option, + /// Provenance when a reviewer split a merged finding. The deduplication rule must + /// never re-merge a split lineage (FR-032), or a reviewer's judgement is silently + /// reverted by the next campaign and the split becomes unrepeatable work. + #[serde(default)] + pub split_from: Option, + /// Reviewer prose. Excluded from every hash — annotating a finding must never + /// re-key it. + #[serde(default)] + pub notes: String, +} + +impl Finding { + /// A newly admitted finding: untriaged, unclassified, one witness. + pub fn newly_admitted(signature: Signature, witness: Witness, campaign_id: &str) -> Finding { + Finding { + id: signature.finding_id(), + signature, + witnesses: vec![witness], + classification: None, + state: FindingState::Untriaged, + first_observed: campaign_id.to_string(), + last_observed: campaign_id.to_string(), + promoted_to: None, + split_from: None, + notes: String::new(), + } + } + + /// Whether this finding is in a state that requires a classification (`triaged` or + /// later, excluding `split`, whose classification belongs to its children — Q10). + pub fn requires_classification(&self) -> bool { + state_requires_classification(self.state) + } + + /// A split child's `fnd-`, over `parentId ‖ witnessId…` (**T070**). + /// + /// A child **cannot** use [`Signature::finding_id`], and the reason is structural + /// rather than stylistic: a split separates witnesses, not signatures, so every child + /// carries its parent's signature verbatim. Deriving from the signature would give the + /// parent and all its children one id — the duplicate the derivation exists to make + /// unrepresentable. + /// + /// Anchoring instead on `parent ‖ the witnesses this child claims` keeps the property + /// that matters: the id is a function of what the child *is*, so re-authoring the same + /// split reproduces the same ids (and therefore the same triage state), while moving a + /// witness between children correctly re-keys both. Witness ids are hashed in the order + /// given, and [`split_finding`] preserves the parent's witness order, so the result is + /// stable rather than dependent on iteration order. + pub fn derive_child_id(parent_id: &str, witness_ids: &[&str]) -> String { + let mut parts: Vec<&str> = Vec::with_capacity(witness_ids.len() + 1); + parts.push(parent_id); + parts.extend_from_slice(witness_ids); + format!("fnd-{}", super::hash8(&parts)) + } + + /// This finding's id as [`Finding::derive_child_id`] computes it from its own parent + /// and witnesses — `None` when it is not a split child. + pub fn derived_child_id(&self) -> Option { + let parent = self.split_from.as_deref()?; + let witnesses: Vec<&str> = self.witnesses.iter().map(|w| w.id.as_str()).collect(); + Some(Finding::derive_child_id(parent, &witnesses)) + } + + /// Record a reviewer's classification (**T069**), advancing `untriaged → triaged`. + /// + /// Re-classifying a finding that is already `triaged` or `no-longer-reproducing` is + /// **not** a transition and deliberately leaves the state alone: only a campaign that + /// actually reproduces a finding may move it out of `no-longer-reproducing` + /// (contracts/findings-queue.md, "Reproduction lifecycle"). Advancing it here would + /// assert an observation nothing made and would silently empty the FR-033 bucket the + /// report exists to show. + /// + /// Refused for `promoted` (terminal) and `split` (an inert ancestor that surrendered + /// classification to its children — Q10). + pub fn triage( + &mut self, + classification: Classification, + notes: Option<&str>, + ) -> Result { + if matches!(self.state, FindingState::Promoted | FindingState::Split) { + return Err(TransitionError::NotPermitted { + finding: self.id.clone(), + from: self.state.as_str(), + to: FindingState::Triaged.as_str(), + }); + } + if self.state == FindingState::Untriaged { + // Asserted rather than assumed: if the table ever stopped declaring this + // arrow, triage must fail loudly instead of quietly writing a state the + // lifecycle no longer recognizes. + if !self.state.may_transition_to(FindingState::Triaged) { + return Err(TransitionError::NotPermitted { + finding: self.id.clone(), + from: self.state.as_str(), + to: FindingState::Triaged.as_str(), + }); + } + self.state = FindingState::Triaged; + } + self.classification = Some(classification); + if let Some(notes) = notes { + self.notes = notes.to_string(); + } + Ok(self.state) + } + + /// Move a triaged finding to `promoted`, naming the registry case that now carries it. + /// + /// Refuses a non-promotable classification **by construction** (FR-035) rather than + /// leaving it to the checker: `check` runs over committed data, so a promotion that + /// only failed there would already have been written, and the record it wrote is the + /// one claiming coverage that cannot exist. + /// + /// Whether `case_id` resolves to a real case is **D3**'s question (US5, T080) — this + /// method cannot answer it without the registry, and inventing a partial check here + /// would make the real one look redundant. + pub fn promote(&mut self, case_id: &str) -> Result<(), TransitionError> { + let Some(classification) = self.classification else { + return Err(TransitionError::MissingClassification { + finding: self.id.clone(), + to: FindingState::Promoted.as_str(), + }); + }; + if !classification.is_promotable() { + return Err(TransitionError::NonPromotable { + finding: self.id.clone(), + classification: classification.as_str(), + }); + } + if !self.state.may_transition_to(FindingState::Promoted) { + return Err(TransitionError::NotPermitted { + finding: self.id.clone(), + from: self.state.as_str(), + to: FindingState::Promoted.as_str(), + }); + } + self.state = FindingState::Promoted; + self.promoted_to = Some(case_id.to_string()); + Ok(()) + } + + /// Record that this finding's difference stopped reproducing (FR-033). + /// + /// **A state, not a deletion.** The disappearance is information: it may mean a fix + /// landed, or it may mean the generator stopped reaching the input, and only the + /// retained record tells those apart. [`upsert_finding`] revives it to `triaged`, + /// keeping its classification, if a later campaign reproduces it. + pub fn mark_no_longer_reproducing(&mut self) -> Result<(), TransitionError> { + if self.classification.is_none() { + return Err(TransitionError::MissingClassification { + finding: self.id.clone(), + to: FindingState::NoLongerReproducing.as_str(), + }); + } + if !self + .state + .may_transition_to(FindingState::NoLongerReproducing) + { + return Err(TransitionError::NotPermitted { + finding: self.id.clone(), + from: self.state.as_str(), + to: FindingState::NoLongerReproducing.as_str(), + }); + } + self.state = FindingState::NoLongerReproducing; + Ok(()) + } +} + +/// Whether a state requires exactly one classification (Q4/Q10). +/// +/// `split` is deliberately excluded: a split ancestor surrendered its classification to +/// its children, because a split exists precisely because one classification could not +/// describe them all. +fn state_requires_classification(state: FindingState) -> bool { + matches!( + state, + FindingState::Triaged | FindingState::Promoted | FindingState::NoLongerReproducing + ) +} + +/// Reject an empty witness list at deserialize time (invariant Q2). +/// +/// Enforced here, at the shape that reads the file, rather than only in `check`: a +/// finding with no witness is a claim with no evidence, and it should not be +/// representable in memory at all. +fn non_empty_witnesses<'de, D>(de: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Vec::::deserialize(de)?; + if value.is_empty() { + return Err(serde::de::Error::custom( + "a finding must carry at least one witness — a finding with no witness is a \ + claim with no evidence behind it", + )); + } + Ok(value) +} + +// --------------------------------------------------------------------------- +// T017 — Campaign, PinnedInputSet, Budget, CampaignOutcome (data-model.md § 4) +// --------------------------------------------------------------------------- + +/// Which lane ran a campaign. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum CampaignLane { + /// A scheduled (nightly or weekly) run. + Scheduled, + /// An explicitly invoked run. + Invoked, +} + +impl CampaignLane { + /// The stable wire spelling. + pub fn as_str(self) -> &'static str { + match self { + CampaignLane::Scheduled => "scheduled", + CampaignLane::Invoked => "invoked", + } + } +} + +/// The four campaign tiers and, implicitly, their prerequisites (research D10). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum CampaignTier { + /// Deacon-only relation evaluation. Needs **no** oracle, Docker, or network. + Metamorphic, + /// deacon vs the pinned oracle over configuration resolution. The **nightly** + /// scheduled tier. + ConfigDifferential, + /// The same comparison with containers brought up. Invoked-only: at the 5-minute + /// per-candidate ceiling a handful of candidates would consume the whole scheduled + /// window, so it gets its own budget rather than starving the fast tier. + ContainerDifferential, + /// The pinned real-world workspace corpus. The **weekly** network-backed tier — an + /// ecological canary that runs only on request cannot warn anyone. + Corpus, +} + +impl CampaignTier { + /// The stable wire spelling. + pub fn as_str(self) -> &'static str { + match self { + CampaignTier::Metamorphic => "metamorphic", + CampaignTier::ConfigDifferential => "config-differential", + CampaignTier::ContainerDifferential => "container-differential", + CampaignTier::Corpus => "corpus", + } + } + + /// Every tier, in declaration order. + pub fn all() -> &'static [CampaignTier] { + &[ + CampaignTier::Metamorphic, + CampaignTier::ConfigDifferential, + CampaignTier::ContainerDifferential, + CampaignTier::Corpus, + ] + } + + /// Whether this tier requires the verified pinned oracle. + pub fn requires_oracle(self) -> bool { + !matches!(self, CampaignTier::Metamorphic) + } +} + +/// All **seven** elements of the pinned input set (FR-002, data-model.md § 4). +/// +/// Every element is mandatory. A finding is a claim about a specific pinned pair of +/// implementations, and an element left unrecorded is a dimension along which the claim +/// silently stops being checkable. +/// +/// The last three are not registry revisions but properties of this repository's own +/// code, and they are separate fields for a reason: +/// +/// - **`grammarVersion`** binds the campaign to the constraint inventory. A re-vendored +/// schema pin regenerates the inventory, which changes this string, which correctly +/// invalidates every finding bound to the old value with no separate bookkeeping. +/// - **`generatorVersion`** covers the two things that determine output but are neither +/// a grammar nor a mutation: the pseudorandom stream's algorithm identity (FR-001 +/// depends on it) and the reduction catalogue's *order* (FR-020 depends on it). +/// - **`mutationCatalogVersion`** names the mutation **operator set** and nothing else. +/// Folding either of the above into it would name them for something they are not, so +/// a deliberate change to reduction order would look like a change to the operators. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct PinnedInputSet { + /// The vendored schema revision pin (`conformance/schemas/`) — **D5**. + pub schema_pin: String, + /// The vendored spec-prose revision pin (`conformance/spec/`) — **D5**. + pub prose_pin: String, + /// The exact, verified oracle version (FR-003) — **D5**. + pub oracle_version: String, + /// `NORMALIZER_VERSION` at the time of the run. + pub normalizer_version: String, + /// The constraint inventory's `revision` (research D1). + pub grammar_version: String, + /// The mutation operator set's identity. + pub mutation_catalog_version: String, + /// The PRNG algorithm identity plus the reduction-catalogue order (research D2/D5). + pub generator_version: String, +} + +/// The default scheduled wall-clock budget, in seconds (research D10). +pub const DEFAULT_WALL_CLOCK_SECONDS: u64 = 1800; +/// The default per-candidate ceiling for the hermetic tiers, in seconds. +pub const DEFAULT_PER_CANDIDATE_SECONDS_HERMETIC: u64 = 60; +/// The default per-candidate ceiling for the container-backed tier, in seconds. +pub const DEFAULT_PER_CANDIDATE_SECONDS_CONTAINER: u64 = 300; +/// The default admission cap: newly distinct signatures admitted per campaign. +/// +/// Set from **reviewer throughput**, not machine capacity (research D10): a nightly run +/// that admits more than a couple of dozen genuinely new signatures has produced a +/// backlog nobody clears before the next run. The excess is *reported* +/// ([`CampaignOutcome::signatures_suppressed`]), never discarded silently — so a +/// campaign that keeps hitting the cap is itself a visible signal that something +/// systemic is diverging. +pub const DEFAULT_ADMISSION_CAP: u64 = 25; + +/// A campaign's declared budget (data-model.md § 4). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Budget { + /// Wall-clock ceiling for the whole campaign. + pub wall_clock_seconds: u64, + /// Ceiling for one candidate; a candidate that exceeds it is discarded **and + /// counted**, never left to hang the run. + pub per_candidate_seconds: u64, + /// Shrink steps allowed per finding before the best reduction is emitted with + /// `isMinimal: false`. + pub shrink_steps_per_finding: u64, + /// Newly distinct signatures admitted per campaign; the excess is reported. + pub admission_cap: u64, +} + +/// What a campaign did (data-model.md § 4). +/// +/// **Volume is reported even when nothing was found** (FR-062). A campaign that found +/// nothing and a campaign that never ran are completely different facts, and without +/// `candidatesGenerated` / `candidatesExecuted` they are indistinguishable from the +/// outside — which would make "no findings" the most comfortable possible way for the +/// machinery to be broken. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CampaignOutcome { + /// Candidates the generator produced. + pub candidates_generated: u64, + /// Candidates actually executed against an implementation. + pub candidates_executed: u64, + /// Candidates discarded as unsafe before execution (FR-011). + pub candidates_discarded_unsafe: u64, + /// Candidates that failed at document parsing — the SC-002 numerator. Above 10% of + /// `candidatesGenerated`, the campaign explored the parser rather than the tool. + pub parse_stage_failures: u64, + /// Whether the wall-clock budget ran out (FR-005). + pub budget_exhausted: bool, + /// The fraction of the declared space covered, reported when the budget was + /// exhausted (FR-005). + pub space_covered_fraction: f64, + /// Applications per mutation category. **All eleven keys are always present**, + /// including zeroes (FR-010): a category absent from the map is indistinguishable + /// from a category that was never applied, and FR-010 requires zero to be reported + /// as an explicit generation deficiency — which needs the key there. `IndexMap` + /// keeps the catalogue's declaration order (constitution VI). + pub mutation_applications: IndexMap, + /// Distinct signatures observed. + pub signatures_observed: u64, + /// Distinct signatures admitted to the queue. + pub signatures_admitted: u64, + /// Distinct signatures the admission cap suppressed (FR-034b) — **never silent**. + pub signatures_suppressed: u64, +} + +/// Provenance for one run (data-model.md § 4). Append-only: a campaign record is never +/// rewritten, because a finding names the campaign that observed it and a rewritten +/// campaign would retroactively change what that finding claims. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Campaign { + /// `cmp-`, derived by [`Campaign::derive_id`] — see that function for what the + /// id hashes and why the tier is part of it. + pub id: String, + /// The recorded seed, hex (FR-001). **Never defaulted** at the CLI: a defaulted seed + /// would let a campaign run without its reproducibility input being a conscious + /// choice. + pub seed: String, + /// Which lane ran it. + pub lane: CampaignLane, + /// Which tier it ran. + pub tier: CampaignTier, + /// The certification profile the run happened under — a `prof-` id declared in + /// `profiles.json` (**D1** when it does not resolve). + /// + /// A campaign is a claim about *this* environment: the same seed against the same + /// pinned pair on `linux/amd64` under Docker and on `linux/amd64` under Podman are two + /// different observations, and the profile is what the registry already uses to say + /// which. Recording it also completes the id's substance — see [`Campaign::derive_id`]. + pub profile: String, + /// The seven pinned inputs. + pub pinned_input_set: PinnedInputSet, + /// The declared budget. + pub budget: Budget, + /// What it did. + pub outcome: CampaignOutcome, +} + +impl Campaign { + /// `cmp-` over `seed ‖ canonical(pinnedInputSet) ‖ lane ‖ profile ‖ tier`. + /// + /// # The four documented components, and the fifth + /// + /// data-model.md § 1 lists **four**: `seed ‖ canonical(pinnedInputSet) ‖ lane ‖ + /// profile`. The `tier` is included here as a fifth, deliberately and visibly, because + /// the four-component form cannot distinguish two campaigns that genuinely exist: + /// `--seed 0x1 --tier metamorphic` and `--seed 0x1 --tier config-differential`, run in + /// the same lane under the same profile against the same pins, would share an id. Two + /// records with one id is a duplicate-id **D1** violation, so the append-only history + /// could not hold both — and the second run would either be rejected or silently + /// overwrite the first's provenance, taking every finding that names it along. + /// + /// Substance-anchoring exists so that a record which *is* the same thing keeps its id + /// and a record which is a different thing gets a different one. A tier is a different + /// thing: it decides which implementations are compared, over which observable + /// channels, under which prerequisites. Excluding it would make the id anchor to less + /// than the record's substance, which is the one failure the derivation is for. + /// + /// The `canonical(pinnedInputSet)` term is `serde_json`'s rendering of the struct, + /// whose field order is fixed by the declaration — so it is canonical by construction + /// rather than by a sorting pass that could be forgotten. + pub fn derive_id( + seed: &str, + pinned_input_set: &PinnedInputSet, + lane: CampaignLane, + profile: &str, + tier: CampaignTier, + ) -> String { + let pins = serde_json::to_string(pinned_input_set) + .unwrap_or_else(|e| unreachable!("a pinned input set always serializes: {e}")); + format!( + "cmp-{}", + super::hash8(&[seed, &pins, lane.as_str(), profile, tier.as_str()]) + ) + } + + /// Recompute this campaign's id from its own fields. + /// + /// The identity check **D1** relies on: without it a mis-assigned campaign id is + /// undetectable, and an undetectable mis-assignment is worse than a missing id — every + /// finding that names the campaign inherits provenance the record does not actually + /// carry. + pub fn derived_id(&self) -> String { + Campaign::derive_id( + &self.seed, + &self.pinned_input_set, + self.lane, + &self.profile, + self.tier, + ) + } +} + +// --------------------------------------------------------------------------- +// T018 — the strict-JSON loader +// --------------------------------------------------------------------------- + +/// The `findings.json` envelope. +/// +/// `records` is **mandatory**, not `#[serde(default)]`. The writer always emits it, so +/// defaulting buys nothing and costs exactly the failure mode FR-029 exists to prevent: a +/// truncated file would load as an empty queue and validate clean, and "the data was +/// lost" would read as "nothing was found". +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct FindingsFile { + /// Schema version of the file format — rejected at load unless it is + /// [`SCHEMA_VERSION`], so a future v2 file can never be read under v1 semantics and + /// silently rewritten as v1. + #[serde(deserialize_with = "supported_schema_version")] + pub schema_version: u32, + /// The findings, in admission order. + pub records: Vec, +} + +impl Default for FindingsFile { + fn default() -> Self { + FindingsFile { + schema_version: SCHEMA_VERSION, + records: Vec::new(), + } + } +} + +/// The `campaigns.json` envelope. `records` is mandatory for the same reason as +/// [`FindingsFile::records`]. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CampaignsFile { + /// Schema version of the file format — see [`FindingsFile::schema_version`]. + #[serde(deserialize_with = "supported_schema_version")] + pub schema_version: u32, + /// The campaigns, append-only in run order. + pub records: Vec, +} + +impl Default for CampaignsFile { + fn default() -> Self { + CampaignsFile { + schema_version: SCHEMA_VERSION, + records: Vec::new(), + } + } +} + +/// The current discovery data-root schema version. +pub const SCHEMA_VERSION: u32 = 1; + +/// Reject a `schemaVersion` this build does not understand. +/// +/// Loading a future version under today's semantics would be bad enough on its own; the +/// real hazard is the *write* that follows, because every writer stamps +/// [`SCHEMA_VERSION`]. A `triage` against a v2 file would therefore silently downgrade +/// it, discarding whatever v2 added. Refusing to read is the only way to guarantee not +/// destroying it. +fn supported_schema_version<'de, D>(de: D) -> Result +where + D: Deserializer<'de>, +{ + let value = u32::deserialize(de)?; + if value != SCHEMA_VERSION { + return Err(serde::de::Error::custom(format!( + "unsupported schemaVersion {value}: this build reads and writes version \ + {SCHEMA_VERSION}, and writing would stamp {SCHEMA_VERSION} over it" + ))); + } + Ok(value) +} + +/// The loaded discovery data root — all **three** collection files. +/// +/// `corpus.json` is loaded here alongside the queue and the campaign history (US7, +/// T105/T106) precisely so the **D4** immutable-reference class runs wherever [`check`] +/// runs: hermetically, in the fast lane, on every pull request. A manifest validated only +/// by the network-backed lane would be validated on the one run per week that can least +/// afford to discover a mutable pin. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct DiscoveryData { + /// The findings queue, in file order. + pub findings: Vec, + /// The campaign history, in file order. + pub campaigns: Vec, + /// The pinned real-world corpus manifest, in file order. + pub corpus: Vec, +} + +/// Paths of the three discovery data files under `dir`. +pub fn findings_path(dir: &Path) -> PathBuf { + dir.join("findings.json") +} + +/// Path of the campaign history under `dir`. +pub fn campaigns_path(dir: &Path) -> PathBuf { + dir.join("campaigns.json") +} + +/// Path of the corpus manifest under `dir` (loaded by US7). +pub fn corpus_path(dir: &Path) -> PathBuf { + dir.join("corpus.json") +} + +impl DiscoveryData { + /// Load the discovery data root at `dir`, collecting **all** file errors in one pass + /// (the same discipline as [`crate::load::Registry::load`] — a validator that stops + /// at the first bad file makes fixing a batch an iterative guessing game). + /// + /// A missing file is an EMPTY collection, not an error: a fixture root, or a + /// repository before the first campaign, legitimately has none. A file that is + /// **present but malformed** is a located [`LoadError::Schema`] — never silently + /// empty (constitution IV). + pub fn load(dir: &Path) -> Result { + let mut errors: Vec = Vec::new(); + + let findings_file = findings_path(dir); + let campaigns_file = campaigns_path(dir); + let corpus_file = corpus_path(dir); + let findings = load_file::(&findings_file, &mut errors) + .map(|f| f.records) + .unwrap_or_default(); + let campaigns = load_file::(&campaigns_file, &mut errors) + .map(|f| f.records) + .unwrap_or_default(); + let corpus = load_file::(&corpus_file, &mut errors) + .map(|f| f.records) + .unwrap_or_default(); + + // Duplicate ids are rejected HERE, not only by `check`. Every by-id lookup + // ([`DiscoveryData::finding_mut`], the CLI's triage/split/scaffold) takes the + // FIRST match, so a duplicate would let `triage` mutate one record while the + // other survives untouched — a write that appears to succeed and silently does + // not. A checker that catches it after the fact is too late. + errors.extend(duplicate_id_errors( + &findings_file, + "finding", + findings.iter().map(|f| f.id.as_str()), + )); + errors.extend(duplicate_id_errors( + &campaigns_file, + "campaign", + campaigns.iter().map(|c| c.id.as_str()), + )); + errors.extend(duplicate_id_errors( + &corpus_file, + "corpus entry", + corpus.iter().map(|e| e.id.as_str()), + )); + + if !errors.is_empty() { + return Err(LoadError::Schema(errors)); + } + Ok(DiscoveryData { + findings, + campaigns, + corpus, + }) + } + + /// Load the workspace's default discovery data root. + pub fn load_default() -> Result { + DiscoveryData::load(&crate::default_discovery_dir()) + } + + /// Find a finding by id. + pub fn finding(&self, id: &str) -> Option<&Finding> { + self.findings.iter().find(|f| f.id == id) + } + + /// Find a finding by id, mutably. + pub fn finding_mut(&mut self, id: &str) -> Option<&mut Finding> { + self.findings.iter_mut().find(|f| f.id == id) + } + + /// Find a campaign by id. + pub fn campaign(&self, id: &str) -> Option<&Campaign> { + self.campaigns.iter().find(|c| c.id == id) + } + + /// Find a corpus entry by id. + pub fn corpus_entry(&self, id: &str) -> Option<&super::corpus::CorpusEntry> { + self.corpus.iter().find(|e| e.id == id) + } +} + +/// Collect duplicate-id records as located schema errors. +fn duplicate_id_errors<'a>( + path: &Path, + kind: &str, + ids: impl Iterator, +) -> Vec { + let mut seen: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new(); + let mut out = Vec::new(); + for id in ids { + if !seen.insert(id) { + out.push(SchemaError { + file: path.to_path_buf(), + location: Some(id.to_string()), + message: format!( + "duplicate {kind} id `{id}` — every by-id lookup takes the first \ + match, so a duplicate makes a write appear to succeed while the \ + other record survives untouched" + ), + }); + } + } + out +} + +/// Read and strictly deserialize one collection file. +/// +/// Only a genuine `NotFound` yields `None`. `Path::exists` was the wrong test: it returns +/// `false` for **any** error, so a present-but-unreadable file (a permission problem, a +/// broken symlink) would read as "this repository has not run a campaign yet" instead of +/// as the located error this function promises (constitution IV — no silent fallback). +fn load_file( + path: &Path, + errors: &mut Vec, +) -> Option { + match std::fs::metadata(path) { + Ok(_) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return None, + Err(e) => { + errors.push(SchemaError { + file: path.to_path_buf(), + location: None, + message: format!("could not stat file: {e}"), + }); + return None; + } + } + match read_file(path).and_then(|raw| deserialize_located::(path, &raw)) { + Ok(value) => Some(value), + Err(e) => { + errors.push(e); + None + } + } +} + +// --------------------------------------------------------------------------- +// T019 — the atomic writer +// --------------------------------------------------------------------------- + +/// Render a findings file in its canonical, byte-stable form (2-space pretty JSON, +/// trailing newline) — the same rendering every machine-owned artifact in this crate +/// uses, so a queue update produces a reviewable git diff rather than a reformat. +pub fn render_findings(file: &FindingsFile) -> String { + render(file) +} + +/// Render a campaigns file in its canonical, byte-stable form. +pub fn render_campaigns(file: &CampaignsFile) -> String { + render(file) +} + +fn render(value: &T) -> String { + let mut out = serde_json::to_string_pretty(value) + .unwrap_or_else(|e| unreachable!("discovery record serialization is infallible: {e}")); + out.push('\n'); + out +} + +/// Atomically write the findings queue to `dir/findings.json`. +/// +/// Delegates to the single [`crate::atomic_write`] primitive (unique temp file + +/// `fs::rename`), so a shorter payload over a longer one can never leave trailing +/// bytes — the flaky-JSON failure mode a plain `fs::write` has. +pub fn write_findings(dir: &Path, findings: &[Finding]) -> std::io::Result<()> { + let file = FindingsFile { + schema_version: SCHEMA_VERSION, + records: findings.to_vec(), + }; + crate::atomic_write(&findings_path(dir), &render_findings(&file)) +} + +/// Atomically write the campaign history to `dir/campaigns.json`. +/// +/// Refuses to write a campaign whose `spaceCoveredFraction` is not finite. `serde_json` +/// renders NaN and ±Infinity as **bare `null`** and does **not** error, and `null` is not +/// a valid `f64` on the way back in — so a single division by a zero +/// `candidatesGenerated` (an aborted campaign is exactly that shape) would produce a +/// `campaigns.json` that writes cleanly and can never be loaded again, taking the whole +/// queue's provenance with it. Failing the write is recoverable; writing the file is not. +pub fn write_campaigns(dir: &Path, campaigns: &[Campaign]) -> std::io::Result<()> { + if let Some(bad) = campaigns + .iter() + .find(|c| !c.outcome.space_covered_fraction.is_finite()) + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "campaign `{}` has a non-finite spaceCoveredFraction ({}); refusing to \ + write. serde_json renders it as bare `null`, which never loads back — \ + the file would be permanently unreadable. Report 0.0 for an unmeasurable \ + fraction, or fix the divisor.", + bad.id, bad.outcome.space_covered_fraction + ), + )); + } + let file = CampaignsFile { + schema_version: SCHEMA_VERSION, + records: campaigns.to_vec(), + }; + crate::atomic_write(&campaigns_path(dir), &render_campaigns(&file)) +} + +// --------------------------------------------------------------------------- +// T020 — signature-keyed upsert +// --------------------------------------------------------------------------- + +/// What an [`upsert_finding`] call did — reported so a campaign can distinguish "we +/// found something new" from "we re-observed something known", which is the difference +/// between a queue that reflects distinct problems and one that reflects campaign volume. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Upsert { + /// A new finding was inserted. + Inserted, + /// A witness was appended to an existing finding. + WitnessAppended, + /// The finding already carried this exact witness (the same campaign re-observing + /// the same candidate); nothing changed. + AlreadyWitnessed, + /// The signature belongs to a **split lineage**, so nothing was merged and nothing was + /// created (FR-032). + /// + /// Reported distinctly from [`Upsert::AlreadyWitnessed`] because the two say different + /// things to a reader of the campaign log: "we have seen this exact observation before" + /// versus "a reviewer decided this signature covers several causes, and attributing a + /// new observation to one of them is a judgement only a reviewer can make". + SplitLineage, +} + +/// Insert a new finding for `signature`, or append `witness` to the existing one +/// (FR-030). +/// +/// The merge key is the **signature**, and the finding id is derived from it, so a +/// second record for the same signature is unrepresentable rather than merely +/// discouraged. +/// +/// Two non-merge rules are honored here because both are load-bearing: +/// +/// - **A split lineage is never re-merged** (FR-032). A finding in state +/// [`FindingState::Split`] is an inert ancestor; a new observation of that signature +/// must not append to it, or the next campaign silently reverts the reviewer's +/// judgement and the split becomes unrepeatable work. Such an observation is reported +/// as [`Upsert::AlreadyWitnessed`] against the ancestor — it is genuinely not new, and +/// attributing it to one of the children is a judgement only a reviewer can make. +/// - **Distinct signatures stay distinct** even when they map to the same behavior +/// (FR-031). Nothing here consults behaviors, which is what guarantees it. +/// +/// Re-observing a finding also refreshes [`Finding::last_observed`] and revives a +/// [`FindingState::NoLongerReproducing`] record to `triaged` **keeping its +/// classification** — re-triaging a finding a reviewer already judged would be wasted +/// work (contracts/findings-queue.md, "Reproduction lifecycle"). +pub fn upsert_finding( + findings: &mut Vec, + signature: Signature, + witness: Witness, + campaign_id: &str, +) -> Upsert { + let id = signature.finding_id(); + + let Some(existing) = findings.iter_mut().find(|f| f.id == id) else { + // A split lineage is never re-merged (FR-032) — including when the ANCESTOR is + // gone and only children remain. Without this clause the lookup above misses (a + // child's id is derived from its parent and witnesses, never from the signature), + // a fresh merged record is created under the ancestor's id, and the reviewer's + // decision that these witnesses have different causes is silently undone by the + // next campaign. Checking the children's own signature is what makes the rule hold + // on the lineage rather than on one record's continued existence. + if findings + .iter() + .any(|f| f.split_from.is_some() && f.signature.id == signature.id) + { + return Upsert::SplitLineage; + } + findings.push(Finding::newly_admitted(signature, witness, campaign_id)); + return Upsert::Inserted; + }; + + if existing.state == FindingState::Split { + // An inert ancestor accepts no new witnesses (invariant Q10). Attributing the + // observation to one of its children is a judgement only a reviewer can make. + return Upsert::SplitLineage; + } + if existing.witnesses.iter().any(|w| w.id == witness.id) { + return Upsert::AlreadyWitnessed; + } + + existing.witnesses.push(witness); + existing.last_observed = campaign_id.to_string(); + if existing.state == FindingState::NoLongerReproducing { + existing.state = FindingState::Triaged; + } + Upsert::WitnessAppended +} + +// --------------------------------------------------------------------------- +// T070 — split with `splitFrom` lineage +// --------------------------------------------------------------------------- + +/// What a [`split_finding`] call produced: the ancestor and the children it now has. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Split { + /// The finding that became an inert ancestor. + pub parent: String, + /// The children, in the parent's own witness order. Always ≥ 2 (invariant Q10). + pub children: Vec, +} + +/// Split a signature-merged finding whose witnesses turn out to have different causes +/// (FR-032, invariant Q10). +/// +/// # What the split does +/// +/// The parent becomes an **inert ancestor**: it keeps its witnesses as historical record, +/// accepts no new ones ([`upsert_finding`] enforces that), and surrenders its +/// classification, because a split exists precisely because one classification could not +/// describe them all. A parent that kept a classification would assert the judgement the +/// split rejected. +/// +/// # Why one child per witness +/// +/// `discovery split ` takes no grouping argument (contracts/discovery-cli.md), so +/// the only partition derivable from the reviewer's actual statement — "these witnesses +/// have different causes" — is the finest one. Guessing a coarser grouping would re-assert +/// a merge the reviewer just rejected, and it would have to guess *which* witnesses share +/// a cause, which is the very judgement the split was invoked to record. A reviewer who +/// concludes two children share a cause triages them identically; nothing is lost, whereas +/// a wrong merge is not recoverable without re-splitting. +/// +/// Each child carries its parent's signature verbatim (a split separates witnesses, not +/// signatures), takes its own [`Finding::derive_child_id`], starts `untriaged` with no +/// classification, and names the parent in `splitFrom`. +/// +/// # Errors +/// +/// - [`TransitionError::UnknownFinding`] when nothing carries that id. +/// - [`TransitionError::NotPermitted`] when the finding is already `split` or `promoted`, +/// or is not in a state the lifecycle lets a split leave (see +/// [`FindingState::may_transition_to`]). +/// - [`TransitionError::NothingToSplit`] with fewer than two witnesses. +/// +/// On any error the queue is left **untouched**: every check runs before the first +/// mutation, so a refused split cannot leave a parent marked inert with no children — a +/// shape that is both a **D2** and a Q10 violation and that nothing would then repair. +pub fn split_finding( + findings: &mut Vec, + parent_id: &str, +) -> Result { + let Some(index) = findings.iter().position(|f| f.id == parent_id) else { + return Err(TransitionError::UnknownFinding { + finding: parent_id.to_string(), + }); + }; + let parent = &findings[index]; + + // Witness arity first, deliberately. It is the more fundamental objection — a finding + // with one witness has nothing to separate whatever state it is in — so reporting it + // first tells a reviewer the thing they can act on rather than sending them to triage a + // finding that still could not be split afterwards. + if parent.witnesses.len() < 2 { + return Err(TransitionError::NothingToSplit { + finding: parent.id.clone(), + witnesses: parent.witnesses.len(), + }); + } + if !parent.state.may_transition_to(FindingState::Split) { + return Err(TransitionError::NotPermitted { + finding: parent.id.clone(), + from: parent.state.as_str(), + to: FindingState::Split.as_str(), + }); + } + + let signature = parent.signature.clone(); + let children: Vec = parent + .witnesses + .iter() + .map(|witness| Finding { + id: Finding::derive_child_id(parent_id, &[witness.id.as_str()]), + signature: signature.clone(), + witnesses: vec![witness.clone()], + classification: None, + state: FindingState::Untriaged, + // The child's provenance is its own witness's campaign, not the parent's: + // `firstObserved` / `lastObserved` assert an observation of THIS finding, and + // the parent's may name a campaign that witnessed a sibling instead. + first_observed: witness.campaign_id.clone(), + last_observed: witness.campaign_id.clone(), + promoted_to: None, + split_from: Some(parent_id.to_string()), + notes: String::new(), + }) + .collect(); + + let parent = &mut findings[index]; + parent.state = FindingState::Split; + parent.classification = None; + + let split = Split { + parent: parent_id.to_string(), + children: children.iter().map(|c| c.id.clone()).collect(), + }; + findings.extend(children); + Ok(split) +} + +// --------------------------------------------------------------------------- +// T021 — violation classes D1 and D5 +// --------------------------------------------------------------------------- + +/// Everything `discovery check` needs from the **registry** to resolve the queue's +/// outward references. +/// +/// A borrowed view rather than a whole [`crate::load::Registry`] so the check is +/// explicit about the only four things it reads — the declared channel set, the recorded +/// revisions, the declared profiles, and the declared case ids — and so a test can supply +/// them without building a registry. +/// +/// Note the direction: the discovery check reads the registry, never the reverse. That +/// asymmetry is what makes the queue unreachable from `certify`. +#[derive(Debug, Clone, Default)] +pub struct RegistryView<'a> { + /// Declared observable channel ids (`channels.json`). + pub channels: Vec<&'a str>, + /// Recorded source revisions (`revisions.json`). + pub revisions: Vec<&'a SourceRevision>, + /// Declared certification profile ids (`profiles.json`) — what a campaign's + /// [`Campaign::profile`] must resolve to. + pub profiles: Vec<&'a str>, + /// Declared case ids (`cases/.json`) — what a promoted finding's + /// [`Finding::promoted_to`] must resolve to (**D3**). + pub cases: Vec<&'a str>, +} + +impl<'a> RegistryView<'a> { + /// Build the view from a loaded registry. + pub fn from_registry(registry: &'a crate::load::Registry) -> RegistryView<'a> { + RegistryView { + channels: registry.channels.iter().map(|c| c.id.as_str()).collect(), + revisions: registry.revisions.iter().collect(), + profiles: registry.profiles.iter().map(|p| p.id.as_str()).collect(), + cases: registry.cases.iter().map(|c| c.id.as_str()).collect(), + } + } + + fn declares_channel(&self, channel: &str) -> bool { + self.channels.contains(&channel) + } + + fn declares_profile(&self, profile: &str) -> bool { + self.profiles.contains(&profile) + } + + fn declares_case(&self, case: &str) -> bool { + self.cases.contains(&case) + } + + /// Whether a revision of `kind` with this `pin` is recorded. + /// + /// Matched on the **pin**, not the id, because a `pinnedInputSet` element records + /// the pin (`113500f4`, `0.87.0`) rather than the `rev-` id. Accepting the id too + /// would let two spellings of the same claim diverge. + fn records_pin(&self, kind: RevisionKind, pin: &str) -> bool { + self.revisions + .iter() + .any(|r| r.kind == kind && r.pin == pin) + } +} + +/// Run the discovery-side violation classes over a loaded data root. +/// +/// Reports **all** violations in one pass, matching `validate` — a checker that stops at +/// the first problem makes fixing a batch an iterative guessing game. +/// +/// Emits **D1** – **D5**: +/// +/// | Class | Checked here | +/// |---|---| +/// | **D1** | derived-id mismatch (signature, finding, **split child**, witness, **campaign**); duplicate id; empty `witnesses`; a signature naming an undeclared channel; a `firstObserved`/`lastObserved` that does not resolve **or is not backed by a witness**; a witness naming an unresolvable campaign; a `splitFrom` that is self-referential or unresolvable; a `split` ancestor with fewer than two children; a campaign `profile` absent from `profiles.json`; a non-finite `spaceCoveredFraction`; an empty seed; an empty repo-owned pinned-input element | +/// | **D2** | a `triaged`/`promoted`/`no-longer-reproducing` finding with no classification; an `untriaged` or `split` finding carrying one; a `promoted` finding classified `normalizer-defect` / `fixture-defect` | +/// | **D3** | a `promoted` finding with no `promotedTo`, or one naming a case absent from the registry; a `promotedTo` carried in any state **other** than `promoted` | +/// | **D4** | a corpus entry whose `commit` is not a 40-hex object name; a malformed `contentDigest`; a corpus id that does not derive from its own substance; a duplicate corpus id or name | +/// | **D5** | a `schemaPin` / `prosePin` / `oracleVersion` naming a revision absent from `revisions.json` | +/// +/// **D4**'s second clause — a digest recorded and then removed — is a property of a +/// *change* rather than of a file, so it lives in +/// [`corpus::check_drift`](super::corpus::check_drift), which takes a baseline explicitly. +/// +/// D2 is checked here *and* refused by [`Finding::promote`], deliberately: `check` runs +/// over committed data, so a promotion that only failed here would already have been +/// written, and the record it wrote is the one claiming coverage that cannot exist. +/// +/// Empty `witnesses` is listed under D1 above and is *also* rejected at deserialize time +/// ([`non_empty_witnesses`]). That is deliberate belt-and-braces: the loader rejection +/// makes the state unrepresentable in memory, and the D1 clause makes a +/// programmatically-built queue (a campaign in flight) fail the same way rather than +/// only failing on the next load. +pub fn check(data: &DiscoveryData, registry: &RegistryView<'_>) -> Vec { + let mut violations = Vec::new(); + check_findings(data, registry, &mut violations); + check_classifications(data, &mut violations); + check_promotions(data, registry, &mut violations); + check_campaigns(data, registry, &mut violations); + violations.extend(super::corpus::check(&data.corpus)); + violations +} + +/// D1 over the findings queue. +fn check_findings( + data: &DiscoveryData, + registry: &RegistryView<'_>, + out: &mut Vec, +) { + let mut seen: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new(); + + for finding in &data.findings { + // Identity: the id must be derivable from the signature, and the signature's id + // from its own fields. A hand-edited record that broke either would silently + // stop deduplicating — two records for one signature, which the derived id + // exists to make impossible. + let derived_signature_id = finding.signature.derived_id(); + if finding.signature.id != derived_signature_id { + out.push(DiscoveryError::MalformedRecord { + record: finding.id.clone(), + cause: format!( + "signature id `{}` does not match its substance (expected `{derived_signature_id}`)", + finding.signature.id + ), + }); + } + // Two id rules, one per lineage position (**T070**). + // + // A finding the DEDUPLICATION owns takes its id from its signature, which is what + // makes two findings with one signature unrepresentable. A split CHILD cannot: it + // carries its parent's signature verbatim (a split separates witnesses, not + // signatures), so signature-derivation would give the parent and every child one + // id — exactly the duplicate the derivation exists to prevent. A child therefore + // anchors on `parent ‖ its own witnesses`, which keeps the same property (the id is + // a function of what the record is) without the collision. + // + // Both are checked. An unchecked derivation is a comment: a hand-edited child id + // would silently detach the record from its lineage, and re-authoring the same + // split would then produce a second copy under the correct id. + match finding.derived_child_id() { + None => { + let derived_finding_id = finding.signature.finding_id(); + if finding.id != derived_finding_id { + out.push(DiscoveryError::MalformedRecord { + record: finding.id.clone(), + cause: format!( + "finding id does not match its signature (expected \ + `{derived_finding_id}`) — the id is derived precisely so two \ + findings cannot share a signature" + ), + }); + } + } + Some(derived_child_id) => { + if finding.id != derived_child_id { + out.push(DiscoveryError::MalformedRecord { + record: finding.id.clone(), + cause: format!( + "split-child id does not match `splitFrom ‖ its witness ids` \ + (expected `{derived_child_id}`) — a child that is not keyed to \ + its own lineage detaches from it, and re-authoring the same \ + split then produces a second copy" + ), + }); + } + } + } + if !seen.insert(finding.id.as_str()) { + out.push(DiscoveryError::MalformedRecord { + record: finding.id.clone(), + cause: "duplicate finding id".to_string(), + }); + } + + // Q2: at least one witness. + if finding.witnesses.is_empty() { + out.push(DiscoveryError::MalformedRecord { + record: finding.id.clone(), + cause: "no witnesses — a finding with no witness is a claim with no evidence" + .to_string(), + }); + } + + // Q3: the signature's channel is declared. + if !registry.declares_channel(&finding.signature.channel) { + out.push(DiscoveryError::UnknownChannel { + record: finding.id.clone(), + channel: finding.signature.channel.clone(), + }); + } + + // Q8: firstObserved / lastObserved resolve — and are actually WITNESSED. + // + // Existence alone is too weak. `firstObserved` is "the campaign that first + // admitted it" and `lastObserved` is "the most recent campaign that reproduced + // it": both descriptions entail a witness from that campaign. A record naming a + // real but unwitnessed campaign is precisely the "claims provenance it does not + // have" the error text describes, and it is the shape a careless hand edit + // produces (bump `lastObserved`, forget the witness). + for (field, campaign_id) in [ + ("firstObserved", &finding.first_observed), + ("lastObserved", &finding.last_observed), + ] { + if data.campaign(campaign_id).is_none() { + out.push(DiscoveryError::UnresolvableReference { + record: finding.id.clone(), + kind: format!("{field} campaign"), + reference: campaign_id.clone(), + }); + } else if !finding + .witnesses + .iter() + .any(|w| &w.campaign_id == campaign_id) + { + out.push(DiscoveryError::MalformedRecord { + record: finding.id.clone(), + cause: format!( + "`{field}` names campaign `{campaign_id}`, which witnessed nothing \ + for this finding — the field asserts an observation, so it must be \ + backed by a witness" + ), + }); + } + } + + // Each witness's campaign resolves, and its id matches its substance. + for witness in &finding.witnesses { + if data.campaign(&witness.campaign_id).is_none() { + out.push(DiscoveryError::UnresolvableReference { + record: format!("{}/{}", finding.id, witness.id), + kind: "witness campaign".to_string(), + reference: witness.campaign_id.clone(), + }); + } + let derived = Witness::derived_id(&witness.campaign_id, &witness.candidate_id); + if witness.id != derived { + out.push(DiscoveryError::MalformedRecord { + record: format!("{}/{}", finding.id, witness.id), + cause: format!( + "witness id does not match `campaignId ‖ candidateId` (expected `{derived}`)" + ), + }); + } + } + + // splitFrom, when set, must name a real OTHER finding. A self-reference resolves + // trivially and would otherwise pass, leaving a finding that is its own ancestor + // — a lineage the deduplication rule would then refuse to merge into itself. + if let Some(parent) = &finding.split_from { + if parent == &finding.id { + out.push(DiscoveryError::MalformedRecord { + record: finding.id.clone(), + cause: "`splitFrom` names the finding itself — a split lineage has a \ + parent and children, never one record that is both" + .to_string(), + }); + } else if data.finding(parent).is_none() { + out.push(DiscoveryError::UnresolvableReference { + record: finding.id.clone(), + kind: "splitFrom finding".to_string(), + reference: parent.clone(), + }); + } + } + + // Q10: a split ancestor has at least two children naming it. One child is not a + // split — it is the same finding with a new id — and zero children is the shape a + // half-finished split leaves behind: a parent that has surrendered its + // classification and accepts no new witnesses, with nothing carrying its evidence + // forward. [`split_finding`] cannot produce either, so reaching them means a hand + // edit, which is exactly what a checker is for. + if finding.state == FindingState::Split { + let children = data + .findings + .iter() + .filter(|f| f.split_from.as_deref() == Some(finding.id.as_str())) + .count(); + if children < 2 { + out.push(DiscoveryError::MalformedRecord { + record: finding.id.clone(), + cause: format!( + "state `split` with {children} child(ren); a split ancestor needs \ + at least two findings naming it in `splitFrom`, because a split \ + exists precisely to separate witnesses one classification could \ + not describe" + ), + }); + } + } + } +} + +/// **D2** — classification arity and promotability (**T068**). +/// +/// Q4 and Q6 of contracts/findings-queue.md, plus the half of Q10 that is about +/// classification rather than lineage. All four shapes are one class because they are one +/// defect: the queue asserting a judgement nobody made, or holding one that cannot lead +/// anywhere. +/// +/// **"More than one classification" is unrepresentable**, not unchecked: [`Finding`] +/// carries `Option`, and a JSON array where a string belongs is rejected by +/// the strict loader before this ever runs. The arity rule therefore reduces here to its +/// only reachable violation — *zero* where exactly one is required. That is the stronger +/// outcome, not a gap: the invariant holds by construction on the side a checker would +/// otherwise have to police. +fn check_classifications(data: &DiscoveryData, out: &mut Vec) { + for finding in &data.findings { + match (finding.state, finding.classification) { + // Q4 — exactly one, once triaged or later. + (state, None) if state_requires_classification(state) => { + out.push(DiscoveryError::ClassificationArity { + record: finding.id.clone(), + cause: format!( + "state `{}` requires exactly one classification and the record \ + carries none", + state.as_str() + ), + }); + } + // Q4 — `null` only while untriaged. + (FindingState::Untriaged, Some(classification)) => { + out.push(DiscoveryError::ClassificationArity { + record: finding.id.clone(), + cause: format!( + "state `untriaged` carries classification `{}` — untriaged means \ + nobody has looked, so a classification here makes the FR-029 \ + bucket count something that has in fact been judged", + classification.as_str() + ), + }); + } + // Q10 — a split ancestor surrenders its classification to its children. + (FindingState::Split, Some(classification)) => { + out.push(DiscoveryError::ClassificationArity { + record: finding.id.clone(), + cause: format!( + "state `split` carries classification `{}` — a split exists \ + precisely because one classification could not describe all its \ + witnesses, so a parent that keeps one asserts the judgement the \ + split rejected", + classification.as_str() + ), + }); + } + _ => {} + } + // Q6 — the two machinery classifications can never reach `promoted` (FR-035). + // Checked independently of the arity arms above so a promoted record with a + // non-promotable classification is reported for what it is rather than passing + // because it satisfied the arity rule. + if finding.state == FindingState::Promoted + && let Some(classification) = finding.classification + && !classification.is_promotable() + { + out.push(DiscoveryError::ClassificationArity { + record: finding.id.clone(), + cause: format!( + "state `promoted` with classification `{}`, which is not promotable \ + (FR-035): it describes a defect in the discovery or comparison \ + machinery, not a behavior of either implementation, so no registry \ + case can legitimately carry it", + classification.as_str() + ), + }); + } + } +} + +/// **D3** — promotion resolution (**T080**, FR-042, data-model.md § 3). +/// +/// Q7 of contracts/findings-queue.md: a `promoted` finding names the registry case that +/// now carries it, and that case exists. Three shapes, one class, because all three are +/// the same defect — **the queue claiming coverage that does not exist**: +/// +/// - **`promoted` with no `promotedTo`.** The state says a case carries this finding and +/// the record declines to say which, so the claim cannot be checked by anyone. FR-042 +/// exists precisely so a promoted finding is not rediscovered and re-triaged, and a +/// promotion naming nothing gives a reviewer no way to reach the case that would tell +/// them the work is done. +/// - **`promotedTo` naming a case the registry does not declare.** The most dangerous +/// shape, and the reason this class needs the registry at all: it reads as covered +/// from inside the queue while nothing anywhere executes the finding. A case deleted +/// or renamed after the promotion produces exactly this, which is why it is checked +/// against the loaded registry on every run rather than only at the moment of +/// promotion. +/// - **`promotedTo` in any other state.** data-model.md § 3 declares the field set *only* +/// in state `promoted`. A `triaged` record carrying one asserts a promotion that never +/// happened; a `no-longer-reproducing` one asserts a case still carries a difference +/// the queue has recorded as gone. Both mislead in the direction of claiming more +/// coverage than exists, which is the direction that matters. +/// +/// The check is deliberately **one-way**: it reads the registry's declared case ids and +/// writes nothing. Discovery never authors a case (FR-036), so the only thing D3 can do +/// about an unresolvable promotion is name it. +fn check_promotions( + data: &DiscoveryData, + registry: &RegistryView<'_>, + out: &mut Vec, +) { + for finding in &data.findings { + match (finding.state, finding.promoted_to.as_deref()) { + (FindingState::Promoted, None) => { + out.push(DiscoveryError::PromotionUnresolved { + record: finding.id.clone(), + cause: "state `promoted` carries no `promotedTo` — the state asserts \ + that a registry case now carries this finding, so the record \ + must name which one or the claim cannot be checked" + .to_string(), + }); + } + (FindingState::Promoted, Some(case)) if !registry.declares_case(case) => { + out.push(DiscoveryError::PromotionUnresolved { + record: finding.id.clone(), + cause: format!( + "`promotedTo` names case `{case}`, which is not declared in \ + `conformance/registry/cases/`. The finding reads as covered from \ + inside the queue while nothing executes it — the one shape a \ + promotion must never take" + ), + }); + } + (state, Some(case)) if state != FindingState::Promoted => { + out.push(DiscoveryError::PromotionUnresolved { + record: finding.id.clone(), + cause: format!( + "state `{}` carries `promotedTo` `{case}` — the field is set only \ + in state `promoted`, and carrying one anywhere else asserts a \ + promotion that did not happen", + state.as_str() + ), + }); + } + _ => {} + } + } +} + +/// D1 + D5 over the campaign history. +fn check_campaigns( + data: &DiscoveryData, + registry: &RegistryView<'_>, + out: &mut Vec, +) { + let mut seen: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new(); + + for campaign in &data.campaigns { + if !seen.insert(campaign.id.as_str()) { + out.push(DiscoveryError::MalformedRecord { + record: campaign.id.clone(), + cause: "duplicate campaign id".to_string(), + }); + } + // Identity: the id must be derivable from the record's own substance. Without + // this, a mis-assigned campaign id is undetectable — and every finding naming that + // campaign silently inherits provenance the record does not carry. + let derived = campaign.derived_id(); + if campaign.id != derived { + out.push(DiscoveryError::MalformedRecord { + record: campaign.id.clone(), + cause: format!( + "campaign id does not match its substance `seed ‖ \ + canonical(pinnedInputSet) ‖ lane ‖ profile ‖ tier` (expected \ + `{derived}`)" + ), + }); + } + // The profile is what says *which environment* the claim is about; a profile + // nothing declares is a claim about nowhere. + if !registry.declares_profile(&campaign.profile) { + out.push(DiscoveryError::UnresolvableReference { + record: campaign.id.clone(), + kind: "certification profile".to_string(), + reference: campaign.profile.clone(), + }); + } + if campaign.seed.trim().is_empty() { + out.push(DiscoveryError::MalformedRecord { + record: campaign.id.clone(), + cause: "empty seed — the seed is the reproducibility input and is never \ + defaulted or inferred (FR-001)" + .to_string(), + }); + } + // Caught here as well as at the write (see `write_campaigns`), so a record built + // in memory by a campaign in flight fails the same way a committed one would. + if !campaign.outcome.space_covered_fraction.is_finite() { + out.push(DiscoveryError::MalformedRecord { + record: campaign.id.clone(), + cause: format!( + "non-finite spaceCoveredFraction ({}) — it serializes as bare `null`, \ + which never loads back", + campaign.outcome.space_covered_fraction + ), + }); + } + out.extend(check_pinned_input_set( + &campaign.id, + &campaign.pinned_input_set, + registry, + )); + } +} + +/// **D5** — every pinned-input-set element that names a *registry revision* must resolve +/// in `revisions.json`. +/// +/// Three of the seven elements name revisions: the schema pin, the prose pin, and the +/// oracle version. The other four (`normalizerVersion`, `grammarVersion`, +/// `mutationCatalogVersion`, `generatorVersion`) are properties of this repository's own +/// code rather than upstream revisions, so they are *required to be non-empty* here but +/// are not looked up — inventing a `rev-` record for them would claim an upstream +/// provenance they do not have. +pub fn check_pinned_input_set( + record: &str, + pins: &PinnedInputSet, + registry: &RegistryView<'_>, +) -> Vec { + let mut out = Vec::new(); + + for (element, value, kind) in [ + ("schemaPin", &pins.schema_pin, RevisionKind::Schema), + ("prosePin", &pins.prose_pin, RevisionKind::Spec), + ("oracleVersion", &pins.oracle_version, RevisionKind::Oracle), + ] { + if !registry.records_pin(kind, value) { + out.push(DiscoveryError::StalePin { + record: record.to_string(), + element: element.to_string(), + value: value.clone(), + }); + } + } + + for (element, value) in [ + ("normalizerVersion", &pins.normalizer_version), + ("grammarVersion", &pins.grammar_version), + ("mutationCatalogVersion", &pins.mutation_catalog_version), + ("generatorVersion", &pins.generator_version), + ] { + if value.trim().is_empty() { + out.push(DiscoveryError::MalformedRecord { + record: record.to_string(), + cause: format!( + "pinned input `{element}` is empty — all seven elements are mandatory \ + (FR-002); an unrecorded element is a dimension along which the finding \ + silently stops being checkable" + ), + }); + } + } + + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::discovery::signature::{Divergence, DivergenceKind}; + use serde_json::json; + + fn revisions() -> Vec { + vec![ + SourceRevision { + id: "rev-schema-113500f4".into(), + kind: RevisionKind::Schema, + pin: "113500f4".into(), + url: String::new(), + verified_against: None, + }, + SourceRevision { + id: "rev-spec-113500f4".into(), + kind: RevisionKind::Spec, + pin: "113500f4".into(), + url: String::new(), + verified_against: None, + }, + SourceRevision { + id: "rev-oracle-0-87-0".into(), + kind: RevisionKind::Oracle, + pin: "0.87.0".into(), + url: String::new(), + verified_against: None, + }, + ] + } + + /// The certification profile every test campaign runs under. + const TEST_PROFILE: &str = "prof-linux-amd64-docker-0870"; + + /// The one case id the synthetic registry view declares — what a promotion may + /// legitimately name (**D3**). + const TEST_CASE: &str = "case-readconfig-decl-image"; + + fn view<'a>(revisions: &'a [SourceRevision]) -> RegistryView<'a> { + RegistryView { + channels: vec!["chan-structured-output", "chan-exit-code"], + revisions: revisions.iter().collect(), + profiles: vec![TEST_PROFILE], + cases: vec![TEST_CASE], + } + } + + fn pins() -> PinnedInputSet { + PinnedInputSet { + schema_pin: "113500f4".into(), + prose_pin: "113500f4".into(), + oracle_version: "0.87.0".into(), + normalizer_version: "6".into(), + grammar_version: "rev-schema-113500f4".into(), + mutation_catalog_version: "v1".into(), + generator_version: "splitmix64-seed+xoshiro256starstar/v1".into(), + } + } + + /// A **self-consistent** campaign: its id is derived from its own substance, so + /// `check` sees the record the derivation promises rather than a hand-picked id that + /// would (correctly) fail the D1 identity clause. `tag` varies the seed, which is what + /// makes two calls two different campaigns. + fn campaign(tag: &str) -> Campaign { + let seed = format!("0x5eed{tag}"); + let pinned_input_set = pins(); + let lane = CampaignLane::Invoked; + let tier = CampaignTier::ConfigDifferential; + Campaign { + id: Campaign::derive_id(&seed, &pinned_input_set, lane, TEST_PROFILE, tier), + seed, + lane, + tier, + profile: TEST_PROFILE.to_string(), + pinned_input_set, + budget: Budget { + wall_clock_seconds: DEFAULT_WALL_CLOCK_SECONDS, + per_candidate_seconds: DEFAULT_PER_CANDIDATE_SECONDS_HERMETIC, + shrink_steps_per_finding: 64, + admission_cap: DEFAULT_ADMISSION_CAP, + }, + outcome: CampaignOutcome { + candidates_generated: 10, + candidates_executed: 10, + candidates_discarded_unsafe: 0, + parse_stage_failures: 0, + budget_exhausted: false, + space_covered_fraction: 0.0, + mutation_applications: IndexMap::new(), + signatures_observed: 1, + signatures_admitted: 1, + signatures_suppressed: 0, + }, + } + } + + /// The derived id of `campaign(tag)` — what a witness or a `firstObserved` must name. + fn cid(tag: &str) -> String { + campaign(tag).id + } + + fn signature(path: &str) -> Signature { + let d = json!("vscode"); + let r = json!("root"); + Signature::derive( + "chan-structured-output", + &Divergence { + kind: DivergenceKind::Value, + path, + deacon: Some(&d), + reference: Some(&r), + }, + ) + } + + fn witness(campaign_id: &str, candidate_id: &str) -> Witness { + Witness { + id: Witness::derived_id(campaign_id, candidate_id), + campaign_id: campaign_id.into(), + candidate_id: candidate_id.into(), + minimal_input: json!({ "image": "alpine:3.18" }), + is_minimal: true, + reduction_steps: vec!["drop-optional-key".into()], + observed_values: ObservedValues { + deacon: Some(json!("vscode")), + reference: Some(json!("root")), + }, + mutation_operators: vec!["mop-wrong-type".into()], + } + } + + fn clean_data() -> DiscoveryData { + let sig = signature("configuration.remoteUser"); + let w = witness(&cid("a"), "cnd-11111111"); + DiscoveryData { + findings: vec![Finding::newly_admitted(sig, w, &cid("a"))], + campaigns: vec![campaign("a")], + corpus: Vec::new(), + } + } + + // --- T016/T017 record models ------------------------------------------ + + #[test] + fn the_contract_example_record_loads() { + // contracts/findings-queue.md § Record schema, with ids made self-consistent. + let sig = signature("configuration.remoteUser"); + let w = witness(&cid("a"), "cnd-11111111"); + let raw = format!( + r#"{{ + "schemaVersion": 1, + "records": [ + {{ + "id": "{}", + "signature": {}, + "witnesses": [{}], + "classification": "deacon-regression", + "state": "triaged", + "firstObserved": "{}", + "lastObserved": "{}", + "promotedTo": null, + "splitFrom": null, + "notes": "" + }} + ] + }}"#, + sig.finding_id(), + serde_json::to_string(&sig).unwrap(), + serde_json::to_string(&w).unwrap(), + cid("a"), + cid("a"), + ); + let file: FindingsFile = serde_json::from_str(&raw).expect("the contract example loads"); + let finding = &file.records[0]; + assert_eq!(finding.state, FindingState::Triaged); + assert_eq!( + finding.classification, + Some(Classification::DeaconRegression) + ); + assert_eq!(finding.witnesses.len(), 1); + } + + #[test] + fn unknown_fields_are_rejected_at_load() { + for raw in [ + r#"{"schemaVersion":1,"records":[],"extra":1}"#, + r#"{"schemaVersion":1,"records":[{"id":"fnd-1","typo":true}]}"#, + ] { + let err = serde_json::from_str::(raw) + .expect_err("strict JSON must reject unknown fields"); + let msg = err.to_string(); + assert!( + msg.contains("extra") || msg.contains("typo") || msg.contains("unknown field"), + "the diagnosis must name the offending field, got: {msg}" + ); + } + let err = serde_json::from_str::( + r#"{"schemaVersion":1,"records":[],"surprise":true}"#, + ) + .expect_err("strict JSON must reject unknown fields"); + assert!(err.to_string().contains("surprise")); + } + + #[test] + fn a_truncated_file_does_not_load_as_an_empty_queue() { + // The failure mode FR-029 exists to prevent: "the data was lost" must never read + // as "nothing was found". The writer always emits `records`, so its absence is + // damage, not a default. + let err = serde_json::from_str::(r#"{"schemaVersion":1}"#) + .expect_err("a records-less findings file must not load"); + assert!(err.to_string().contains("records"), "got: {err}"); + let err = serde_json::from_str::(r#"{"schemaVersion":1}"#) + .expect_err("a records-less campaigns file must not load"); + assert!(err.to_string().contains("records"), "got: {err}"); + } + + #[test] + fn an_unsupported_schema_version_is_refused_rather_than_downgraded() { + // Reading a v2 file under v1 semantics would be bad; the real hazard is the write + // that follows, since every writer stamps SCHEMA_VERSION and would silently + // downgrade the file. Refusing to read is the only way to guarantee not + // destroying it. + let err = serde_json::from_str::(r#"{"schemaVersion":2,"records":[]}"#) + .expect_err("a future schema version must not load"); + let msg = err.to_string(); + assert!(msg.contains("unsupported schemaVersion 2"), "got: {msg}"); + assert!(msg.contains("writing would stamp"), "got: {msg}"); + } + + #[test] + fn duplicate_ids_are_rejected_at_load_not_only_by_check() { + // Every by-id lookup takes the FIRST match, so a duplicate makes `triage` mutate + // one record while the other survives untouched — a write that appears to succeed + // and silently does not. Catching it only in `check` is too late. + let dir = tempfile::tempdir().expect("tempdir"); + let data = clean_data(); + let mut findings = data.findings.clone(); + findings.push(findings[0].clone()); + write_findings(dir.path(), &findings).expect("write"); + write_campaigns(dir.path(), &data.campaigns).expect("write"); + + let err = DiscoveryData::load(dir.path()).expect_err("duplicates must not load"); + let msg = err.to_string(); + assert!(msg.contains("duplicate finding id"), "got: {msg}"); + assert!( + msg.contains("first match"), + "the message must name the consequence" + ); + } + + // Unix-only: the simulation depends on `ENOTDIR`, which has no Windows equivalent — + // Windows maps a stat under a non-directory to `ErrorKind::NotFound`, so the probe + // takes `load_file`'s legitimate "the file is absent" branch and never reaches the + // error path this test is about. The PRODUCTION guarantee is platform-independent + // (`load_file` treats only `NotFound` as absent and turns every other stat error into + // a `SchemaError`); it is the way of provoking a non-`NotFound` stat error that is + // Unix-specific. + #[cfg(unix)] + #[test] + fn an_unreadable_file_is_not_mistaken_for_an_absent_one() { + // `Path::exists` returns false for ANY error, so a present-but-unreadable file + // would read as "this repository has not run a campaign yet". Simulate the + // stat failure with a path whose parent is a FILE, which yields ENOTDIR. + let dir = tempfile::tempdir().expect("tempdir"); + let blocker = dir.path().join("blocker"); + std::fs::write(&blocker, "not a directory").expect("write"); + let err = DiscoveryData::load(&blocker).expect_err("an unusable root must not load empty"); + let msg = err.to_string(); + assert!(msg.contains("findings.json"), "got: {msg}"); + } + + #[test] + fn an_empty_witness_list_is_rejected_at_deserialize_time() { + let sig = signature("p"); + let raw = format!( + r#"{{"schemaVersion":1,"records":[{{"id":"{}","signature":{},"witnesses":[], + "state":"untriaged","firstObserved":"cmp-a","lastObserved":"cmp-a"}}]}}"#, + sig.finding_id(), + serde_json::to_string(&sig).unwrap() + ); + let err = serde_json::from_str::(&raw) + .expect_err("a witness-less finding must not be representable"); + assert!(err.to_string().contains("at least one witness")); + } + + #[test] + fn classification_promotability_is_a_closed_property() { + assert!(Classification::DeaconRegression.is_promotable()); + assert!(Classification::ReferenceQuirk.is_promotable()); + assert!(Classification::SpecAmbiguity.is_promotable()); + assert!(Classification::UnsupportedBehavior.is_promotable()); + assert!( + !Classification::NormalizerDefect.is_promotable(), + "a normalizer defect is a defect in the machinery, not a behavior" + ); + assert!(!Classification::FixtureDefect.is_promotable()); + for c in Classification::all() { + assert_eq!(Classification::parse(c.as_str()), Some(*c)); + } + assert_eq!(Classification::parse("probably-fine"), None); + } + + #[test] + fn campaign_records_round_trip_through_strict_json() { + let c = campaign("a"); + let raw = serde_json::to_string_pretty(&c).expect("serializes"); + assert!(raw.contains("\"tier\": \"config-differential\"")); + assert!(raw.contains("\"lane\": \"invoked\"")); + assert!(raw.contains("\"perCandidateSeconds\": 60")); + let back: Campaign = serde_json::from_str(&raw).expect("round-trips"); + assert_eq!(back, c); + assert!(CampaignTier::ConfigDifferential.requires_oracle()); + assert!( + !CampaignTier::Metamorphic.requires_oracle(), + "the metamorphic tier needs no oracle, Docker, or network (research D12)" + ); + } + + // --- T018 loader ------------------------------------------------------- + + #[test] + fn the_committed_data_root_loads_and_is_empty() { + let data = DiscoveryData::load_default().expect("the committed discovery root must load"); + assert!(data.findings.is_empty()); + assert!(data.campaigns.is_empty()); + } + + #[test] + fn a_missing_root_is_empty_but_a_malformed_file_is_a_located_error() { + let dir = tempfile::tempdir().expect("tempdir"); + let empty = DiscoveryData::load(dir.path()).expect("a missing file is an empty collection"); + assert_eq!(empty, DiscoveryData::default()); + + std::fs::write(findings_path(dir.path()), "{ not json").expect("write"); + let err = + DiscoveryData::load(dir.path()).expect_err("a malformed file must not load empty"); + let msg = err.to_string(); + assert!(msg.contains("findings.json"), "got: {msg}"); + } + + #[test] + fn every_malformed_file_is_reported_in_one_pass() { + let dir = tempfile::tempdir().expect("tempdir"); + std::fs::write(findings_path(dir.path()), "{ nope").expect("write"); + std::fs::write(campaigns_path(dir.path()), "[ also nope").expect("write"); + let err = DiscoveryData::load(dir.path()).expect_err("both files are malformed"); + let msg = err.to_string(); + assert!( + msg.contains("findings.json") && msg.contains("campaigns.json"), + "a checker that stops at the first bad file makes fixing a batch a guessing \ + game; got: {msg}" + ); + } + + // --- T019 atomic writer ----------------------------------------------- + + #[test] + fn the_writer_is_atomic_byte_stable_and_round_trips() { + let dir = tempfile::tempdir().expect("tempdir"); + let data = clean_data(); + + write_findings(dir.path(), &data.findings).expect("write findings"); + write_campaigns(dir.path(), &data.campaigns).expect("write campaigns"); + + let back = DiscoveryData::load(dir.path()).expect("re-loads"); + assert_eq!(back, data, "the write/read round trip must be lossless"); + + // Byte-stable: writing the same content twice produces identical bytes, and a + // shorter payload leaves no trailing bytes from the longer one. + let first = std::fs::read_to_string(findings_path(dir.path())).expect("read"); + write_findings(dir.path(), &data.findings).expect("rewrite"); + assert_eq!( + first, + std::fs::read_to_string(findings_path(dir.path())).unwrap() + ); + write_findings(dir.path(), &[]).expect("truncate"); + let short = std::fs::read_to_string(findings_path(dir.path())).expect("read"); + assert_eq!(short, "{\n \"schemaVersion\": 1,\n \"records\": []\n}\n"); + + // No temp files survive a successful write. + let leftovers: Vec<_> = std::fs::read_dir(dir.path()) + .unwrap() + .filter_map(Result::ok) + .filter(|e| e.file_name().to_string_lossy().contains(".tmp.")) + .collect(); + assert!(leftovers.is_empty(), "temp files must be renamed away"); + } + + #[test] + fn the_rendered_empty_file_matches_the_committed_data_root() { + // The T003 seed files were hand-written; assert the writer reproduces them + // byte-for-byte, so the first campaign's write is a content diff and not a + // whole-file reformat. + let rendered = render_findings(&FindingsFile::default()); + for name in ["findings.json", "campaigns.json"] { + let committed = + std::fs::read_to_string(crate::default_discovery_dir().join(name)).expect(name); + assert_eq!( + committed, rendered, + "{name} must match the canonical rendering" + ); + } + + // `corpus.json` is no longer empty (US7 T106 populated it with the 33 pinned + // entries), so the claim is the same one in the form it can still take: the + // committed file IS the canonical rendering of its own records. That is what makes + // a digest recorded at first materialization a one-line diff rather than a + // whole-file reformat — the property this test exists for, restated over content + // rather than over emptiness. + let corpus_path = crate::default_discovery_dir().join("corpus.json"); + let committed = std::fs::read_to_string(&corpus_path).expect("corpus.json"); + let parsed: crate::discovery::corpus::CorpusFile = + serde_json::from_str(&committed).expect("corpus.json parses"); + assert!( + !parsed.records.is_empty(), + "the corpus manifest is populated; an empty one would make this assertion \ + vacuous rather than satisfied" + ); + assert_eq!( + committed, + crate::discovery::corpus::render(&parsed), + "corpus.json must match the canonical rendering" + ); + } + + // --- T020 upsert ------------------------------------------------------- + + #[test] + fn upsert_inserts_once_and_then_appends_witnesses() { + let mut findings: Vec = Vec::new(); + let sig = signature("configuration.remoteUser"); + + assert_eq!( + upsert_finding( + &mut findings, + sig.clone(), + witness(&cid("a"), "cnd-1"), + &cid("a") + ), + Upsert::Inserted + ); + assert_eq!(findings.len(), 1); + + // A different campaign observing the SAME signature appends rather than + // inserting — the queue reflects distinct problems, not campaign volume. + assert_eq!( + upsert_finding( + &mut findings, + sig.clone(), + witness(&cid("b"), "cnd-2"), + &cid("b") + ), + Upsert::WitnessAppended + ); + assert_eq!(findings.len(), 1, "a duplicate finding is unrepresentable"); + assert_eq!(findings[0].witnesses.len(), 2); + assert_eq!(findings[0].first_observed, cid("a")); + assert_eq!(findings[0].last_observed, cid("b")); + + // Re-observing the exact same (campaign, candidate) changes nothing. + assert_eq!( + upsert_finding( + &mut findings, + sig.clone(), + witness(&cid("b"), "cnd-2"), + &cid("b") + ), + Upsert::AlreadyWitnessed + ); + assert_eq!(findings[0].witnesses.len(), 2); + + // A DIFFERENT signature stays a different finding. + assert_eq!( + upsert_finding( + &mut findings, + signature("configuration.remoteEnv"), + witness(&cid("b"), "cnd-3"), + &cid("b") + ), + Upsert::Inserted + ); + assert_eq!(findings.len(), 2); + } + + #[test] + fn re_observation_revives_a_no_longer_reproducing_finding_keeping_its_classification() { + let mut findings = vec![Finding::newly_admitted( + signature("p"), + witness(&cid("a"), "cnd-1"), + &cid("a"), + )]; + findings[0].state = FindingState::NoLongerReproducing; + findings[0].classification = Some(Classification::DeaconRegression); + + upsert_finding( + &mut findings, + signature("p"), + witness(&cid("b"), "cnd-2"), + &cid("b"), + ); + assert_eq!(findings[0].state, FindingState::Triaged); + assert_eq!( + findings[0].classification, + Some(Classification::DeaconRegression), + "re-triaging a finding a reviewer already judged would be wasted work" + ); + } + + #[test] + fn a_split_ancestor_never_accepts_a_new_witness() { + let mut findings = vec![Finding::newly_admitted( + signature("p"), + witness(&cid("a"), "cnd-1"), + &cid("a"), + )]; + findings[0].state = FindingState::Split; + + assert_eq!( + upsert_finding( + &mut findings, + signature("p"), + witness(&cid("b"), "cnd-2"), + &cid("b") + ), + Upsert::SplitLineage, + "re-merging a split lineage silently reverts a reviewer's judgement (FR-032)" + ); + assert_eq!(findings[0].witnesses.len(), 1); + assert_eq!(findings[0].last_observed, cid("a")); + } + + // --- T070 split lineage ------------------------------------------------- + + #[test] + fn splitting_produces_one_lineage_keyed_child_per_witness() { + let mut findings = vec![Finding::newly_admitted( + signature("p"), + witness(&cid("a"), "cnd-1"), + &cid("a"), + )]; + // Two witnesses and a reviewer's classification: the shape a split acts on. + upsert_finding( + &mut findings, + signature("p"), + witness(&cid("b"), "cnd-2"), + &cid("b"), + ); + findings[0] + .triage(Classification::DeaconRegression, None) + .expect("triage"); + + let parent_id = findings[0].id.clone(); + let split = split_finding(&mut findings, &parent_id).expect("split"); + assert_eq!(split.children.len(), 2); + + // The parent is an inert ancestor: witnesses retained, classification surrendered. + let parent = findings.iter().find(|f| f.id == parent_id).expect("parent"); + assert_eq!(parent.state, FindingState::Split); + assert_eq!(parent.classification, None); + assert_eq!(parent.witnesses.len(), 2); + + for (child_id, source) in split.children.iter().zip(parent.witnesses.clone()) { + let child = findings.iter().find(|f| &f.id == child_id).expect("child"); + assert_eq!(child.split_from.as_deref(), Some(parent_id.as_str())); + assert_eq!(child.state, FindingState::Untriaged); + assert_eq!(child.classification, None); + assert_eq!(child.witnesses, vec![source.clone()]); + assert_eq!(child.first_observed, source.campaign_id); + assert_eq!(child.last_observed, source.campaign_id); + // Keyed to its own lineage, and NOT to the signature it shares with its parent. + assert_eq!(child.derived_child_id().as_deref(), Some(child_id.as_str())); + assert_ne!(child.id, child.signature.finding_id()); + } + assert_ne!(split.children[0], split.children[1]); + } + + #[test] + fn a_refused_split_leaves_the_queue_untouched() { + // Every check runs before the first mutation, so a refusal cannot leave a parent + // marked inert with no children — a shape that is both a D2 and a Q10 violation + // and that nothing would then repair. + let mut findings = vec![Finding::newly_admitted( + signature("p"), + witness(&cid("a"), "cnd-1"), + &cid("a"), + )]; + let before = findings.clone(); + let id = findings[0].id.clone(); + + let err = split_finding(&mut findings, &id).expect_err("one witness cannot be split"); + assert!(matches!(err, TransitionError::NothingToSplit { .. })); + assert!(err.to_string().contains("nothing to separate")); + assert_eq!(findings, before); + + assert!(matches!( + split_finding(&mut findings, "fnd-nowhere"), + Err(TransitionError::UnknownFinding { .. }) + )); + assert_eq!(findings, before); + } + + #[test] + fn a_split_lineage_is_never_re_merged_even_after_the_ancestor_is_gone() { + // The clause that makes FR-032 hold on the LINEAGE rather than on one record's + // continued existence. Without it the id lookup misses (a child is keyed to its + // parent, not to the signature), a fresh merged record is created under the + // ancestor's id, and the reviewer's decision is silently undone. + let mut findings = vec![Finding::newly_admitted( + signature("p"), + witness(&cid("a"), "cnd-1"), + &cid("a"), + )]; + upsert_finding( + &mut findings, + signature("p"), + witness(&cid("b"), "cnd-2"), + &cid("b"), + ); + let parent_id = findings[0].id.clone(); + findings[0] + .triage(Classification::DeaconRegression, None) + .expect("triage"); + split_finding(&mut findings, &parent_id).expect("split"); + + findings.retain(|f| f.id != parent_id); + assert_eq!( + upsert_finding( + &mut findings, + signature("p"), + witness(&cid("a"), "cnd-9"), + &cid("a") + ), + Upsert::SplitLineage + ); + assert_eq!(findings.len(), 2, "no merged record may be resurrected"); + } + + // --- T069 state machine ------------------------------------------------- + + #[test] + fn the_transition_table_is_exactly_the_declared_state_machine() { + use FindingState::*; + let permitted = [ + (Untriaged, Triaged), + (Triaged, Split), + (Triaged, Promoted), + (Triaged, NoLongerReproducing), + (NoLongerReproducing, Triaged), + ]; + for &from in FindingState::all() { + for &to in FindingState::all() { + assert_eq!( + from.may_transition_to(to), + permitted.contains(&(from, to)), + "{} -> {}", + from.as_str(), + to.as_str() + ); + } + } + // The three arrows that look plausible and are absent on purpose. + assert!(!Untriaged.may_transition_to(NoLongerReproducing)); + assert!(!Untriaged.may_transition_to(Split)); + assert!(!NoLongerReproducing.may_transition_to(Promoted)); + // Terminal states go nowhere. + for &to in FindingState::all() { + assert!(!Promoted.may_transition_to(to)); + assert!(!Split.may_transition_to(to)); + } + } + + #[test] + fn triage_advances_only_out_of_untriaged_and_never_out_of_a_terminal_state() { + let mut finding = + Finding::newly_admitted(signature("p"), witness(&cid("a"), "cnd-1"), &cid("a")); + + assert_eq!( + finding + .triage(Classification::DeaconRegression, Some("because")) + .expect("first triage"), + FindingState::Triaged + ); + assert_eq!(finding.notes, "because"); + + // Re-classifying is not a transition: the state stays put. + assert_eq!( + finding.triage(Classification::SpecAmbiguity, None).unwrap(), + FindingState::Triaged + ); + assert_eq!(finding.classification, Some(Classification::SpecAmbiguity)); + + // And a no-longer-reproducing finding is classifiable WITHOUT being revived: only + // a campaign that reproduces it may do that. + finding + .mark_no_longer_reproducing() + .expect("stops reproducing"); + assert_eq!( + finding + .triage(Classification::DeaconRegression, None) + .unwrap(), + FindingState::NoLongerReproducing + ); + + for terminal in [FindingState::Promoted, FindingState::Split] { + finding.state = terminal; + let err = finding + .triage(Classification::DeaconRegression, None) + .expect_err("a terminal state is not the reviewer's to re-classify"); + assert!(matches!(err, TransitionError::NotPermitted { .. })); + } + } + + #[test] + fn promotion_requires_a_classification_and_refuses_the_non_promotable_two() { + let mut finding = + Finding::newly_admitted(signature("p"), witness(&cid("a"), "cnd-1"), &cid("a")); + + // Untriaged: no classification at all. + assert!(matches!( + finding.promote("case-x"), + Err(TransitionError::MissingClassification { .. }) + )); + + for non_promotable in [ + Classification::NormalizerDefect, + Classification::FixtureDefect, + ] { + finding.state = FindingState::Untriaged; + finding.classification = None; + finding.triage(non_promotable, None).expect("triage"); + let err = finding + .promote("case-x") + .expect_err("a machinery defect is not a behavior of either implementation"); + assert!(matches!(err, TransitionError::NonPromotable { .. })); + assert!(err.to_string().contains("not promotable")); + assert_eq!( + finding.state, + FindingState::Triaged, + "a refused promotion must not move the finding" + ); + assert_eq!(finding.promoted_to, None); + } + + finding + .triage(Classification::DeaconRegression, None) + .expect("triage"); + finding.promote("case-real").expect("promotable"); + assert_eq!(finding.state, FindingState::Promoted); + assert_eq!(finding.promoted_to.as_deref(), Some("case-real")); + + // Promoted is terminal. + assert!(matches!( + finding.promote("case-other"), + Err(TransitionError::NotPermitted { .. }) + )); + } + + #[test] + fn an_untriaged_finding_that_stops_reproducing_stays_untriaged() { + // The state requires a classification (D2), so allowing the arrow would manufacture + // a violation out of a campaign merely not re-observing something nobody had looked + // at. + let mut finding = + Finding::newly_admitted(signature("p"), witness(&cid("a"), "cnd-1"), &cid("a")); + let err = finding + .mark_no_longer_reproducing() + .expect_err("untriaged has no judgement for the disappearance to invalidate"); + assert!(matches!(err, TransitionError::MissingClassification { .. })); + assert_eq!(finding.state, FindingState::Untriaged); + } + + // --- T068 D2 ------------------------------------------------------------ + + #[test] + fn d2_a_triaged_or_later_finding_must_carry_a_classification() { + let revs = revisions(); + for state in [ + FindingState::Triaged, + FindingState::Promoted, + FindingState::NoLongerReproducing, + ] { + let mut data = clean_data(); + data.findings[0].state = state; + data.findings[0].classification = None; + let violations = check(&data, &view(&revs)); + assert!( + violations + .iter() + .any(|v| v.class() == "D2" && v.to_string().contains("carries none")), + "state {}: {violations:?}", + state.as_str() + ); + } + } + + #[test] + fn d2_an_untriaged_or_split_finding_must_not_carry_one() { + let revs = revisions(); + + let mut data = clean_data(); + data.findings[0].classification = Some(Classification::DeaconRegression); + assert!( + check(&data, &view(&revs)) + .iter() + .any(|v| v.class() == "D2" && v.to_string().contains("nobody has looked")), + "an untriaged finding carrying a classification makes the FR-029 bucket lie" + ); + + let mut data = clean_data(); + data.findings[0].state = FindingState::Split; + data.findings[0].classification = Some(Classification::DeaconRegression); + assert!( + check(&data, &view(&revs)) + .iter() + .any(|v| v.class() == "D2" + && v.to_string().contains("the judgement the split rejected")), + "a split parent that keeps a classification asserts what the split rejected" + ); + } + + #[test] + fn d2_a_promoted_finding_may_not_be_classified_normalizer_or_fixture_defect() { + let revs = revisions(); + for non_promotable in [ + Classification::NormalizerDefect, + Classification::FixtureDefect, + ] { + let mut data = clean_data(); + data.findings[0].state = FindingState::Promoted; + data.findings[0].classification = Some(non_promotable); + data.findings[0].promoted_to = Some("case-anything".into()); + let violations = check(&data, &view(&revs)); + assert!( + violations + .iter() + .any(|v| v.class() == "D2" && v.to_string().contains("not promotable")), + "{}: {violations:?}", + non_promotable.as_str() + ); + } + } + + // --- T080 D3 ------------------------------------------------------------ + + /// A promotion the registry *can* back is clean — the positive control, without which + /// every assertion below would pass on a check that rejected all promotions. + #[test] + fn d3_a_promotion_naming_a_declared_case_is_clean() { + let revs = revisions(); + let mut data = clean_data(); + data.findings[0].state = FindingState::Promoted; + data.findings[0].classification = Some(Classification::DeaconRegression); + data.findings[0].promoted_to = Some(TEST_CASE.to_string()); + assert_eq!(check(&data, &view(&revs)), Vec::new()); + } + + #[test] + fn d3_a_promoted_finding_must_name_a_case() { + let revs = revisions(); + let mut data = clean_data(); + data.findings[0].state = FindingState::Promoted; + data.findings[0].classification = Some(Classification::DeaconRegression); + data.findings[0].promoted_to = None; + let violations = check(&data, &view(&revs)); + assert!( + violations + .iter() + .any(|v| v.class() == "D3" && v.to_string().contains("carries no `promotedTo`")), + "a promotion that names nothing is a claim nobody can check: {violations:?}" + ); + } + + #[test] + fn d3_a_promotion_naming_an_unknown_case_is_coverage_that_does_not_exist() { + let revs = revisions(); + let mut data = clean_data(); + data.findings[0].state = FindingState::Promoted; + data.findings[0].classification = Some(Classification::DeaconRegression); + data.findings[0].promoted_to = Some("case-that-was-deleted".to_string()); + let violations = check(&data, &view(&revs)); + assert!( + violations.iter().any(|v| v.class() == "D3" + && v.to_string().contains("case-that-was-deleted") + && v.to_string().contains("not declared")), + "the diagnosis must NAME the case that does not resolve: {violations:?}" + ); + } + + /// `promotedTo` outside state `promoted` asserts a promotion that did not happen — + /// and it always errs toward claiming MORE coverage than exists, which is the + /// direction that matters. + #[test] + fn d3_promoted_to_is_set_only_in_state_promoted() { + let revs = revisions(); + for (state, classification) in [ + (FindingState::Untriaged, None), + ( + FindingState::Triaged, + Some(Classification::DeaconRegression), + ), + ( + FindingState::NoLongerReproducing, + Some(Classification::DeaconRegression), + ), + ] { + let mut data = clean_data(); + data.findings[0].state = state; + data.findings[0].classification = classification; + // A case that DOES resolve, so the only objection left is the state. + data.findings[0].promoted_to = Some(TEST_CASE.to_string()); + let violations = check(&data, &view(&revs)); + assert!( + violations + .iter() + .any(|v| v.class() == "D3" + && v.to_string().contains("set only in state `promoted`")), + "state {}: {violations:?}", + state.as_str() + ); + } + } + + #[test] + fn d1_a_split_ancestor_needs_at_least_two_children() { + let revs = revisions(); + let mut data = clean_data(); + data.findings[0].state = FindingState::Split; + let violations = check(&data, &view(&revs)); + assert!( + violations + .iter() + .any(|v| v.class() == "D1" && v.to_string().contains("0 child(ren)")), + "{violations:?}" + ); + } + + // --- T021 D1 / D5 ------------------------------------------------------ + + #[test] + fn a_clean_data_root_reports_no_violations() { + let revs = revisions(); + assert_eq!(check(&clean_data(), &view(&revs)), Vec::new()); + } + + #[test] + fn the_committed_data_root_validates_clean() { + let registry = crate::load::Registry::load(&crate::default_registry_dir()) + .expect("the committed registry must load"); + let data = DiscoveryData::load_default().expect("the committed discovery root must load"); + let violations = check(&data, &RegistryView::from_registry(®istry)); + assert!(violations.is_empty(), "{violations:?}"); + } + + #[test] + fn d1_undeclared_channel() { + let revs = revisions(); + let mut data = clean_data(); + data.findings[0].signature.channel = "chan-invented".into(); + // Re-derive so the identity checks do not also fire and mask the point. + data.findings[0].signature.id = data.findings[0].signature.derived_id(); + data.findings[0].id = data.findings[0].signature.finding_id(); + + let violations = check(&data, &view(&revs)); + assert_eq!(violations.len(), 1, "{violations:?}"); + assert_eq!(violations[0].class(), "D1"); + assert!(violations[0].to_string().contains("chan-invented")); + } + + #[test] + fn d1_unresolvable_campaign_reference() { + let revs = revisions(); + let mut data = clean_data(); + data.findings[0].last_observed = "cmp-ghost".into(); + + let violations = check(&data, &view(&revs)); + assert!( + violations + .iter() + .any(|v| v.class() == "D1" && v.to_string().contains("cmp-ghost")), + "{violations:?}" + ); + } + + #[test] + fn d1_empty_witnesses_and_broken_identity() { + let revs = revisions(); + let mut data = clean_data(); + data.findings[0].witnesses.clear(); + data.findings[0].id = "fnd-deadbeef".into(); + + let violations = check(&data, &view(&revs)); + let text: Vec = violations.iter().map(|v| v.to_string()).collect(); + assert!( + text.iter().any(|m| m.contains("no witnesses")), + "expected the empty-witness violation in {text:?}" + ); + assert!( + text.iter() + .any(|m| m.contains("does not match its signature")), + "expected the derived-id violation in {text:?}" + ); + assert!(violations.iter().all(|v| v.class() == "D1")); + } + + #[test] + fn d1_duplicate_ids_and_dangling_split_parent() { + let revs = revisions(); + let mut data = clean_data(); + let dup = data.findings[0].clone(); + data.findings.push(dup); + data.findings[1].split_from = Some("fnd-nowhere".into()); + data.campaigns.push(campaign("a")); + + let text: Vec = check(&data, &view(&revs)) + .iter() + .map(|v| v.to_string()) + .collect(); + assert!( + text.iter().any(|m| m.contains("duplicate finding id")), + "{text:?}" + ); + assert!( + text.iter().any(|m| m.contains("duplicate campaign id")), + "{text:?}" + ); + assert!(text.iter().any(|m| m.contains("fnd-nowhere")), "{text:?}"); + } + + #[test] + fn d1_witness_id_must_match_its_substance() { + let revs = revisions(); + let mut data = clean_data(); + data.findings[0].witnesses[0].id = "wit-00000000".into(); + + let violations = check(&data, &view(&revs)); + assert!( + violations + .iter() + .any(|v| v.to_string().contains("witness id does not match")), + "{violations:?}" + ); + } + + #[test] + fn d1_a_mis_assigned_campaign_id_is_detected() { + // data-model.md § 1 declares the campaign id derived, but a derivation nothing + // checks is a comment. Without this clause a mis-assigned id is undetectable, and + // every finding naming that campaign silently inherits provenance the record does + // not carry. + let revs = revisions(); + let mut data = clean_data(); + let real = data.campaigns[0].id.clone(); + let forged = "cmp-deadbeef".to_string(); + data.campaigns[0].id = forged.clone(); + data.findings[0].first_observed = forged.clone(); + data.findings[0].last_observed = forged.clone(); + data.findings[0].witnesses[0].campaign_id = forged.clone(); + data.findings[0].witnesses[0].id = Witness::derived_id(&forged, "cnd-11111111"); + + let violations = check(&data, &view(&revs)); + assert!( + violations.iter().any(|v| v.class() == "D1" + && v.to_string() + .contains("campaign id does not match its substance") + && v.to_string().contains(&real)), + "the diagnosis must name the id the substance derives to: {violations:?}" + ); + } + + #[test] + fn d1_every_component_of_the_campaign_id_changes_it() { + // Substance-anchoring means the id changes exactly when the thing does. `tier` is + // the component data-model.md § 1's four-part list omits: without it, a + // metamorphic and a config-differential run of the same seed in the same lane + // under the same profile would share an id, which the append-only history cannot + // represent (it is a duplicate-id D1). + let base = campaign("a"); + let id = base.id.clone(); + + let mut other_seed = base.clone(); + other_seed.seed = "0xdifferent".into(); + assert_ne!(id, other_seed.derived_id(), "seed"); + + let mut other_lane = base.clone(); + other_lane.lane = CampaignLane::Scheduled; + assert_ne!(id, other_lane.derived_id(), "lane"); + + let mut other_profile = base.clone(); + other_profile.profile = "prof-linux-amd64-podman-0870".into(); + assert_ne!(id, other_profile.derived_id(), "profile"); + + let mut other_pins = base.clone(); + other_pins.pinned_input_set.oracle_version = "0.86.0".into(); + assert_ne!(id, other_pins.derived_id(), "pinnedInputSet"); + + let mut other_tier = base.clone(); + other_tier.tier = CampaignTier::Metamorphic; + assert_ne!( + id, + other_tier.derived_id(), + "a tier decides which implementations are compared over which channels — two \ + tiers are two different observations and must not share an id" + ); + + // The outcome and the budget are NOT substance: re-running the same campaign and + // recording different volumes does not make it a different campaign. + let mut other_outcome = base.clone(); + other_outcome.outcome.candidates_generated += 1; + other_outcome.budget.wall_clock_seconds += 1; + assert_eq!(id, other_outcome.derived_id()); + } + + #[test] + fn d1_a_campaign_profile_must_resolve_in_profiles_json() { + // The profile says *which environment* the claim is about; a profile nothing + // declares is a claim about nowhere. + let revs = revisions(); + let mut data = clean_data(); + data.campaigns[0].profile = "prof-imaginary".into(); + data.campaigns[0].id = data.campaigns[0].derived_id(); + let id = data.campaigns[0].id.clone(); + data.findings[0].first_observed = id.clone(); + data.findings[0].last_observed = id.clone(); + data.findings[0].witnesses[0].campaign_id = id.clone(); + data.findings[0].witnesses[0].id = Witness::derived_id(&id, "cnd-11111111"); + + let violations = check(&data, &view(&revs)); + assert!( + violations.iter().any(|v| v.class() == "D1" + && v.to_string().contains("certification profile") + && v.to_string().contains("prof-imaginary")), + "{violations:?}" + ); + } + + #[test] + fn d5_pins_absent_from_revisions_json() { + let revs = revisions(); + let mut data = clean_data(); + data.campaigns[0].pinned_input_set.schema_pin = "deadbeef".into(); + data.campaigns[0].pinned_input_set.oracle_version = "9.9.9".into(); + + let violations = check(&data, &view(&revs)); + let stale: Vec<&DiscoveryError> = violations.iter().filter(|v| v.class() == "D5").collect(); + assert_eq!(stale.len(), 2, "{violations:?}"); + assert!(stale.iter().any(|v| v.to_string().contains("deadbeef"))); + assert!(stale.iter().any(|v| v.to_string().contains("9.9.9"))); + assert!( + stale[0].to_string().contains("revisions.json"), + "the remedy must name where the revision belongs" + ); + } + + #[test] + fn d5_matches_on_the_pin_not_the_revision_id() { + // A campaign records the PIN (`113500f4`), not the `rev-` id. Accepting the id + // too would let two spellings of the same claim diverge. + let revs = revisions(); + let mut data = clean_data(); + data.campaigns[0].pinned_input_set.schema_pin = "rev-schema-113500f4".into(); + let violations = check(&data, &view(&revs)); + assert!( + violations.iter().any(|v| v.class() == "D5"), + "{violations:?}" + ); + } + + #[test] + fn a_repo_owned_pin_element_may_not_be_empty() { + let revs = revisions(); + let mut data = clean_data(); + data.campaigns[0].pinned_input_set.generator_version = " ".into(); + + let violations = check(&data, &view(&revs)); + assert!( + violations + .iter() + .any(|v| v.to_string().contains("generatorVersion") && v.class() == "D1"), + "all seven elements are mandatory; {violations:?}" + ); + } + + #[test] + fn a_non_finite_space_covered_fraction_is_refused_by_both_the_writer_and_check() { + // serde_json renders NaN/±Inf as bare `null` and does NOT error, and `null` is not + // a valid f64 coming back — so writing one produces a campaigns.json that can + // never be loaded again, taking the queue's whole provenance with it. A zero + // `candidatesGenerated` on an aborted campaign is exactly that shape. + let dir = tempfile::tempdir().expect("tempdir"); + let mut data = clean_data(); + data.campaigns[0].outcome.space_covered_fraction = f64::NAN; + + let err = write_campaigns(dir.path(), &data.campaigns) + .expect_err("a non-finite fraction must not be written"); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert!(err.to_string().contains("non-finite"), "got: {err}"); + assert!( + !campaigns_path(dir.path()).exists(), + "the refusal must leave no file behind" + ); + + // And the in-memory record fails the same way, so a campaign in flight is caught + // before it ever reaches the writer. + let revs = revisions(); + let violations = check(&data, &view(&revs)); + assert!( + violations + .iter() + .any(|v| v.to_string().contains("non-finite spaceCoveredFraction")), + "{violations:?}" + ); + } + + #[test] + fn d1_last_observed_must_be_backed_by_a_witness() { + // Existence alone is too weak: `lastObserved` asserts an OBSERVATION, and a + // record naming a real but unwitnessed campaign claims provenance it does not + // have — the shape a careless hand edit produces. + let revs = revisions(); + let mut data = clean_data(); + data.campaigns.push(campaign("b")); + data.findings[0].last_observed = cid("b"); + + let violations = check(&data, &view(&revs)); + assert!( + violations + .iter() + .any(|v| v.to_string().contains("witnessed nothing for this finding")), + "{violations:?}" + ); + } + + #[test] + fn d1_split_from_may_not_name_the_finding_itself() { + let revs = revisions(); + let mut data = clean_data(); + let id = data.findings[0].id.clone(); + data.findings[0].split_from = Some(id); + + let violations = check(&data, &view(&revs)); + assert!( + violations + .iter() + .any(|v| v.to_string().contains("names the finding itself")), + "a self-reference resolves trivially and would otherwise pass: {violations:?}" + ); + } + + #[test] + fn a_split_child_is_keyed_to_its_lineage_rather_than_to_its_signature() { + // A child shares its parent's signature by construction (a split separates + // witnesses, not signatures), so it cannot also carry the parent's derived id. + // It is not exempt from identity, though — it anchors on `parent ‖ its witnesses`, + // which keeps the property that matters without the collision. + let revs = revisions(); + let mut data = clean_data(); + let mut findings = data.findings.clone(); + data.campaigns.push(campaign("b")); + upsert_finding( + &mut findings, + signature("configuration.remoteUser"), + witness(&cid("b"), "cnd-22222222"), + &cid("b"), + ); + findings[0] + .triage(Classification::DeaconRegression, None) + .expect("triage"); + let parent_id = findings[0].id.clone(); + split_finding(&mut findings, &parent_id).expect("split"); + data.findings = findings; + + let violations = check(&data, &view(&revs)); + assert!( + violations.is_empty(), + "a split produced by the tooling must validate clean: {violations:?}" + ); + + // A hand-edited child id detaches the record from its lineage and is caught. + data.findings[1].id = "fnd-handwritten".into(); + let violations = check(&data, &view(&revs)); + assert!( + violations + .iter() + .any(|v| v.to_string().contains("split-child id does not match")), + "{violations:?}" + ); + + // And a NON-child with a wrong id is still judged against its signature. + data.findings[1].split_from = None; + let violations = check(&data, &view(&revs)); + assert!( + violations + .iter() + .any(|v| v.to_string().contains("does not match its signature")), + "{violations:?}" + ); + } + + #[test] + fn an_empty_seed_is_a_violation() { + let revs = revisions(); + let mut data = clean_data(); + data.campaigns[0].seed = String::new(); + let violations = check(&data, &view(&revs)); + assert!( + violations + .iter() + .any(|v| v.to_string().contains("empty seed")), + "{violations:?}" + ); + } +} diff --git a/crates/conformance/src/discovery/report.rs b/crates/conformance/src/discovery/report.rs new file mode 100644 index 00000000..fa969b7c --- /dev/null +++ b/crates/conformance/src/discovery/report.rs @@ -0,0 +1,1349 @@ +//! Byte-stable discovery reports — `target/discovery/queue.{json,md}` +//! (025-exploratory-parity-discovery, contracts/discovery-cli.md, quickstart.md § 2). +//! +//! ## Reporting never gates +//! +//! `discovery report`'s exit status reflects only whether the artifacts were **written**, +//! never what they say. A queue holding fifty untriaged findings still exits `0`. This is +//! the `coverage report` discipline extended to the discovery surface (FR-058): any +//! command whose status depends on its findings becomes a gate the moment someone wires +//! it into CI, and a stochastic gate makes green non-reproducible. +//! +//! ## Volume is reported even when nothing was found +//! +//! A campaign that found nothing and a campaign that never ran are completely different +//! facts (FR-062). The report therefore always carries `candidatesGenerated` / +//! `candidatesExecuted` per campaign, so "nothing found" can never be confused with +//! "nothing ran" — which would otherwise make a broken pipeline the most comfortable +//! possible state for the machinery to be in. +//! +//! ## Byte-stable +//! +//! No timestamps, no absolute paths, no map iteration order that is not explicitly +//! sorted. Two runs over the same data produce identical bytes, so the artifacts diff +//! cleanly and a change in the report means a change in the queue. +//! +//! ## Grouping is a view, never a merge +//! +//! Distinct signatures stay distinct findings even when they map to the same behavior +//! (FR-031); they are *reported* grouped. Merging them would destroy the ability to tell +//! whether a fix addressed one cause or all of them, so [`FindingGroup`] names the +//! findings that relate and changes nothing about them. + +use std::path::{Path, PathBuf}; + +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; + +use super::mutate; +use super::queue::{Budget, Campaign, DiscoveryData, Finding, FindingState, PinnedInputSet}; + +/// The pins a finding is compared against to decide whether it is **pin-stale**. +/// +/// A finding is a claim about a *specific* pinned pair of implementations. On a pin +/// change the claim is neither true nor false — it is unverified — so such findings are +/// listed for re-evaluation rather than carried forward, which would assert that a +/// difference observed against oracle *v0.86* still holds against *v0.87* +/// (contracts/findings-queue.md, "Pin invalidation"). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CurrentPins { + /// The vendored schema revision pin. + pub schema_pin: String, + /// The vendored spec-prose revision pin. + pub prose_pin: String, + /// The recorded oracle version, or `None` when the registry records no oracle + /// revision at all. + /// + /// `None` means **"oracle staleness is not decidable"**, not "the current oracle + /// version is the empty string". The distinction is the whole point: comparing every + /// campaign's real oracle version against `""` would mark the entire queue pin-stale + /// for a registry that simply had not recorded an oracle yet, which reads as a + /// catastrophic pin bump instead of as missing metadata. + pub oracle_version: Option, +} + +impl CurrentPins { + /// Resolve the current pins from a loaded registry. + /// + /// The schema and prose pins fall back to this crate's compiled-in constants, which + /// are the same values the registry is validated against (V14); the oracle pin has no + /// such constant, so its absence is represented rather than invented. + pub fn from_registry(registry: &crate::load::Registry) -> CurrentPins { + let pin_of = |kind: crate::model::RevisionKind| -> Option { + registry + .revisions + .iter() + .find(|r| r.kind == kind) + .map(|r| r.pin.clone()) + }; + CurrentPins { + schema_pin: pin_of(crate::model::RevisionKind::Schema) + .unwrap_or_else(|| crate::CURRENT_SCHEMA_PIN.to_string()), + prose_pin: pin_of(crate::model::RevisionKind::Spec) + .unwrap_or_else(|| crate::CURRENT_SPEC_PIN.to_string()), + oracle_version: pin_of(crate::model::RevisionKind::Oracle), + } + } + + /// Whether `campaign` ran under a different pinned pair than the current one. + /// + /// An undecidable element (see [`oracle_version`](Self::oracle_version)) contributes + /// nothing: "we cannot tell" must never be reported as "it differs". + fn differs_from(&self, campaign: &Campaign) -> bool { + let pins = &campaign.pinned_input_set; + pins.schema_pin != self.schema_pin + || pins.prose_pin != self.prose_pin + || self + .oracle_version + .as_ref() + .is_some_and(|current| &pins.oracle_version != current) + } +} + +/// One finding as the report renders it — enough to triage from, without the witness +/// payloads that would make the artifact unreadable. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FindingSummary { + /// `fnd-`. + pub id: String, + /// The signature's channel. + pub channel: String, + /// The signature's observable path. + pub path: String, + /// The difference kind's wire spelling. + pub kind: String, + /// The value-shape class's wire spelling. + pub value_shape_class: String, + /// How many witnesses back it. + pub witnesses: usize, + /// Its classification, if triaged. + pub classification: Option, + /// The campaign that first admitted it. + pub first_observed: String, + /// The most recent campaign that reproduced it — for `no-longer-reproducing`, this + /// is *the campaign that last observed it*, which FR-033 requires to be reported + /// rather than deleted along with the record. + pub last_observed: String, + /// The registry case carrying it, when promoted. + pub promoted_to: Option, +} + +impl FindingSummary { + fn of(finding: &Finding) -> FindingSummary { + FindingSummary { + id: finding.id.clone(), + channel: finding.signature.channel.clone(), + path: finding.signature.path.clone(), + kind: finding.signature.kind.as_str().to_string(), + value_shape_class: finding.signature.value_shape_class.as_str().to_string(), + witnesses: finding.witnesses.len(), + classification: finding.classification.map(|c| c.as_str().to_string()), + first_observed: finding.first_observed.clone(), + last_observed: finding.last_observed.clone(), + promoted_to: finding.promoted_to.clone(), + } + } +} + +/// Resolves a promoted finding to the behavior identities its case claims (**T073**). +/// +/// Built from the registry rather than from the queue because a finding **never names a +/// behavior**. That is not an omission: FR-025 forbids a discovery program inventing a +/// behavior identity, so the only behavior a finding can honestly be grouped under is the +/// one a human already attached to it — by promoting it into a case that names behaviors. +/// Anything else would be the report asserting a mapping nobody reviewed. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct BehaviorIndex { + /// `case-` → the behaviors that case claims, in the case's own declaration order. + case_behaviors: std::collections::BTreeMap>, +} + +impl BehaviorIndex { + /// Index every registry case's behaviors. + pub fn from_registry(registry: &crate::load::Registry) -> BehaviorIndex { + BehaviorIndex { + case_behaviors: registry + .cases + .iter() + .map(|c| (c.id.clone(), c.behaviors.clone())) + .collect(), + } + } + + /// The behaviors a finding is known to map to — empty unless it is promoted into a + /// case that resolves. + fn behaviors_of(&self, finding: &Finding) -> &[String] { + finding + .promoted_to + .as_deref() + .and_then(|case| self.case_behaviors.get(case)) + .map_or(&[][..], Vec::as_slice) + } +} + +/// How a [`FindingGroup`] was keyed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum GroupKind { + /// A `bhv-` identity, resolved through a promoted finding's case. A *reviewed* + /// mapping. + Behavior, + /// `channel ‖ path` — the observable location the findings share. Explicitly **not** a + /// behavior claim: it is the strongest grouping available before a human has decided + /// what these differences mean. + ObservablePath, +} + +/// Several distinct findings reported together (FR-031). +/// +/// **Grouping is a view, never a merge.** FR-031 requires distinct signatures to stay +/// distinct findings even when they map to the same behavior, and permits reporting them +/// grouped. Merging would destroy the ability to tell whether a fix addressed one cause or +/// all of them; grouping without merging keeps both facts — the reviewer sees the +/// relationship, and each finding retains its own witnesses, its own classification, and +/// its own promotion. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FindingGroup { + /// The behavior id, or `channel ‖ path`. + pub key: String, + /// What the key is. + pub kind: GroupKind, + /// The findings sharing it — **≥ 2**, in the queue's own file order. A group of one is + /// the finding itself and would only pad the report. + pub findings: Vec, +} + +/// One campaign's volume line. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CampaignSummary { + /// `cmp-`. + pub id: String, + /// The recorded seed — the reproducibility input. + pub seed: String, + /// The tier's wire spelling. + pub tier: String, + /// The lane's wire spelling. + pub lane: String, + /// Candidates generated — reported even when nothing was found (FR-062). + pub candidates_generated: u64, + /// Candidates executed. + pub candidates_executed: u64, + /// Distinct signatures admitted. + pub signatures_admitted: u64, + /// Distinct signatures the admission cap suppressed — never silent (FR-034b). + pub signatures_suppressed: u64, + /// Whether the wall-clock budget ran out. + pub budget_exhausted: bool, +} + +impl CampaignSummary { + fn of(campaign: &Campaign) -> CampaignSummary { + CampaignSummary { + id: campaign.id.clone(), + seed: campaign.seed.clone(), + tier: campaign.tier.as_str().to_string(), + lane: campaign.lane.as_str().to_string(), + candidates_generated: campaign.outcome.candidates_generated, + candidates_executed: campaign.outcome.candidates_executed, + signatures_admitted: campaign.outcome.signatures_admitted, + signatures_suppressed: campaign.outcome.signatures_suppressed, + budget_exhausted: campaign.outcome.budget_exhausted, + } + } +} + +/// The queue report (quickstart.md § 2's five buckets). +/// +/// `untriaged` is **counted**, not merely listed, which is the whole point of FR-029: +/// "nobody has looked yet" must never read as "nothing found". +/// +/// `pinStale` is **cross-cutting**, not a sixth state: a pin-stale finding also appears +/// in whichever state bucket it is in. Making it a state would force a finding to stop +/// being `triaged` because a pin moved, discarding a reviewer's judgement over an event +/// that says nothing about the finding's cause. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct QueueReport { + /// Total findings in the queue. + pub total: usize, + /// Nobody has looked yet. + pub untriaged: Vec, + /// Classified, awaiting a decision. + pub triaged: Vec, + /// Split into children; an inert ancestor. + pub split: Vec, + /// Now carried by a real case, named. + pub promoted: Vec, + /// Stopped reproducing; names the campaign that last saw it. + pub no_longer_reproducing: Vec, + /// Observed under pins that no longer match; awaiting re-evaluation. + pub pin_stale: Vec, + /// Distinct findings that share a behavior or an observable path, reported together + /// and **never merged** (FR-031). Sorted by key, so the artifact is byte-stable. + pub groups: Vec, + /// The campaign history, in file order. + pub campaigns: Vec, + /// Signatures the admission cap suppressed across every campaign (FR-034b). + /// + /// Surfaced as a queue-level total, not only per campaign, because that is the number + /// that tells a reviewer the queue is a *sample*: fifty untriaged findings beside a + /// suppression count of zero means "this is everything", and beside a count of three + /// hundred it means "this is what we could look at". A silent truncation would render + /// both identically. + pub signatures_suppressed: u64, +} + +/// Build the report with no behavior mapping — grouping falls back to the observable path. +/// +/// Pure: no I/O, no clock, no environment. +pub fn build_queue_report(data: &DiscoveryData, pins: &CurrentPins) -> QueueReport { + build_queue_report_with_behaviors(data, pins, &BehaviorIndex::default()) +} + +/// Build the report, resolving promoted findings to the behaviors their cases claim +/// (**T073**). +/// +/// Pure: no I/O, no clock, no environment. +pub fn build_queue_report_with_behaviors( + data: &DiscoveryData, + pins: &CurrentPins, + behaviors: &BehaviorIndex, +) -> QueueReport { + let mut report = QueueReport { + total: data.findings.len(), + campaigns: data.campaigns.iter().map(CampaignSummary::of).collect(), + signatures_suppressed: data + .campaigns + .iter() + .map(|c| c.outcome.signatures_suppressed) + .sum(), + ..QueueReport::default() + }; + + // Keyed collection for the FR-031 grouping view. A finding contributes to a behavior + // group for EVERY behavior its case claims — a case covering two behaviors relates its + // finding to both — and to its observable-path group regardless, so the relationship is + // visible before anything is promoted as well as after. + let mut behavior_groups: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + let mut path_groups: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + + for finding in &data.findings { + let summary = FindingSummary::of(finding); + match finding.state { + FindingState::Untriaged => report.untriaged.push(summary.clone()), + FindingState::Triaged => report.triaged.push(summary.clone()), + FindingState::Split => report.split.push(summary.clone()), + FindingState::Promoted => report.promoted.push(summary.clone()), + FindingState::NoLongerReproducing => report.no_longer_reproducing.push(summary.clone()), + } + // Pin staleness is judged against the campaign that LAST observed the finding: + // that is the most recent evidence, and it is the run a re-evaluation would be + // compared against. An unresolvable reference is D1's problem, not the + // report's, so a dangling id simply does not mark the finding stale. + if data + .campaign(&finding.last_observed) + .is_some_and(|c| pins.differs_from(c)) + { + report.pin_stale.push(summary); + } + + for behavior in behaviors.behaviors_of(finding) { + behavior_groups + .entry(behavior.clone()) + .or_default() + .push(finding.id.clone()); + } + path_groups + .entry(format!( + "{} {}", + finding.signature.channel, finding.signature.path + )) + .or_default() + .push(finding.id.clone()); + } + + // Only groups of two or more: a group of one is the finding itself. + for (kind, source) in [ + (GroupKind::Behavior, behavior_groups), + (GroupKind::ObservablePath, path_groups), + ] { + for (key, findings) in source { + if findings.len() >= 2 { + report.groups.push(FindingGroup { + key, + kind, + findings, + }); + } + } + } + + report +} + +/// Render the canonical, byte-stable JSON document (2-space, trailing newline). +pub fn render_json(report: &QueueReport) -> String { + let mut out = serde_json::to_string_pretty(report) + .unwrap_or_else(|e| unreachable!("queue-report serialization is infallible: {e}")); + out.push('\n'); + out +} + +/// Render the human-review Markdown document. +/// +/// Deterministic by construction: every list is emitted in the queue's own file order and +/// nothing here reads a clock, an environment variable, or an absolute path. +pub fn render_md(report: &QueueReport) -> String { + use std::fmt::Write as _; + let mut out = String::new(); + + let _ = writeln!(out, "# Discovery findings queue\n"); + let _ = writeln!( + out, + "Findings are **candidates for assertions**, never assertions. Nothing here \ + reaches `certify`, and this report never gates.\n" + ); + + let _ = writeln!(out, "## Summary\n"); + let _ = writeln!(out, "| Bucket | Count |"); + let _ = writeln!(out, "|---|---|"); + let _ = writeln!(out, "| total | {} |", report.total); + let _ = writeln!(out, "| untriaged | {} |", report.untriaged.len()); + let _ = writeln!(out, "| triaged | {} |", report.triaged.len()); + let _ = writeln!(out, "| split | {} |", report.split.len()); + let _ = writeln!(out, "| promoted | {} |", report.promoted.len()); + let _ = writeln!( + out, + "| no-longer-reproducing | {} |", + report.no_longer_reproducing.len() + ); + let _ = writeln!(out, "| pin-stale | {} |", report.pin_stale.len()); + let _ = writeln!(out, "| campaigns | {} |", report.campaigns.len()); + let _ = writeln!( + out, + "| signatures suppressed | {} |\n", + report.signatures_suppressed + ); + + if report.signatures_suppressed > 0 { + let _ = writeln!( + out, + "**This queue is a sample.** {} distinct signature(s) were observed and not \ + admitted, because their campaigns reached the admission cap. Read every count \ + above against that number: suppression is reported precisely so \"we found 25 \ + things\" can never be mistaken for \"we found many more than we can review\", \ + and a campaign that keeps hitting its cap is itself a signal that something \ + systemic is diverging (FR-034b).\n", + report.signatures_suppressed + ); + } + + if report.total == 0 { + let _ = writeln!( + out, + "The queue is empty. Note what that does **not** say: it does not say the two \ + implementations agree. Check the campaign table below — a queue that is empty \ + because nothing ran looks exactly like a queue that is empty because nothing \ + differed.\n" + ); + } + + for (title, findings, note) in [ + ( + "Untriaged", + &report.untriaged, + "Nobody has looked yet. This bucket is counted so it can never read as \ + \"nothing found\".", + ), + ( + "Triaged", + &report.triaged, + "Classified, awaiting a decision.", + ), + ( + "Split", + &report.split, + "Inert ancestors: they keep their witnesses as historical record, accept no \ + new ones, and surrender classification to their children.", + ), + ( + "Promoted", + &report.promoted, + "Carried by a real registry case, named below.", + ), + ( + "No longer reproducing", + &report.no_longer_reproducing, + "Retained, not deleted: the disappearance may mean a fix landed, or it may \ + mean the generator stopped reaching the input, and only the retained record \ + tells those apart.", + ), + ( + "Pin-stale", + &report.pin_stale, + "Observed under pins that no longer match. Re-evaluated by the next campaign, \ + never carried forward unverified. Cross-cutting: each also appears in its own \ + state bucket above.", + ), + ] { + let _ = writeln!(out, "## {title} ({})\n", findings.len()); + let _ = writeln!(out, "{note}\n"); + if findings.is_empty() { + let _ = writeln!(out, "_None._\n"); + continue; + } + let _ = writeln!( + out, + "| finding | channel | path | kind | shape | witnesses | classification | last observed | promoted to |" + ); + let _ = writeln!(out, "|---|---|---|---|---|---|---|---|---|"); + for f in findings { + let _ = writeln!( + out, + "| `{}` | {} | `{}` | {} | {} | {} | {} | `{}` | {} |", + f.id, + f.channel, + f.path, + f.kind, + f.value_shape_class, + f.witnesses, + f.classification.as_deref().unwrap_or("—"), + f.last_observed, + f.promoted_to + .as_deref() + .map(|c| format!("`{c}`")) + .unwrap_or_else(|| "—".to_string()), + ); + } + let _ = writeln!(out); + } + + let _ = writeln!(out, "## Related findings ({})\n", report.groups.len()); + let _ = writeln!( + out, + "Findings that share a behavior or an observable path. **Grouping is a view, never \ + a merge** (FR-031): each finding below keeps its own witnesses, its own \ + classification, and its own promotion, because merging them would destroy the \ + ability to tell whether a fix addressed one cause or all of them.\n" + ); + let _ = writeln!( + out, + "A `behavior` key is a *reviewed* mapping — it exists only because a human promoted \ + a finding into a case naming that behavior. An `observable-path` key is not a \ + behavior claim; it is the strongest relationship available before anyone has \ + decided what these differences mean.\n" + ); + let _ = writeln!( + out, + "One shape to read carefully: a **split lineage** appears as a group whose members \ + share a single signature, because a split separates witnesses rather than \ + signatures. That is the lineage, not several distinct causes — the causes are what \ + the reviewer separated them to record, one classification per child.\n" + ); + if report.groups.is_empty() { + let _ = writeln!(out, "_None._\n"); + } else { + let _ = writeln!(out, "| key | kind | findings |"); + let _ = writeln!(out, "|---|---|---|"); + for group in &report.groups { + let _ = writeln!( + out, + "| `{}` | {} | {} |", + group.key, + match group.kind { + GroupKind::Behavior => "behavior", + GroupKind::ObservablePath => "observable-path", + }, + group + .findings + .iter() + .map(|f| format!("`{f}`")) + .collect::>() + .join(", ") + ); + } + let _ = writeln!(out); + } + + let _ = writeln!(out, "## Campaigns ({})\n", report.campaigns.len()); + let _ = writeln!( + out, + "Volume is reported whether or not anything was found (FR-062): a campaign that \ + found nothing and a campaign that never ran are different facts.\n" + ); + if report.campaigns.is_empty() { + let _ = writeln!(out, "_None._\n"); + } else { + let _ = writeln!( + out, + "| campaign | seed | tier | lane | generated | executed | admitted | suppressed | budget exhausted |" + ); + let _ = writeln!(out, "|---|---|---|---|---|---|---|---|---|"); + for c in &report.campaigns { + let _ = writeln!( + out, + "| `{}` | `{}` | {} | {} | {} | {} | {} | {} | {} |", + c.id, + c.seed, + c.tier, + c.lane, + c.candidates_generated, + c.candidates_executed, + c.signatures_admitted, + c.signatures_suppressed, + c.budget_exhausted, + ); + } + let _ = writeln!(out); + } + + out +} + +/// Atomically write `queue.json` + `queue.md` into `dir`, returning the written paths in +/// a deterministic order. +pub fn write_queue_report(dir: &Path, report: &QueueReport) -> std::io::Result> { + let json = dir.join("queue.json"); + let md = dir.join("queue.md"); + crate::atomic_write(&json, &render_json(report))?; + crate::atomic_write(&md, &render_md(report))?; + Ok(vec![json, md]) +} + +// --------------------------------------------------------------------------- +// T039 / T123 — the campaign-outcome report +// --------------------------------------------------------------------------- + +/// One campaign's own report (FR-061), rendered from its recorded outcome. +/// +/// Distinct from [`QueueReport`], which is the standing triage queue across all campaigns. +/// This is what a single run emits — the document a reviewer reads to answer "what did +/// last night's campaign actually do?". +/// +/// **Every field here is reported whether or not the campaign found anything** (FR-062). +/// A campaign that found nothing and a campaign that never ran are completely different +/// facts, and without the volume they are indistinguishable from the outside — which would +/// make "no findings" the most comfortable possible way for the machinery to be broken. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CampaignOutcomeReport { + /// `cmp-`. + pub campaign: String, + /// The recorded seed — the reproducibility input (FR-001). + pub seed: String, + /// The tier's wire spelling. + pub tier: String, + /// The lane's wire spelling. + pub lane: String, + /// The certification profile the run happened under. + pub profile: String, + /// All seven pinned inputs, verbatim (FR-002). + pub pinned_input_set: PinnedInputSet, + /// The declared budget. + pub budget: Budget, + /// Candidates the generator produced. + pub candidates_generated: u64, + /// Candidates actually executed against an implementation. + pub candidates_executed: u64, + /// Candidates discarded as unsafe before execution (FR-011). + pub candidates_discarded_unsafe: u64, + /// Candidates that failed at document parsing. + pub parse_stage_failures: u64, + /// `parseStageFailures / candidatesGenerated` — the SC-002 ratio, reported for **every** + /// run rather than only when it breaches, so a rising trend is visible before it does. + /// + /// Zero when nothing was generated: a campaign that produced no candidates failed no + /// candidates either, and reporting a ratio of "not a number" would serialize as bare + /// `null` and never load back. + pub trivial_failure_fraction: f64, + /// The declared SC-002 ceiling this run is judged against. + pub trivial_failure_ceiling: f64, + /// Whether the run breached the ceiling. **Reported, never gating** — the exit status + /// of a discovery command reflects whether it ran, never what it found. + pub trivial_failure_ceiling_breached: bool, + /// Whether the wall-clock budget ran out (FR-005). + pub budget_exhausted: bool, + /// The fraction of the planned space covered, reported when the budget was exhausted + /// (FR-005) so a truncated run is never presented as complete. + pub space_covered_fraction: f64, + /// Applications per mutation category — **all eleven keys, always** (FR-010). + pub mutation_applications: IndexMap, + /// Categories with zero successful applications, named as an explicit generation + /// deficiency (FR-010, SC-003) rather than left to be inferred from a map a reader + /// would have to cross-check against the catalogue. + pub unapplied_categories: Vec, + /// Distinct signatures observed. + pub signatures_observed: u64, + /// Distinct signatures admitted to the queue. + pub signatures_admitted: u64, + /// Distinct signatures the admission cap suppressed (FR-034b) — never silent. + pub signatures_suppressed: u64, + /// The findings this campaign admitted or re-witnessed, in admission order. + pub findings: Vec, +} + +/// The SC-002 ceiling: at most 10% of generated candidates may fail at the +/// document-syntax stage. Above it, the campaign explored the parser rather than the tool. +pub const TRIVIAL_FAILURE_CEILING: f64 = 0.10; + +/// Re-key an application map onto the catalogue's eleven declared categories. +/// +/// Starts from [`mutate::empty_application_counts`] — the single source of the key list — +/// so the result carries every category whatever the caller recorded, and a category the +/// caller invented is dropped rather than silently widening the catalogue. FR-010 needs +/// the keys present to distinguish "never applied" from "never mentioned"; it does not +/// need a twelfth key nothing declares. +pub fn normalized_mutation_applications(recorded: &IndexMap) -> IndexMap { + let mut counts = mutate::empty_application_counts(); + for (category, count) in recorded { + if let Some(slot) = counts.get_mut(category) { + *slot = *count; + } + } + counts +} + +/// Build a campaign's own report. Pure: no I/O, no clock, no environment. +pub fn build_campaign_outcome_report( + campaign: &Campaign, + findings: &[String], +) -> CampaignOutcomeReport { + let outcome = &campaign.outcome; + let mutation_applications = normalized_mutation_applications(&outcome.mutation_applications); + let unapplied: Vec = mutate::unapplied_categories(&mutation_applications) + .into_iter() + .map(str::to_string) + .collect(); + // A ratio over a zero denominator is `NaN`, which `serde_json` renders as bare `null` + // and cannot read back — the same hazard `write_campaigns` refuses for + // `spaceCoveredFraction`. Zero candidates means zero trivial failures, so zero is the + // honest answer rather than a fallback. + let trivial_failure_fraction = if outcome.candidates_generated == 0 { + 0.0 + } else { + outcome.parse_stage_failures as f64 / outcome.candidates_generated as f64 + }; + + CampaignOutcomeReport { + campaign: campaign.id.clone(), + seed: campaign.seed.clone(), + tier: campaign.tier.as_str().to_string(), + lane: campaign.lane.as_str().to_string(), + profile: campaign.profile.clone(), + pinned_input_set: campaign.pinned_input_set.clone(), + budget: campaign.budget, + candidates_generated: outcome.candidates_generated, + candidates_executed: outcome.candidates_executed, + candidates_discarded_unsafe: outcome.candidates_discarded_unsafe, + parse_stage_failures: outcome.parse_stage_failures, + trivial_failure_fraction, + trivial_failure_ceiling: TRIVIAL_FAILURE_CEILING, + trivial_failure_ceiling_breached: trivial_failure_fraction > TRIVIAL_FAILURE_CEILING, + budget_exhausted: outcome.budget_exhausted, + space_covered_fraction: outcome.space_covered_fraction, + mutation_applications, + unapplied_categories: unapplied, + signatures_observed: outcome.signatures_observed, + signatures_admitted: outcome.signatures_admitted, + signatures_suppressed: outcome.signatures_suppressed, + findings: findings.to_vec(), + } +} + +/// Render the campaign report as the byte-stable JSON document the bin prints on stdout. +pub fn render_campaign_json(report: &CampaignOutcomeReport) -> String { + let mut out = serde_json::to_string_pretty(report) + .unwrap_or_else(|e| unreachable!("campaign-report serialization is infallible: {e}")); + out.push('\n'); + out +} + +/// Render the campaign report for human review. +pub fn render_campaign_md(report: &CampaignOutcomeReport) -> String { + use std::fmt::Write as _; + let mut out = String::new(); + + let _ = writeln!(out, "# Discovery campaign `{}`\n", report.campaign); + let _ = writeln!( + out, + "Findings are **candidates for assertions**, never assertions. This report never \ + gates: its existence says the campaign ran, and its contents say what it saw.\n" + ); + + let _ = writeln!(out, "## Reproducing this run\n"); + let _ = writeln!(out, "| Input | Value |"); + let _ = writeln!(out, "|---|---|"); + let _ = writeln!(out, "| seed | `{}` |", report.seed); + let _ = writeln!(out, "| tier | {} |", report.tier); + let _ = writeln!(out, "| lane | {} |", report.lane); + let _ = writeln!(out, "| profile | `{}` |", report.profile); + let pins = &report.pinned_input_set; + for (name, value) in [ + ("schemaPin", &pins.schema_pin), + ("prosePin", &pins.prose_pin), + ("oracleVersion", &pins.oracle_version), + ("normalizerVersion", &pins.normalizer_version), + ("grammarVersion", &pins.grammar_version), + ("mutationCatalogVersion", &pins.mutation_catalog_version), + ("generatorVersion", &pins.generator_version), + ] { + let _ = writeln!(out, "| {name} | `{value}` |"); + } + let _ = writeln!(out); + + let _ = writeln!(out, "## Volume\n"); + let _ = writeln!( + out, + "Reported whether or not anything was found (FR-062): a campaign that found \ + nothing and a campaign that never ran are different facts.\n" + ); + let _ = writeln!(out, "| Measure | Value |"); + let _ = writeln!(out, "|---|---|"); + let _ = writeln!( + out, + "| candidatesGenerated | {} |", + report.candidates_generated + ); + let _ = writeln!( + out, + "| candidatesExecuted | {} |", + report.candidates_executed + ); + let _ = writeln!( + out, + "| candidatesDiscardedUnsafe | {} |", + report.candidates_discarded_unsafe + ); + let _ = writeln!( + out, + "| parseStageFailures | {} |", + report.parse_stage_failures + ); + let _ = writeln!( + out, + "| trivialFailureFraction | {:.4} (ceiling {:.2}{}) |", + report.trivial_failure_fraction, + report.trivial_failure_ceiling, + if report.trivial_failure_ceiling_breached { + ", **BREACHED**" + } else { + "" + } + ); + let _ = writeln!(out, "| budgetExhausted | {} |", report.budget_exhausted); + let _ = writeln!( + out, + "| spaceCoveredFraction | {:.4} |", + report.space_covered_fraction + ); + let _ = writeln!( + out, + "| signaturesObserved | {} |", + report.signatures_observed + ); + let _ = writeln!( + out, + "| signaturesAdmitted | {} |", + report.signatures_admitted + ); + let _ = writeln!( + out, + "| signaturesSuppressed | {} |\n", + report.signatures_suppressed + ); + + let _ = writeln!(out, "## Mutation applications\n"); + let _ = writeln!( + out, + "All {} declared categories are listed, including zeroes: a category absent from \ + the table would be indistinguishable from one that was never applied (FR-010).\n", + report.mutation_applications.len() + ); + let _ = writeln!(out, "| Category | Applications |"); + let _ = writeln!(out, "|---|---|"); + for (category, count) in &report.mutation_applications { + let _ = writeln!(out, "| `{category}` | {count} |"); + } + let _ = writeln!(out); + if report.unapplied_categories.is_empty() { + let _ = writeln!(out, "Every declared category was applied at least once.\n"); + } else { + let _ = writeln!( + out, + "**Generation deficiency**: {} categor{} never applied — {}. A category with \ + zero successful applications is a hole in what this campaign explored, not a \ + detail of its bookkeeping.\n", + report.unapplied_categories.len(), + if report.unapplied_categories.len() == 1 { + "y" + } else { + "ies" + }, + report + .unapplied_categories + .iter() + .map(|c| format!("`{c}`")) + .collect::>() + .join(", ") + ); + } + + let _ = writeln!(out, "## Findings ({})\n", report.findings.len()); + if report.findings.is_empty() { + let _ = writeln!( + out, + "This campaign admitted no findings. Read that against the volume above: it \ + says the two implementations agreed on everything this run reached, not that \ + nothing ran.\n" + ); + } else { + for finding in &report.findings { + let _ = writeln!(out, "- `{finding}`"); + } + let _ = writeln!(out); + } + + out +} + +/// Atomically write `campaign-.{json,md}` into `dir`, returning the written paths in a +/// deterministic order. +pub fn write_campaign_outcome_report( + dir: &Path, + report: &CampaignOutcomeReport, +) -> std::io::Result> { + let json = dir.join(format!("campaign-{}.json", report.campaign)); + let md = dir.join(format!("campaign-{}.md", report.campaign)); + crate::atomic_write(&json, &render_campaign_json(report))?; + crate::atomic_write(&md, &render_campaign_md(report))?; + Ok(vec![json, md]) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::discovery::queue::{ + Budget, Campaign, CampaignLane, CampaignOutcome, CampaignTier, Classification, Finding, + ObservedValues, PinnedInputSet, Witness, + }; + use crate::discovery::signature::{Divergence, DivergenceKind, Signature}; + use indexmap::IndexMap; + use serde_json::json; + + fn pins() -> CurrentPins { + CurrentPins { + schema_pin: "113500f4".into(), + prose_pin: "113500f4".into(), + oracle_version: Some("0.87.0".into()), + } + } + + fn campaign(id: &str, oracle: &str) -> Campaign { + Campaign { + id: id.into(), + seed: "0x5eed1234".into(), + lane: CampaignLane::Scheduled, + tier: CampaignTier::ConfigDifferential, + // The report never validates identity — `check` owns D1 — so these fixtures + // keep readable ids rather than derived ones. A report that refused to render + // a structurally invalid queue would hide exactly the queue a reviewer most + // needs to see. + profile: "prof-linux-amd64-docker-0870".into(), + pinned_input_set: PinnedInputSet { + schema_pin: "113500f4".into(), + prose_pin: "113500f4".into(), + oracle_version: oracle.into(), + normalizer_version: "6".into(), + grammar_version: "rev-schema-113500f4".into(), + mutation_catalog_version: "v1".into(), + generator_version: "splitmix64-seed+xoshiro256starstar/v1".into(), + }, + budget: Budget { + wall_clock_seconds: 1800, + per_candidate_seconds: 60, + shrink_steps_per_finding: 64, + admission_cap: 25, + }, + outcome: CampaignOutcome { + candidates_generated: 4820, + candidates_executed: 4629, + candidates_discarded_unsafe: 0, + parse_stage_failures: 191, + budget_exhausted: false, + space_covered_fraction: 0.0, + mutation_applications: IndexMap::new(), + signatures_observed: 2, + signatures_admitted: 2, + signatures_suppressed: 0, + }, + } + } + + fn finding(path: &str, state: FindingState, campaign_id: &str) -> Finding { + let d = json!("vscode"); + let r = json!("root"); + let sig = Signature::derive( + "chan-structured-output", + &Divergence { + kind: DivergenceKind::Value, + path, + deacon: Some(&d), + reference: Some(&r), + }, + ); + let witness = Witness { + id: Witness::derived_id(campaign_id, "cnd-1"), + campaign_id: campaign_id.into(), + candidate_id: "cnd-1".into(), + minimal_input: json!({ "image": "alpine:3.18" }), + is_minimal: true, + reduction_steps: Vec::new(), + observed_values: ObservedValues::default(), + mutation_operators: Vec::new(), + }; + let mut f = Finding::newly_admitted(sig, witness, campaign_id); + f.state = state; + if state != FindingState::Untriaged { + f.classification = Some(Classification::DeaconRegression); + } + f + } + + #[test] + fn an_empty_queue_reports_zeroes_and_says_what_that_does_not_mean() { + let report = build_queue_report(&DiscoveryData::default(), &pins()); + assert_eq!(report.total, 0); + assert!(report.untriaged.is_empty()); + assert!(report.campaigns.is_empty()); + + let md = render_md(&report); + assert!(md.contains("| untriaged | 0 |")); + assert!( + md.contains("nothing ran"), + "an empty queue must not read as agreement between the implementations" + ); + + let json = render_json(&report); + let back: QueueReport = serde_json::from_str(&json).expect("round-trips"); + assert_eq!(back, report); + } + + #[test] + fn findings_land_in_their_state_buckets() { + let data = DiscoveryData { + findings: vec![ + finding("a", FindingState::Untriaged, "cmp-1"), + finding("b", FindingState::Triaged, "cmp-1"), + finding("c", FindingState::Split, "cmp-1"), + finding("d", FindingState::Promoted, "cmp-1"), + finding("e", FindingState::NoLongerReproducing, "cmp-1"), + ], + campaigns: vec![campaign("cmp-1", "0.87.0")], + corpus: Vec::new(), + }; + let report = build_queue_report(&data, &pins()); + assert_eq!(report.total, 5); + assert_eq!(report.untriaged.len(), 1); + assert_eq!(report.triaged.len(), 1); + assert_eq!(report.split.len(), 1); + assert_eq!(report.promoted.len(), 1); + assert_eq!(report.no_longer_reproducing.len(), 1); + assert!(report.pin_stale.is_empty()); + + let md = render_md(&report); + assert!(md.contains("| untriaged | 1 |")); + assert!(md.contains("Nobody has looked yet")); + } + + #[test] + fn a_finding_observed_under_old_pins_is_pin_stale_but_keeps_its_state() { + let data = DiscoveryData { + findings: vec![finding("a", FindingState::Triaged, "cmp-old")], + campaigns: vec![campaign("cmp-old", "0.86.0")], + corpus: Vec::new(), + }; + let report = build_queue_report(&data, &pins()); + assert_eq!(report.pin_stale.len(), 1); + assert_eq!( + report.triaged.len(), + 1, + "pin staleness is cross-cutting: a pin moving must not discard a reviewer's \ + judgement" + ); + } + + #[test] + fn an_undecidable_oracle_pin_does_not_mark_the_whole_queue_stale() { + // A registry that has not recorded an oracle revision means "we cannot tell", + // which must never render as "everything differs" — that would read as a + // catastrophic pin bump instead of as missing metadata. + let undecidable = CurrentPins { + oracle_version: None, + ..pins() + }; + let data = DiscoveryData { + findings: vec![finding("a", FindingState::Triaged, "cmp-1")], + campaigns: vec![campaign("cmp-1", "0.87.0")], + corpus: Vec::new(), + }; + assert!(build_queue_report(&data, &undecidable).pin_stale.is_empty()); + + // The other two elements are still compared, so a real schema-pin bump is caught + // even while the oracle pin is undecidable. + let bumped = CurrentPins { + schema_pin: "deadbeef".into(), + ..undecidable + }; + assert_eq!(build_queue_report(&data, &bumped).pin_stale.len(), 1); + } + + #[test] + fn campaign_volume_is_reported_even_with_no_findings() { + let data = DiscoveryData { + findings: Vec::new(), + campaigns: vec![campaign("cmp-1", "0.87.0")], + corpus: Vec::new(), + }; + let report = build_queue_report(&data, &pins()); + assert_eq!(report.total, 0); + assert_eq!(report.campaigns[0].candidates_generated, 4820); + let md = render_md(&report); + assert!( + md.contains("4820"), + "nothing-found must not read as nothing-ran" + ); + } + + #[test] + fn the_rendering_is_byte_stable_and_writes_atomically() { + let data = DiscoveryData { + findings: vec![finding("a", FindingState::Untriaged, "cmp-1")], + campaigns: vec![campaign("cmp-1", "0.87.0")], + corpus: Vec::new(), + }; + let report = build_queue_report(&data, &pins()); + assert_eq!(render_json(&report), render_json(&report)); + assert_eq!(render_md(&report), render_md(&report)); + + let dir = tempfile::tempdir().expect("tempdir"); + let written = write_queue_report(dir.path(), &report).expect("writes"); + assert_eq!(written.len(), 2); + assert!(written[0].ends_with("queue.json")); + assert!(written[1].ends_with("queue.md")); + let json = std::fs::read_to_string(&written[0]).expect("read"); + assert_eq!(json, render_json(&report)); + assert!(json.ends_with("}\n"), "trailing newline for a clean diff"); + } + + // --- T039 / T123: the campaign-outcome report ------------------------- + + #[test] + fn the_campaign_report_always_carries_all_eleven_mutation_keys() { + // FR-010: a category absent from the map is indistinguishable from a category + // that was never applied. A campaign that recorded only the categories that fired + // is exactly the shape this normalization exists to repair. + let mut c = campaign("cmp-1", "0.87.0"); + c.outcome.mutation_applications = IndexMap::from([("wrong-type".to_string(), 12u64)]); + + let report = build_campaign_outcome_report(&c, &[]); + assert_eq!(report.mutation_applications.len(), 11); + let keys: Vec<&str> = report + .mutation_applications + .keys() + .map(String::as_str) + .collect(); + assert_eq!( + keys, + crate::discovery::mutate::category_names(), + "the key list is the catalogue's, in declaration order — never restated here" + ); + assert_eq!(report.mutation_applications["wrong-type"], 12); + assert_eq!(report.mutation_applications["ordering-change"], 0); + assert_eq!( + report.unapplied_categories.len(), + 10, + "every zero-count category is named as an explicit generation deficiency" + ); + assert!( + report + .unapplied_categories + .contains(&"null-value".to_string()) + ); + + let md = render_campaign_md(&report); + for category in crate::discovery::mutate::category_names() { + assert!( + md.contains(&format!("`{category}`")), + "{category} missing from {md}" + ); + } + assert!(md.contains("Generation deficiency")); + } + + #[test] + fn a_category_the_catalogue_does_not_declare_is_not_admitted_by_the_report() { + // The map's job is to make a missing category visible, not to widen the catalogue: + // an invented key would report a generation category nothing can ever apply. + let mut c = campaign("cmp-1", "0.87.0"); + c.outcome.mutation_applications = + IndexMap::from([("invented-category".to_string(), 99u64)]); + let report = build_campaign_outcome_report(&c, &[]); + assert_eq!(report.mutation_applications.len(), 11); + assert!( + !report + .mutation_applications + .contains_key("invented-category") + ); + assert_eq!(report.unapplied_categories.len(), 11); + } + + #[test] + fn a_full_catalogue_reports_no_deficiency() { + let mut c = campaign("cmp-1", "0.87.0"); + let mut counts = crate::discovery::mutate::empty_application_counts(); + for (i, name) in crate::discovery::mutate::category_names() + .iter() + .enumerate() + { + counts.insert((*name).to_string(), i as u64 + 1); + } + c.outcome.mutation_applications = counts; + let report = build_campaign_outcome_report(&c, &[]); + assert!(report.unapplied_categories.is_empty()); + assert!(render_campaign_md(&report).contains("Every declared category was applied")); + } + + #[test] + fn a_zero_finding_campaign_still_reports_the_volume_it_covered() { + // T123 / FR-062: "nothing found" must be distinguishable from "nothing ran". + // Without the volume, a broken pipeline is the most comfortable possible state for + // the machinery to be in — it reports the same thing as a clean run. + let c = campaign("cmp-1", "0.87.0"); + let report = build_campaign_outcome_report(&c, &[]); + assert!(report.findings.is_empty()); + assert_eq!(report.candidates_generated, 4820); + assert_eq!(report.candidates_executed, 4629); + + let json = render_campaign_json(&report); + assert!(json.contains("\"candidatesGenerated\": 4820")); + assert!(json.contains("\"candidatesExecuted\": 4629")); + let back: CampaignOutcomeReport = serde_json::from_str(&json).expect("round-trips"); + assert_eq!(back, report); + + let md = render_campaign_md(&report); + assert!(md.contains("4820")); + assert!(md.contains("4629")); + assert!( + md.contains("not that \nnothing ran") || md.contains("not that nothing ran"), + "the empty-findings note must say what the absence does NOT mean: {md}" + ); + } + + #[test] + fn a_campaign_that_ran_nothing_reports_zeroes_rather_than_a_not_a_number_ratio() { + // `NaN` serializes as bare `null` and never loads back — the same hazard + // `write_campaigns` refuses for `spaceCoveredFraction`. An aborted campaign is + // exactly the zero-denominator shape. + let mut c = campaign("cmp-1", "0.87.0"); + c.outcome.candidates_generated = 0; + c.outcome.candidates_executed = 0; + c.outcome.parse_stage_failures = 0; + + let report = build_campaign_outcome_report(&c, &[]); + assert_eq!(report.trivial_failure_fraction, 0.0); + assert!(report.trivial_failure_fraction.is_finite()); + assert!(!report.trivial_failure_ceiling_breached); + let json = render_campaign_json(&report); + let raw: serde_json::Value = serde_json::from_str(&json).expect("valid JSON"); + for field in ["trivialFailureFraction", "spaceCoveredFraction"] { + assert!( + raw[field].is_number(), + "`{field}` rendered as {} — a non-finite f64 serializes as bare `null` and \ + never loads back", + raw[field] + ); + } + serde_json::from_str::(&json).expect("loads back"); + } + + #[test] + fn the_trivial_failure_ratio_is_reported_for_every_run_and_never_gates() { + let mut c = campaign("cmp-1", "0.87.0"); + c.outcome.candidates_generated = 100; + c.outcome.parse_stage_failures = 4; + let ok = build_campaign_outcome_report(&c, &[]); + assert!((ok.trivial_failure_fraction - 0.04).abs() < 1e-9); + assert!(!ok.trivial_failure_ceiling_breached); + + c.outcome.parse_stage_failures = 40; + let breached = build_campaign_outcome_report(&c, &[]); + assert!(breached.trivial_failure_ceiling_breached); + assert!(render_campaign_md(&breached).contains("BREACHED")); + // Reporting the breach is the whole response: the report has no exit status of its + // own, and the bin's status reflects whether the campaign ran (FR-058). + assert_eq!(breached.trivial_failure_ceiling, TRIVIAL_FAILURE_CEILING); + } + + #[test] + fn the_campaign_report_is_byte_stable_and_writes_atomically() { + let c = campaign("cmp-1", "0.87.0"); + let report = build_campaign_outcome_report(&c, &["fnd-11111111".to_string()]); + assert_eq!(render_campaign_json(&report), render_campaign_json(&report)); + assert_eq!(render_campaign_md(&report), render_campaign_md(&report)); + + let dir = tempfile::tempdir().expect("tempdir"); + let written = write_campaign_outcome_report(dir.path(), &report).expect("writes"); + assert_eq!(written.len(), 2); + assert!(written[0].ends_with("campaign-cmp-1.json")); + assert!(written[1].ends_with("campaign-cmp-1.md")); + assert_eq!( + std::fs::read_to_string(&written[0]).expect("read"), + render_campaign_json(&report) + ); + assert!( + std::fs::read_to_string(&written[1]) + .expect("read") + .contains("fnd-11111111") + ); + } + + #[test] + fn the_campaign_report_states_every_pinned_input_and_the_seed() { + // FR-061: a run's report must state what would be needed to reproduce it. A report + // that omits one of the seven pins describes a run nobody can re-run. + let c = campaign("cmp-1", "0.87.0"); + let md = render_campaign_md(&build_campaign_outcome_report(&c, &[])); + for expected in [ + "0x5eed1234", + "schemaPin", + "prosePin", + "oracleVersion", + "normalizerVersion", + "grammarVersion", + "mutationCatalogVersion", + "generatorVersion", + "prof-linux-amd64-docker-0870", + ] { + assert!(md.contains(expected), "{expected} missing from the report"); + } + } + + #[test] + fn a_dangling_last_observed_does_not_mark_a_finding_stale() { + // An unresolvable campaign reference is D1's problem. The report must not turn a + // structural violation into a silently different classification. + let data = DiscoveryData { + findings: vec![finding("a", FindingState::Triaged, "cmp-ghost")], + campaigns: Vec::new(), + corpus: Vec::new(), + }; + let report = build_queue_report(&data, &pins()); + assert!(report.pin_stale.is_empty()); + assert_eq!(report.triaged.len(), 1); + } +} diff --git a/crates/conformance/src/discovery/rng.rs b/crates/conformance/src/discovery/rng.rs new file mode 100644 index 00000000..d9274122 --- /dev/null +++ b/crates/conformance/src/discovery/rng.rs @@ -0,0 +1,393 @@ +//! The in-repo deterministic pseudorandom stream +//! (025-exploratory-parity-discovery, research D2, T010/T011). +//! +//! SplitMix64 seed expansion feeding a xoshiro256\*\* stream — roughly forty lines of +//! wrapping integer arithmetic and rotations, no `unsafe`, no new dependency. +//! +//! ## Why this is in-repo rather than the `rand` crate +//! +//! FR-001 requires a recorded seed to reproduce an identical candidate sequence, and +//! FR-034 requires findings to persist indefinitely — so a seed recorded today must +//! still reproduce after arbitrary dependency updates. `rand` explicitly does **not** +//! offer value-stream stability across versions; its own documentation treats the stream +//! as an implementation detail. Depending on it would make every finding's +//! reproducibility hostage to a `cargo update`, and a security advisory could force a +//! bump that silently invalidates the entire recorded corpus with no signal. +//! +//! Making the stream a property of *our* committed code inverts that: the algorithm +//! identity becomes an explicit component of `generatorVersion` (the seventh element of +//! the pinned input set, data-model.md § 4), so a deliberate change is a recorded, +//! reviewable pin change — exactly like `NORMALIZER_VERSION`, and for the identical +//! reason. +//! +//! ## What makes the stream a *pin* rather than an accident +//! +//! The unit tests below check the stream against **published reference vectors** for +//! both stages. Without them, "the stream is stable" would be an assertion about code +//! nobody compared to anything; with them, an accidental edit to a shift constant fails +//! a named test instead of silently re-rolling every campaign ever recorded. +//! +//! ## What counts as a change to `generatorVersion` +//! +//! Everything in this module that can move a draw: the two stages' constants, the order +//! in which [`Prng::from_seed`] expands the seed into state, and the derivation of every +//! helper below ([`Prng::next_bounded`], [`Prng::choose`], [`Prng::shuffle`], …). A +//! "harmless refactor" of `next_bounded`'s rejection loop changes which candidates a +//! seed produces, so it is a pin change and must bump [`PRNG_VERSION`]. + +/// The algorithm identity of the stream — one half of the `generatorVersion` element of +/// the pinned input set (data-model.md § 4). The other half is the reduction +/// catalogue's *order*, which lives with the shrinker (data-model.md § 6). +/// +/// Both are reproducibility-critical — FR-001 depends on the stream, FR-020 on the +/// order — and folding either into `mutationCatalogVersion` would name it for something +/// it is not, so a deliberate change to reduction order would look like a change to the +/// mutation operators. +pub const PRNG_ALGORITHM: &str = "splitmix64-seed+xoshiro256starstar"; + +/// The revision of *this module's* derivation, bumped whenever any draw could move. +/// +/// Distinct from [`PRNG_ALGORITHM`] because the algorithm can stay xoshiro256\*\* while +/// a helper's derivation changes: both determine the candidate sequence, only one is +/// named by the algorithm string. +pub const PRNG_VERSION: u32 = 1; + +/// The composed algorithm identity recorded in a campaign's `generatorVersion`. +pub fn prng_identity() -> String { + format!("{PRNG_ALGORITHM}/v{PRNG_VERSION}") +} + +/// SplitMix64 — the seed expander (Steele, Lea & Flood 2014). +/// +/// Used **only** to turn a single 64-bit seed into xoshiro256\*\*'s 256-bit state, which +/// is the expansion the xoshiro reference implementation itself recommends: a +/// poorly-distributed state (all-zero, or nearly so) makes xoshiro's output correlated +/// for a long prefix, and a user-chosen seed like `0x5eed1234` is exactly that. +/// +/// The expansion also makes an all-zero state — xoshiro's one fixed point, from which it +/// never escapes — unreachable: SplitMix64 is a bijection on 64 bits, so it maps exactly +/// one input to zero, and four *consecutive* counter values can therefore never all map +/// to zero. No runtime guard is needed, and adding one would be a divergence from the +/// reference for a state that cannot occur. +#[derive(Debug, Clone)] +struct SplitMix64 { + state: u64, +} + +impl SplitMix64 { + fn new(seed: u64) -> Self { + SplitMix64 { state: seed } + } + + fn next(&mut self) -> u64 { + self.state = self.state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } +} + +/// Rotate-left, the one primitive both stages need. +#[inline] +fn rotl(x: u64, k: u32) -> u64 { + x.rotate_left(k) +} + +/// The campaign pseudorandom stream: xoshiro256\*\* over a SplitMix64-expanded seed. +/// +/// Deterministic and self-contained. Two `Prng`s built from the same seed produce the +/// same sequence forever, on every platform and every toolchain, because nothing here +/// depends on pointer width, endianness, floating point, or hash-map iteration order. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Prng { + s: [u64; 4], +} + +impl Prng { + /// Expand a 64-bit seed into the 256-bit state via four SplitMix64 draws. + /// + /// The expansion order (`s[0]`, `s[1]`, `s[2]`, `s[3]`, in that order) is part of + /// the pin: permuting it would produce a different, equally valid stream, and every + /// finding recorded under the old order would silently stop reproducing. + pub fn from_seed(seed: u64) -> Self { + let mut sm = SplitMix64::new(seed); + Prng { + s: [sm.next(), sm.next(), sm.next(), sm.next()], + } + } + + /// Construct directly from a raw 256-bit state. + /// + /// Exists so the reference vectors can be checked against the canonical published + /// state `[1, 2, 3, 4]` — the only way to compare this implementation to the + /// upstream test data, since that state is not reachable from any seed. + /// + /// An all-zero state is xoshiro's fixed point (it emits zeros forever), so it is + /// rejected rather than accepted as a silently dead generator. + pub fn from_state(s: [u64; 4]) -> Option { + if s == [0; 4] { None } else { Some(Prng { s }) } + } + + /// The next 64-bit draw (xoshiro256\*\*, Blackman & Vigna). + pub fn next_u64(&mut self) -> u64 { + let result = rotl(self.s[1].wrapping_mul(5), 7).wrapping_mul(9); + let t = self.s[1] << 17; + + self.s[2] ^= self.s[0]; + self.s[3] ^= self.s[1]; + self.s[1] ^= self.s[2]; + self.s[0] ^= self.s[3]; + self.s[2] ^= t; + self.s[3] = rotl(self.s[3], 45); + + result + } + + /// A uniform draw in `0..bound`, or `None` when `bound == 0` (an empty range has no + /// member to return, and returning `0` would be a silent fallback). + /// + /// Uses Lemire's multiply-shift with the exact rejection threshold, so the result is + /// **unbiased** — not merely "close enough". A modulo reduction would over-represent + /// the low residues, which for a generator drawing from a constraint grammar means + /// systematically over-exploring whichever alternatives happen to be declared first + /// and under-exploring the rest. That is a coverage defect disguised as a rounding + /// detail. + /// + /// The rejection loop terminates with probability 1 and, for every `bound`, expects + /// well under two draws; it is not an unbounded search. + pub fn next_bounded(&mut self, bound: u64) -> Option { + if bound == 0 { + return None; + } + let mut x = self.next_u64(); + let mut m = (x as u128).wrapping_mul(bound as u128); + let mut low = m as u64; + if low < bound { + // Reject the short tail so every residue is equally likely. + let threshold = bound.wrapping_neg() % bound; + while low < threshold { + x = self.next_u64(); + m = (x as u128).wrapping_mul(bound as u128); + low = m as u64; + } + } + Some((m >> 64) as u64) + } + + /// A uniform index into a slice of `len` elements, or `None` when it is empty. + pub fn next_index(&mut self, len: usize) -> Option { + self.next_bounded(len as u64).map(|v| v as usize) + } + + /// A uniform boolean (the stream's top bit — the best-distributed one for + /// xoshiro\*\*, whose lowest bits are the weakest). + pub fn next_bool(&mut self) -> bool { + (self.next_u64() >> 63) != 0 + } + + /// A uniformly chosen element of `items`, or `None` when it is empty. + pub fn choose<'a, T>(&mut self, items: &'a [T]) -> Option<&'a T> { + self.next_index(items.len()).map(|i| &items[i]) + } + + /// Fisher–Yates shuffle in place, iterating **downward** from the last index. + /// + /// The direction is part of the pin: the upward variant consumes the same draws in a + /// different order and produces a different permutation from the same seed. + pub fn shuffle(&mut self, items: &mut [T]) { + if items.len() < 2 { + return; + } + for i in (1..items.len()).rev() { + // `i >= 1`, so the bound is at least 2 and `next_bounded` always yields. + // Written as a `let ... else` rather than an `expect` so no runtime path can + // panic (constitution V) — a shuffle that somehow could not draw leaves the + // prefix unpermuted rather than aborting a campaign mid-run. + let Some(j) = self.next_bounded((i + 1) as u64) else { + return; + }; + items.swap(i, j as usize); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The published SplitMix64 output sequence for seed `0`. These are the values every + /// reference port reproduces; they also happen to be the four words xoshiro's own + /// reference seeding produces for a zero seed. + const SPLITMIX64_SEED_0: [u64; 6] = [ + 0xE220_A839_7B1D_CDAF, + 0x6E78_9E6A_A1B9_65F4, + 0x06C4_5D18_8009_454F, + 0xF88B_B8A8_724C_81EC, + 0x1B39_896A_51A8_749B, + 0x53CB_9F0C_747E_A2EA, + ]; + + /// The published xoshiro256\*\* output sequence for the canonical test state + /// `s = [1, 2, 3, 4]`. This is the vector that makes the stream a reviewable pin: + /// an accidental edit to a shift, a rotation, or the state-update order fails here + /// rather than silently re-rolling every campaign ever recorded. + const XOSHIRO256SS_STATE_1234: [u64; 10] = [ + 11520, + 0, + 1_509_978_240, + 1_215_971_899_390_074_240, + 1_216_172_134_540_287_360, + 607_988_272_756_665_600, + 16_172_922_978_634_559_625, + 8_476_171_486_693_032_832, + 10_595_114_339_597_558_777, + 2_904_607_092_377_533_576, + ]; + + #[test] + fn splitmix64_matches_published_reference_vectors() { + let mut sm = SplitMix64::new(0); + for (i, expected) in SPLITMIX64_SEED_0.iter().enumerate() { + let got = sm.next(); + assert_eq!( + got, *expected, + "SplitMix64(seed=0) draw {i}: got {got:#018x}, published {expected:#018x}" + ); + } + } + + #[test] + fn xoshiro256ss_matches_published_reference_vectors() { + let mut prng = Prng::from_state([1, 2, 3, 4]).expect("non-zero state"); + for (i, expected) in XOSHIRO256SS_STATE_1234.iter().enumerate() { + let got = prng.next_u64(); + assert_eq!( + got, *expected, + "xoshiro256** from s=[1,2,3,4] draw {i}: got {got}, published {expected}" + ); + } + } + + #[test] + fn seed_expansion_uses_splitmix64_in_declaration_order() { + // The expansion order is part of the pin, so assert it directly rather than + // only through the composed stream: a permuted expansion would still look + // "random" but would break every recorded seed. + let prng = Prng::from_seed(0); + assert_eq!( + prng.s, + [ + SPLITMIX64_SEED_0[0], + SPLITMIX64_SEED_0[1], + SPLITMIX64_SEED_0[2], + SPLITMIX64_SEED_0[3], + ] + ); + } + + #[test] + fn the_same_seed_reproduces_the_same_sequence() { + // FR-001 in miniature: this is the property every finding's reproducibility + // rests on. + let mut a = Prng::from_seed(0x5EED_1234); + let mut b = Prng::from_seed(0x5EED_1234); + let left: Vec = (0..64).map(|_| a.next_u64()).collect(); + let right: Vec = (0..64).map(|_| b.next_u64()).collect(); + assert_eq!(left, right); + + let mut other = Prng::from_seed(0x5EED_1235); + let different: Vec = (0..64).map(|_| other.next_u64()).collect(); + assert_ne!(left, different, "a different seed must not alias"); + } + + #[test] + fn an_all_zero_state_is_refused() { + assert!( + Prng::from_state([0, 0, 0, 0]).is_none(), + "the all-zero state is xoshiro's fixed point — a generator that emits zeros \ + forever must never be constructible" + ); + // No seed can reach it: SplitMix64 is a bijection, so four consecutive counter + // values cannot all map to zero. Spot-check the pathological seeds anyway. + for seed in [0u64, 1, u64::MAX, 0x9E37_79B9_7F4A_7C15] { + assert_ne!(Prng::from_seed(seed).s, [0; 4], "seed {seed:#x}"); + } + } + + #[test] + fn next_bounded_stays_in_range_and_refuses_an_empty_range() { + let mut prng = Prng::from_seed(7); + assert_eq!(prng.next_bounded(0), None, "an empty range has no member"); + assert_eq!(prng.next_bounded(1), Some(0), "a singleton range is forced"); + for _ in 0..10_000 { + let v = prng.next_bounded(7).expect("non-zero bound"); + assert!(v < 7, "draw {v} escaped the bound"); + } + } + + #[test] + fn next_bounded_is_close_to_uniform() { + // Not a statistical proof — a smoke test that the rejection arithmetic is not + // inverted. With 120_000 draws over 6 buckets the expected count is 20_000; + // a modulo-style bias or an off-by-one in the threshold moves a bucket far + // outside this window, while genuine sampling noise does not. + let mut prng = Prng::from_seed(0xABCD_EF01); + let mut counts = [0usize; 6]; + for _ in 0..120_000 { + counts[prng.next_bounded(6).expect("non-zero bound") as usize] += 1; + } + for (bucket, count) in counts.iter().enumerate() { + assert!( + (18_500..=21_500).contains(count), + "bucket {bucket} got {count} of 120000 draws (expected ~20000): {counts:?}" + ); + } + } + + #[test] + fn choose_and_shuffle_are_seed_reproducible() { + let items = ["a", "b", "c", "d", "e"]; + let mut a = Prng::from_seed(42); + let mut b = Prng::from_seed(42); + assert_eq!(a.choose(&items), b.choose(&items)); + + let mut left = items; + let mut right = items; + let mut pa = Prng::from_seed(99); + let mut pb = Prng::from_seed(99); + pa.shuffle(&mut left); + pb.shuffle(&mut right); + assert_eq!(left, right, "the same seed must yield the same permutation"); + + let mut sorted = left; + sorted.sort_unstable(); + assert_eq!( + sorted, items, + "shuffle must be a permutation, not a rewrite" + ); + + let empty: [u8; 0] = []; + assert_eq!(Prng::from_seed(1).choose(&empty), None); + } + + #[test] + fn shuffle_handles_degenerate_lengths() { + let mut prng = Prng::from_seed(3); + let mut empty: [u8; 0] = []; + prng.shuffle(&mut empty); + let mut single = [7u8]; + prng.shuffle(&mut single); + assert_eq!(single, [7]); + } + + #[test] + fn the_identity_string_names_the_algorithm_and_the_revision() { + assert_eq!( + prng_identity(), + "splitmix64-seed+xoshiro256starstar/v1", + "generatorVersion's PRNG component is a reviewed pin — changing it is a \ + deliberate act, not a refactor" + ); + } +} diff --git a/crates/conformance/src/discovery/shrink.rs b/crates/conformance/src/discovery/shrink.rs new file mode 100644 index 00000000..2f3f01eb --- /dev/null +++ b/crates/conformance/src/discovery/shrink.rs @@ -0,0 +1,1536 @@ +//! Structural delta-debugging over the parsed configuration document +//! (025-exploratory-parity-discovery, data-model.md § 6, US2). +//! +//! Reduction never happens at the byte or line level: text-level ddmin on JSON produces +//! syntactically broken intermediates that cannot reproduce a signature living past +//! parsing, and each one costs a full oracle invocation to discover (research D5). +//! +//! The reproduction predicate is a **parameter** ([`ReproductionProbe`]), not a call into +//! the oracle, so the reduction *strategy* stays hermetic and unit-testable against a +//! synthetic predicate while the live campaign supplies the real one +//! (`parity_harness::discovery::minimize`). Without that split the shrinker could only be +//! tested by running a campaign. +//! +//! ## The order is a pin, not a style choice +//! +//! The **ordered catalogue** below is one half of `generatorVersion` — the seventh element +//! of every campaign's pinned input set (data-model.md § 4) — so a campaign cannot record +//! its own provenance without it. The order is reproducibility-critical (FR-020 requires +//! the same finding and seed to yield the identical minimal input, and greedy reduction is +//! order-sensitive), which is why the order lives here once rather than being restated in +//! the generator: two statements of one order are two statements that can disagree. +//! +//! ## Why the reduction terminates, and why that is not incidental +//! +//! Every accepted step strictly decreases a [`Complexity`] triple that is bounded below, so +//! the greedy loop cannot cycle. This matters because one catalogue step — +//! `un-apply-mutation` — can make the document *larger*: reversing an operator that emptied +//! a collection restores the collection. A size-only objective would let the shrinker +//! oscillate between emptying and un-applying forever, spending the expensive step (an +//! oracle invocation) on a loop. The triple orders "corruptions still standing" ahead of +//! size precisely so that reversal always counts as progress. +//! +//! ## What `isMinimal` claims, and what it does not +//! +//! [`Reduction::is_minimal`] is `true` only when a **complete pass over all seven steps** +//! produced no proposal that both reduced the document and preserved the signature. That is +//! FR-021's claim exactly: minimal *with respect to the declared catalogue*, a finite and +//! checkable statement — never the unfalsifiable assertion that no smaller input exists. +//! On budget exhaustion the best reduction found is emitted with `is_minimal: false` and a +//! [`Reduction::not_minimal_reason`] (FR-022); a partially reduced input is never silently +//! presented as minimal. + +use std::future::Future; + +use serde_json::{Map, Value}; + +use super::mutate::Mutation; +use super::signature::Signature; + +/// The seven reduction steps, **in application order** (data-model.md § 6): +/// `drop-optional-key`, `un-apply-mutation`, `empty-collection`, +/// `collapse-extends-level`, `drop-compose-service`, `minimize-scalar`, `drop-feature`. +/// +/// Ordered because greedy reduction is order-sensitive: the same finding and seed must +/// yield the identical minimal input (FR-020), and a different order is a different fixed +/// point. Reordering these names is therefore a pin change, not a refactor — which is why +/// the order belongs to `generatorVersion` rather than to `mutationCatalogVersion`, whose +/// subject is the mutation operator set and which would misdescribe it. +/// +/// `isMinimal` is true only when all seven have been applied once with no step preserving +/// the signature — which is what makes FR-021's minimality claim finite and checkable +/// rather than an unfalsifiable assertion about all possible smaller inputs. +pub const REDUCTION_STEPS: [&str; 7] = [ + "drop-optional-key", + "un-apply-mutation", + "empty-collection", + "collapse-extends-level", + "drop-compose-service", + "minimize-scalar", + "drop-feature", +]; + +/// The revision of the catalogue's **order**, bumped whenever a step is added, removed, +/// or moved. +/// +/// Distinct from the step names themselves: renaming a step for clarity does not change +/// which minimal input a finding reduces to, but moving one does. +pub const REDUCTION_CATALOGUE_VERSION: u32 = 1; + +/// The reduction catalogue's identity, as it appears inside a campaign's +/// `generatorVersion` — the ordered step names plus the order's revision. +/// +/// Spelling the order out rather than hashing it keeps the pinned input set readable: a +/// reviewer comparing two campaigns can see *which* step moved, which an opaque digest +/// would hide behind "the generator changed". +pub fn reduction_catalogue_identity() -> String { + format!( + "reduce[{}]/v{REDUCTION_CATALOGUE_VERSION}", + REDUCTION_STEPS.join(",") + ) +} + +// --------------------------------------------------------------------------- +// The catalogue, as executable steps +// --------------------------------------------------------------------------- + +/// One declared reduction step (data-model.md § 6). +/// +/// The enum and [`REDUCTION_STEPS`] are cross-checked by a unit test rather than one being +/// derived from the other: the constant is the *pin* (it is embedded in every recorded +/// campaign's `generatorVersion`) and the enum is the *implementation*, and a silent +/// derivation would let a step be added to the implementation without the pin moving. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReductionStep { + /// Remove a root key the grammar does not mark `required`. + DropOptionalKey, + /// Reverse one recorded mutation operator, exactly (`mutate::Reversal`). + UnApplyMutation, + /// Replace a non-empty array or object with an empty one of the same shape. + EmptyCollection, + /// Inline one `extends` level and remove the link. + CollapseExtendsLevel, + /// Remove one Compose service the document does not reference through `service`. + DropComposeService, + /// Replace a scalar with the schema-minimal value of its own type. + MinimizeScalar, + /// Remove one entry from `features`. + DropFeature, +} + +impl ReductionStep { + /// Every step, **in application order**. + pub fn all() -> &'static [ReductionStep; 7] { + &[ + ReductionStep::DropOptionalKey, + ReductionStep::UnApplyMutation, + ReductionStep::EmptyCollection, + ReductionStep::CollapseExtendsLevel, + ReductionStep::DropComposeService, + ReductionStep::MinimizeScalar, + ReductionStep::DropFeature, + ] + } + + /// The stable wire spelling, recorded in `Witness::reduction_steps`. + pub fn name(self) -> &'static str { + match self { + ReductionStep::DropOptionalKey => "drop-optional-key", + ReductionStep::UnApplyMutation => "un-apply-mutation", + ReductionStep::EmptyCollection => "empty-collection", + ReductionStep::CollapseExtendsLevel => "collapse-extends-level", + ReductionStep::DropComposeService => "drop-compose-service", + ReductionStep::MinimizeScalar => "minimize-scalar", + ReductionStep::DropFeature => "drop-feature", + } + } +} + +// --------------------------------------------------------------------------- +// The predicate (the parameter, research D4/D5) +// --------------------------------------------------------------------------- + +/// What a probe found when it re-ran a reduced input. +/// +/// Three answers rather than a boolean, because FR-023 needs the third: a step that changes +/// the signature must be *rejected for the finding under reduction* **and** the signature it +/// produced instead must be captured as a separate candidate finding. A `bool` predicate +/// would throw that away at the moment it was observed, and the difference would have to be +/// rediscovered by a later campaign — or never. +#[derive(Debug, Clone, PartialEq)] +pub enum Reproduction { + /// The signature under reduction reproduced, unchanged. + Preserved, + /// The input still differs, but not at the signature under reduction. Every signature + /// observed instead is carried so the caller can admit it separately (FR-023). + Drifted(Vec), + /// Nothing differed at all: the reduction removed the difference. + Absent, +} + +/// The reproduction predicate — the **parameter** that keeps this module hermetic. +/// +/// The live implementation re-runs both CLIs (`parity_harness::discovery::minimize`), which +/// is why the method is async; the hermetic tests here implement it synchronously over a +/// declared rule and never start a process. That is the whole point of research D4/D5: +/// without the split, the reduction *strategy* could only be exercised by running a +/// campaign against the pinned oracle. +pub trait ReproductionProbe { + /// Whatever the implementation's probe can fail with. A probe failure aborts the + /// reduction rather than being read as "did not reproduce": an input whose comparison + /// could not be run has said nothing about whether it reproduces, and treating silence + /// as a rejection would quietly stop reducing. + type Error; + + /// Whether `document` still reproduces the signature under reduction. + fn probe( + &mut self, + document: &Value, + ) -> impl Future>; +} + +// --------------------------------------------------------------------------- +// Input and result +// --------------------------------------------------------------------------- + +/// What a reduction starts from. +#[derive(Debug, Clone, Default)] +pub struct ReductionInput { + /// The document as the campaign observed it. + pub document: Value, + /// The mutation operators applied to it, in application order, each carrying its exact + /// reversal. + pub mutations: Vec, + /// The root keys the grammar marks `required` for this document's branch. + /// + /// Supplied by the caller rather than re-derived here: the grammar is the authority on + /// which keys a valid instance must carry (research D1), and a second view of the + /// pinned surface could disagree with the one generation used. An empty list means + /// "nothing is protected", which reduces harder rather than less — the predicate still + /// rejects anything that stops reproducing. + pub required_keys: Vec, +} + +/// A signature a rejected step produced **instead** of the one under reduction (FR-023). +/// +/// Carries the document it was seen on, not merely the signature. The reduction rejected +/// that proposal and went elsewhere, so its own final document very likely does not +/// reproduce this signature at all — and a witness naming an input that does not reproduce +/// its own signature is a record nobody can re-examine, which is the one thing a finding +/// must never be. +#[derive(Debug, Clone, PartialEq)] +pub struct DriftedFinding { + /// The signature observed instead. + pub signature: Signature, + /// The rejected proposal it was observed on. + pub document: Value, +} + +/// The outcome of one reduction. +#[derive(Debug, Clone, PartialEq)] +pub struct Reduction { + /// The reduced document. + pub document: Value, + /// The catalogue steps applied, in application order — what + /// `Witness::reduction_steps` records. + pub steps: Vec, + /// **Only** true when a complete pass over all seven steps preserved nothing + /// (FR-021). + pub is_minimal: bool, + /// Why it is not minimal. `Some` exactly when [`is_minimal`](Self::is_minimal) is + /// `false` (FR-022): a partially reduced input is never presented as minimal, and + /// never presented as not-minimal without saying why. + pub not_minimal_reason: Option, + /// How many probes the reduction spent — the expensive unit, and the one the budget + /// bounds. + pub probes: u64, + /// Signatures a rejected step produced *instead* of the one under reduction (FR-023), + /// each with the input it was seen on. Deduplicated by signature id, in first-observed + /// order. + pub drifted: Vec, + /// The `mop-` operators still standing on the reduced document. + pub remaining_mutations: Vec, + /// Node count of the input document. + pub original_size: usize, + /// Node count of the reduced document — the SC-004 measure. + pub reduced_size: usize, +} + +impl Reduction { + /// The identity reduction: the input unchanged, explicitly **not** minimal, carrying + /// the reason no reduction was attempted. + /// + /// The caller that skips minimization — a re-witness of a finding the queue already + /// carries, or a campaign whose wall clock has run out — still has to produce a + /// witness, and FR-022 forbids presenting an unreduced input as minimal every bit as + /// much as it forbids presenting a partially reduced one that way. Routing the skip + /// through this constructor means "not minimal, and here is why" is the only shape a + /// skipped reduction can take. + pub fn not_attempted(input: &ReductionInput, reason: impl Into) -> Reduction { + let size = node_count(&input.document); + Reduction { + document: input.document.clone(), + steps: Vec::new(), + is_minimal: false, + not_minimal_reason: Some(reason.into()), + probes: 0, + drifted: Vec::new(), + remaining_mutations: input.mutations.iter().map(|m| m.operator.clone()).collect(), + original_size: size, + reduced_size: size, + } + } + + /// The fraction of the input the reduction removed, in nodes (SC-004's measure). + /// + /// `0.0` for an empty input rather than a division by zero: a document with no nodes + /// was already as small as it can be, and reporting a non-finite fraction would + /// serialize as bare `null` and never load back. + pub fn size_reduction_fraction(&self) -> f64 { + if self.original_size == 0 { + return 0.0; + } + 1.0 - (self.reduced_size as f64 / self.original_size as f64) + } +} + +// --------------------------------------------------------------------------- +// The greedy loop +// --------------------------------------------------------------------------- + +/// Reduce `input` while preserving the signature `probe` tests for (FR-019 – FR-023). +/// +/// `budget` bounds **probes**, not accepted steps: the probe is the expensive unit (the +/// live one runs two CLIs), and a budget counting accepted steps would place no bound at +/// all on a pass that proposes many reductions and accepts none. +/// +/// The loop restarts from the first catalogue step after every acceptance. That is what +/// makes the exit condition meaningful: when it returns with `is_minimal: true`, a complete +/// pass over all seven steps has just been made against the returned document with nothing +/// accepted — FR-021's claim, established rather than asserted. +pub async fn reduce( + input: &ReductionInput, + budget: u64, + probe: &mut P, +) -> Result { + let original_size = node_count(&input.document); + let mut current = input.document.clone(); + let mut mutations = input.mutations.clone(); + let mut steps: Vec = Vec::new(); + let mut drifted: Vec = Vec::new(); + let mut probes: u64 = 0; + let mut exhausted = false; + + 'pass: loop { + for step in *ReductionStep::all() { + for proposal in proposals(step, ¤t, &mutations, input) { + // Only strictly-reducing proposals are ever probed. The check is pure and + // free; the probe is neither, and spending an oracle invocation to learn + // that a non-reduction still reproduces is the budget waste research D5 + // objects to at generation time, relocated to minimization. + if complexity(&proposal.document, proposal.mutations.len()) + >= complexity(¤t, mutations.len()) + { + continue; + } + + // A proposal byte-identical to the current document cannot change the + // outcome, so it is accepted without a probe. This is not an optimization + // in disguise: `un-apply-mutation` reaches it whenever an earlier step + // already removed everything an operator introduced, and probing it would + // spend the run's most expensive step to re-learn what is already known. + if proposal.document == current { + mutations = proposal.mutations; + steps.push(step.name().to_string()); + continue 'pass; + } + + if probes >= budget { + exhausted = true; + break 'pass; + } + probes += 1; + + match probe.probe(&proposal.document).await? { + Reproduction::Preserved => { + current = proposal.document; + mutations = proposal.mutations; + steps.push(step.name().to_string()); + continue 'pass; + } + Reproduction::Drifted(signatures) => { + // FR-023: rejected for THIS finding, and the new signature is + // captured rather than discarded. Discarding it would mean the + // machinery observed a difference and then deliberately forgot it. + // + // The rejected proposal is carried alongside, because it — not the + // document this reduction eventually settles on — is the input that + // reproduces the drifted signature. + for signature in signatures { + if !drifted.iter().any(|d| d.signature.id == signature.id) { + drifted.push(DriftedFinding { + signature, + document: proposal.document.clone(), + }); + } + } + } + Reproduction::Absent => {} + } + } + } + // A complete pass over all seven steps accepted nothing: minimal with respect to + // the declared catalogue (FR-021). + break; + } + + let not_minimal_reason = exhausted.then(|| { + format!( + "the shrink budget of {budget} probe(s) was exhausted after {} accepted step(s); \ + the best reduction found is reported and is NOT minimal (FR-022)", + steps.len() + ) + }); + + Ok(Reduction { + reduced_size: node_count(¤t), + document: current, + steps, + is_minimal: !exhausted, + not_minimal_reason, + probes, + drifted, + remaining_mutations: mutations.iter().map(|m| m.operator.clone()).collect(), + original_size, + }) +} + +/// Whether any single catalogue step further reduces `document` while preserving the +/// signature — the independent minimality check SC-004 requires of every input reported as +/// minimal. +/// +/// Separate from [`reduce`] on purpose: `reduce` establishes minimality as a *consequence* +/// of how its loop exits, and a claim that rests on control flow is worth re-checking with +/// a function whose only job is to look. Returns the first step that still reduces, or +/// `None` when the input is genuinely minimal. +pub async fn first_further_reduction( + input: &ReductionInput, + probe: &mut P, +) -> Result, P::Error> { + for step in *ReductionStep::all() { + for proposal in proposals(step, &input.document, &input.mutations, input) { + if complexity(&proposal.document, proposal.mutations.len()) + >= complexity(&input.document, input.mutations.len()) + { + continue; + } + if proposal.document == input.document { + return Ok(Some(step.name())); + } + if probe.probe(&proposal.document).await? == Reproduction::Preserved { + return Ok(Some(step.name())); + } + } + } + Ok(None) +} + +// --------------------------------------------------------------------------- +// Complexity: why every accepted step is progress +// --------------------------------------------------------------------------- + +/// The objective the greedy loop strictly decreases: `(corruptions, nodes, non-minimal +/// scalars)`, compared lexicographically. +/// +/// Each component exists because exactly one catalogue step reduces it and no other step +/// increases it without reducing an earlier one: +/// +/// | Component | Reduced by | Why it must lead | +/// |---|---|---| +/// | mutations still standing | `un-apply-mutation` | reversal can *grow* the document, so size cannot judge it | +/// | node count | the five removal steps | the ordinary notion of "smaller" | +/// | scalars not at their type-minimum | `minimize-scalar` | `5 → 0` and `true → false` change no node count and may grow the text | +/// +/// Ordering corruptions first is what forbids an emptying/un-applying oscillation: reversal +/// is always progress, so the loop can never return to a document it has already left. +type Complexity = (usize, usize, usize); + +fn complexity(document: &Value, mutations_remaining: usize) -> Complexity { + ( + mutations_remaining, + node_count(document), + non_minimal_scalars(document), + ) +} + +/// Every JSON node, counting object keys as nodes of their own. +/// +/// Keys count because dropping a key whose value is a scalar must register as a reduction: +/// counting only values would make `{"a": 1}` and `{}` differ by one either way, which is +/// fine, but `{"a": null}` → `{}` would too, and a metric that cannot tell an authored +/// `null` from an omission is the conflation 023 T062 already paid for once. +fn node_count(value: &Value) -> usize { + match value { + Value::Array(items) => 1 + items.iter().map(node_count).sum::(), + Value::Object(map) => 1 + map.values().map(|v| 1 + node_count(v)).sum::(), + _ => 1, + } +} + +/// How many scalar positions hold something other than the minimum of their own type. +fn non_minimal_scalars(value: &Value) -> usize { + match value { + Value::Array(items) => items.iter().map(non_minimal_scalars).sum(), + Value::Object(map) => map.values().map(non_minimal_scalars).sum(), + other => usize::from(minimal_scalar(other).as_ref() != Some(other)), + } +} + +/// The schema-minimal value of a scalar's **own type**, or `None` when there is none. +/// +/// Type-preserving by construction (data-model.md § 6, step 6): a reduction that changed a +/// string into `null` would be a `wrong-type` *mutation* wearing a reduction's name, and the +/// reduced input would no longer be an instance of what the finding was found on. +fn minimal_scalar(value: &Value) -> Option { + match value { + Value::String(_) => Some(Value::String(String::new())), + Value::Number(_) => Some(Value::from(0)), + Value::Bool(_) => Some(Value::Bool(false)), + // `null` is already minimal, and a container is not a scalar. + _ => None, + } +} + +// --------------------------------------------------------------------------- +// Proposals +// --------------------------------------------------------------------------- + +/// One candidate reduction: the document it would produce and the mutations that would +/// still stand on it. +#[derive(Debug, Clone)] +struct Proposal { + document: Value, + mutations: Vec, +} + +/// Every proposal `step` makes against `document`, in a **deterministic** order. +/// +/// Determinism here is FR-020: the same finding reduces to the same input because the same +/// proposals are offered in the same sequence and the first accepted one wins. Nothing in +/// this module draws from the PRNG — the reduction is seed-*independent*, which is stronger +/// than FR-020 asks for and considerably easier to hold. +fn proposals( + step: ReductionStep, + document: &Value, + mutations: &[Mutation], + input: &ReductionInput, +) -> Vec { + let keep = |doc: Value| Proposal { + document: doc, + mutations: mutations.to_vec(), + }; + match step { + ReductionStep::DropOptionalKey => root_keys(document) + .into_iter() + .filter(|key| !input.required_keys.iter().any(|r| r == key)) + .filter_map(|key| remove_root_key(document, &key).map(keep)) + .collect(), + + ReductionStep::UnApplyMutation => (0..mutations.len()) + .map(|index| { + let mut remaining = mutations.to_vec(); + let removed = remaining.remove(index); + Proposal { + document: removed.reversal.apply(document), + mutations: remaining, + } + }) + .collect(), + + ReductionStep::EmptyCollection => positions(document) + .into_iter() + .filter_map(|path| { + let emptied = match at(document, &path)? { + Value::Array(items) if !items.is_empty() => Value::Array(Vec::new()), + Value::Object(map) if !map.is_empty() => Value::Object(Map::new()), + _ => return None, + }; + replace_at(document, &path, emptied).map(keep) + }) + .collect(), + + ReductionStep::CollapseExtendsLevel => { + collapse_extends(document).into_iter().map(keep).collect() + } + + ReductionStep::DropComposeService => drop_compose_service(document) + .into_iter() + .map(keep) + .collect(), + + ReductionStep::MinimizeScalar => positions(document) + .into_iter() + .filter_map(|path| { + let current = at(document, &path)?; + let minimal = minimal_scalar(current)?; + if &minimal == current { + return None; + } + replace_at(document, &path, minimal).map(keep) + }) + .collect(), + + ReductionStep::DropFeature => feature_ids(document) + .into_iter() + .filter_map(|id| remove_feature(document, &id).map(keep)) + .collect(), + } +} + +/// `collapse-extends-level` — inline one `extends` parent and remove the link. +/// +/// A chain of two or more loses its **last** link (one level inlined); a chain of one loses +/// the `extends` key entirely. There is no third case: a parent this process cannot read +/// contributes nothing to inline, so removing the link *is* the collapse. Naming that +/// explicitly rather than declining to act keeps the step honest — an `extends` chain is one +/// of the largest contributors to a finding's input, and a step that silently did nothing +/// would leave it standing while `isMinimal` still claimed the catalogue was exhausted. +fn collapse_extends(document: &Value) -> Vec { + let Some(extends) = document.get("extends") else { + return Vec::new(); + }; + match extends { + Value::Array(items) if items.len() >= 2 => { + let mut shortened = items.clone(); + shortened.pop(); + replace_at( + document, + &[Seg::Key("extends".to_string())], + Value::Array(shortened), + ) + .into_iter() + .chain(remove_root_key(document, "extends")) + .collect() + } + _ => remove_root_key(document, "extends").into_iter().collect(), + } +} + +/// `drop-compose-service` — remove one service the document does not reference. +/// +/// Expressed over `runServices`, because that is where a *document* names services: the +/// services themselves are declared in the Compose project beside it, and the candidate's +/// Compose file is fixture scaffolding the campaign writes rather than candidate content. +/// The service named by `service` is never dropped — it is referenced by definition, and a +/// document that lists it under `runServices` and then loses it there has changed which +/// services run, not how many are declared. +fn drop_compose_service(document: &Value) -> Vec { + let Some(Value::Array(services)) = document.get("runServices") else { + return Vec::new(); + }; + let primary = document.get("service"); + (0..services.len()) + .filter(|index| Some(&services[*index]) != primary) + .filter_map(|index| { + let mut remaining = services.clone(); + remaining.remove(index); + replace_at( + document, + &[Seg::Key("runServices".to_string())], + Value::Array(remaining), + ) + }) + .collect() +} + +/// The `features` keys, in the map's own (sorted) order. +fn feature_ids(document: &Value) -> Vec { + match document.get("features") { + Some(Value::Object(features)) => features.keys().cloned().collect(), + _ => Vec::new(), + } +} + +fn remove_feature(document: &Value, id: &str) -> Option { + let Some(Value::Object(features)) = document.get("features") else { + return None; + }; + let mut remaining = features.clone(); + remaining.remove(id)?; + replace_at( + document, + &[Seg::Key("features".to_string())], + Value::Object(remaining), + ) +} + +fn root_keys(document: &Value) -> Vec { + match document { + Value::Object(map) => map.keys().cloned().collect(), + _ => Vec::new(), + } +} + +fn remove_root_key(document: &Value, key: &str) -> Option { + let Value::Object(map) = document else { + return None; + }; + let mut out = map.clone(); + out.remove(key)?; + Some(Value::Object(out)) +} + +// --------------------------------------------------------------------------- +// Positions +// --------------------------------------------------------------------------- + +/// One segment of a position within a document. +#[derive(Debug, Clone, PartialEq, Eq)] +enum Seg { + Key(String), + Index(usize), +} + +/// Every position in `document` **except the root**, in pre-order. +/// +/// Pre-order puts shallow positions first, so the largest available reduction is offered +/// before its own descendants — which matters because the loop restarts after each +/// acceptance and a subtree emptied at its root never has to have its children visited at +/// all. The root itself is excluded: emptying it would produce `{}` for every finding and +/// reduce two unrelated defects to the same input. +fn positions(document: &Value) -> Vec> { + let mut out = Vec::new(); + let mut prefix = Vec::new(); + walk(document, &mut prefix, &mut out); + out +} + +fn walk(value: &Value, prefix: &mut Vec, out: &mut Vec>) { + match value { + Value::Object(map) => { + for key in map.keys() { + prefix.push(Seg::Key(key.clone())); + out.push(prefix.clone()); + prefix.pop(); + } + for (key, child) in map { + prefix.push(Seg::Key(key.clone())); + walk(child, prefix, out); + prefix.pop(); + } + } + Value::Array(items) => { + for index in 0..items.len() { + prefix.push(Seg::Index(index)); + out.push(prefix.clone()); + prefix.pop(); + } + for (index, child) in items.iter().enumerate() { + prefix.push(Seg::Index(index)); + walk(child, prefix, out); + prefix.pop(); + } + } + _ => {} + } +} + +fn at<'a>(document: &'a Value, path: &[Seg]) -> Option<&'a Value> { + let mut cursor = document; + for segment in path { + cursor = match segment { + Seg::Key(key) => cursor.as_object()?.get(key)?, + Seg::Index(index) => cursor.as_array()?.get(*index)?, + }; + } + Some(cursor) +} + +fn replace_at(document: &Value, path: &[Seg], value: Value) -> Option { + let Some((head, rest)) = path.split_first() else { + return Some(value); + }; + match (head, document) { + (Seg::Key(key), Value::Object(map)) => { + let child = map.get(key)?; + let replaced = replace_at(child, rest, value)?; + let mut out = map.clone(); + out.insert(key.clone(), replaced); + Some(Value::Object(out)) + } + (Seg::Index(index), Value::Array(items)) => { + let child = items.get(*index)?; + let replaced = replace_at(child, rest, value)?; + let mut out = items.clone(); + out[*index] = replaced; + Some(Value::Array(out)) + } + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::discovery::mutate::{self, MutationCategory}; + use crate::discovery::rng::Prng; + use crate::discovery::signature::{Divergence, DivergenceKind}; + use serde_json::json; + + // ----------------------------------------------------------------------- + // The catalogue's identity (US1) + // ----------------------------------------------------------------------- + + #[test] + fn the_catalogue_declares_the_seven_steps_in_data_model_order() { + assert_eq!( + REDUCTION_STEPS, + [ + "drop-optional-key", + "un-apply-mutation", + "empty-collection", + "collapse-extends-level", + "drop-compose-service", + "minimize-scalar", + "drop-feature", + ], + "the ORDER is part of `generatorVersion` (FR-020): reordering these changes \ + which minimal input every recorded finding reduces to, so it is a reviewed \ + pin change rather than a refactor" + ); + } + + #[test] + fn step_names_are_unique() { + let mut names = REDUCTION_STEPS.to_vec(); + names.sort_unstable(); + let before = names.len(); + names.dedup(); + assert_eq!(before, names.len(), "two reduction steps share a name"); + } + + #[test] + fn the_identity_names_the_order_rather_than_hiding_it_behind_a_digest() { + let identity = reduction_catalogue_identity(); + assert!(identity.starts_with("reduce[drop-optional-key,")); + assert!(identity.ends_with("]/v1")); + for step in REDUCTION_STEPS { + assert!(identity.contains(step), "{step} missing from the identity"); + } + } + + #[test] + fn the_executable_steps_are_the_pinned_steps_in_the_pinned_order() { + // The pin and the implementation are two statements of one order, cross-checked + // rather than derived: deriving the constant from the enum would let a step be + // added to the implementation without `generatorVersion` moving, and every + // recorded campaign would then name a catalogue it did not run. + let executable: Vec<&str> = ReductionStep::all().iter().map(|s| s.name()).collect(); + assert_eq!(executable, REDUCTION_STEPS.to_vec()); + } + + // ----------------------------------------------------------------------- + // Driving the future without an async runtime + // ----------------------------------------------------------------------- + + /// Poll a future to completion on the current thread. + /// + /// This crate deliberately declares **no async runtime**, not even as a + /// dev-dependency: `discovery_hermetic`'s no-network guard (SC-013) asserts that the + /// capability to speak a network protocol is *absent* here rather than merely unused, + /// and an async runtime is the substrate a socket would need. Taking one on so a test + /// could `.await` would trade that structural guarantee for convenience. + /// + /// It costs almost nothing to avoid. [`reduce`] is async only because the *live* + /// predicate runs two CLI processes; the synthetic predicate below never suspends, so + /// the whole future completes on its first poll and there is nothing to schedule. A + /// `Poll::Pending` here would mean an await point that genuinely needs a scheduler, + /// which is a fact worth a loud panic rather than a silent spin. + fn block_on(future: F) -> F::Output { + let mut future = std::pin::pin!(future); + let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); + match future.as_mut().poll(&mut cx) { + std::task::Poll::Ready(value) => value, + std::task::Poll::Pending => panic!( + "the reduction future suspended. The hermetic tests drive it with a \ + synthetic predicate that never awaits anything real, so a pending poll \ + means a new await point that needs a scheduler — and this crate must not \ + acquire one (SC-013)." + ), + } + } + + fn reduce_now( + input: &ReductionInput, + budget: u64, + probe: &mut P, + ) -> Result { + block_on(reduce(input, budget, probe)) + } + + fn first_further_reduction_now( + input: &ReductionInput, + probe: &mut P, + ) -> Result, P::Error> { + block_on(first_further_reduction(input, probe)) + } + + // ----------------------------------------------------------------------- + // A synthetic predicate (research D4/D5): no oracle, no process, no network + // ----------------------------------------------------------------------- + + fn signature(path: &str) -> Signature { + Signature::derive( + "chan-structured-output", + &Divergence { + kind: DivergenceKind::Value, + path, + deacon: None, + reference: None, + }, + ) + } + + /// The signature under reduction in every test below. + fn target() -> Signature { + signature("configuration.remoteUser") + } + + /// A predicate that reproduces exactly while a declared *witness condition* holds. + /// + /// Deliberately synthetic and deliberately declarative: the reduction strategy's job is + /// to walk the catalogue and honor whatever the predicate says, and a predicate whose + /// rule is written down in the test is the only way to assert that independently of an + /// oracle's behavior on a particular day. + struct Synthetic { + /// Reproduces only while every one of these root keys is present. + needs_keys: Vec, + /// …and only while this root key still holds a non-empty value. + needs_non_empty: Option, + /// Documents that instead produce a DIFFERENT signature (FR-023). + drifts_when_missing: Option, + /// Every document the predicate was asked about, in order — so a test can assert + /// determinism over the probe SEQUENCE, not merely over the answer. + seen: Vec, + } + + impl Synthetic { + fn requiring(keys: &[&str]) -> Synthetic { + Synthetic { + needs_keys: keys.iter().map(|k| (*k).to_string()).collect(), + needs_non_empty: None, + drifts_when_missing: None, + seen: Vec::new(), + } + } + + fn and_non_empty(mut self, key: &str) -> Synthetic { + self.needs_non_empty = Some(key.to_string()); + self + } + + fn drifting_when_missing(mut self, key: &str) -> Synthetic { + self.drifts_when_missing = Some(key.to_string()); + self + } + + fn verdict(&self, document: &Value) -> Reproduction { + let present = |key: &str| document.get(key).is_some(); + if let Some(key) = &self.drifts_when_missing + && !present(key) + { + return Reproduction::Drifted(vec![signature("configuration.name")]); + } + if !self.needs_keys.iter().all(|k| present(k)) { + return Reproduction::Absent; + } + if let Some(key) = &self.needs_non_empty { + let non_empty = match document.get(key) { + Some(Value::Array(a)) => !a.is_empty(), + Some(Value::Object(o)) => !o.is_empty(), + Some(Value::String(s)) => !s.is_empty(), + Some(_) => true, + None => false, + }; + if !non_empty { + return Reproduction::Absent; + } + } + Reproduction::Preserved + } + } + + impl ReproductionProbe for Synthetic { + type Error = std::convert::Infallible; + + async fn probe(&mut self, document: &Value) -> Result { + self.seen.push(document.clone()); + Ok(self.verdict(document)) + } + } + + /// A document with something for every step to bite on. + fn rich() -> Value { + json!({ + "name": "Discovery Seed", + "image": "alpine:3.19", + "remoteUser": "vscode", + "features": { + "ghcr.io/devcontainers/features/git:1": { "version": "os-provided" }, + "ghcr.io/devcontainers/features/node:1": {} + }, + "forwardPorts": [3000, 8080], + "runArgs": ["--init", "--rm"], + "containerEnv": { "A": "1", "B": "2" }, + "extends": ["./base.json", "./middle.json"], + "runServices": ["app", "db", "cache"], + "service": "app", + "shutdownAction": "stopContainer" + }) + } + + fn input(document: Value) -> ReductionInput { + ReductionInput { + document, + mutations: Vec::new(), + required_keys: vec!["image".to_string()], + } + } + + // ----------------------------------------------------------------------- + // T041 — signature preservation + // ----------------------------------------------------------------------- + + #[test] + fn the_reduced_input_yields_the_same_signature_as_the_original() { + let mut probe = Synthetic::requiring(&["image", "remoteUser"]); + let start = input(rich()); + let reduction = reduce_now(&start, 512, &mut probe).expect("infallible"); + + // It still reproduces: the probe says so about the document that was returned. + assert_eq!( + probe.verdict(&reduction.document), + Reproduction::Preserved, + "the reduced input must reproduce the signature under reduction (FR-019)" + ); + assert_eq!( + target().id, + target().derived_id(), + "the target is well-formed" + ); + + // …and it really is reduced. + assert!( + reduction.reduced_size < reduction.original_size, + "reduced {} nodes to {} — no reduction happened at all", + reduction.original_size, + reduction.reduced_size + ); + assert!( + reduction.size_reduction_fraction() >= 0.8, + "reduced only {:.0}% of the input; SC-004 expects at least 80% on an input \ + this redundant. Reduced document: {}", + reduction.size_reduction_fraction() * 100.0, + reduction.document + ); + assert!( + !reduction.steps.is_empty(), + "a reduction that reduced must name the steps it applied" + ); + + // Every step it names is a declared catalogue step, not a description someone + // wrote at the call site. + for step in &reduction.steps { + assert!( + REDUCTION_STEPS.contains(&step.as_str()), + "`{step}` is not a declared catalogue step" + ); + } + } + + #[test] + fn a_reduction_never_removes_a_key_the_predicate_still_needs() { + let mut probe = Synthetic::requiring(&["image", "remoteUser"]).and_non_empty("runArgs"); + let start = input(rich()); + let reduction = reduce_now(&start, 512, &mut probe).expect("infallible"); + + assert!(reduction.document.get("image").is_some()); + assert!(reduction.document.get("remoteUser").is_some()); + assert!( + matches!(reduction.document.get("runArgs"), Some(Value::Array(a)) if !a.is_empty()), + "`runArgs` was emptied even though the predicate needs it non-empty: {}", + reduction.document + ); + } + + // ----------------------------------------------------------------------- + // T042 — minimality (FR-021) + // ----------------------------------------------------------------------- + + #[test] + fn no_single_further_catalogue_step_reduces_a_minimal_result() { + let mut probe = Synthetic::requiring(&["image", "remoteUser"]); + let start = input(rich()); + let reduction = reduce_now(&start, 512, &mut probe).expect("infallible"); + assert!( + reduction.is_minimal, + "a generous budget must reach the catalogue's fixed point: {:?}", + reduction.not_minimal_reason + ); + assert!(reduction.not_minimal_reason.is_none()); + + // The independent check: a function whose only job is to look, rather than a + // property inferred from how the loop happened to exit. + let minimal = ReductionInput { + document: reduction.document.clone(), + mutations: Vec::new(), + required_keys: start.required_keys.clone(), + }; + let mut checker = Synthetic::requiring(&["image", "remoteUser"]); + assert_eq!( + first_further_reduction_now(&minimal, &mut checker).expect("infallible"), + None, + "step `{:?}` still reduces a result reported as minimal — FR-021's claim is \ + that NO single declared step does. Document: {}", + first_further_reduction_now( + &minimal, + &mut Synthetic::requiring(&["image", "remoteUser"]) + ) + .expect("infallible"), + reduction.document + ); + + // The check is not vacuous: it does find a reduction on an input that has one. + let mut checker = Synthetic::requiring(&["image", "remoteUser"]); + assert!( + first_further_reduction_now(&start, &mut checker) + .expect("infallible") + .is_some(), + "the minimality check found nothing to reduce on the UNREDUCED input, so its \ + verdict on the reduced one says nothing" + ); + } + + // ----------------------------------------------------------------------- + // T043 — determinism (SC-004 / FR-020) + // ----------------------------------------------------------------------- + + #[test] + fn the_same_finding_and_seed_yield_the_identical_minimal_input() { + const SEED: u64 = 0x5EED_0043; + + // The seed enters through the mutation stream, which is what a real finding + // carries: the reduction itself draws from no PRNG at all, so it is + // seed-INDEPENDENT — a stronger property than FR-020 asks for, and the reason the + // same finding cannot reduce two ways. + let mutated = |seed: u64| -> ReductionInput { + let mut prng = Prng::from_seed(seed); + let applied = mutate::apply(MutationCategory::UnknownField, &rich(), &mut prng) + .expect("the operator has a target"); + ReductionInput { + document: applied.document, + mutations: vec![applied.mutation], + required_keys: vec!["image".to_string()], + } + }; + + let first = reduce_now( + &mutated(SEED), + 512, + &mut Synthetic::requiring(&["image", "remoteUser"]), + ) + .expect("infallible"); + let second = reduce_now( + &mutated(SEED), + 512, + &mut Synthetic::requiring(&["image", "remoteUser"]), + ) + .expect("infallible"); + + assert_eq!( + first.document, second.document, + "the reduced input differed" + ); + assert_eq!( + first.steps, second.steps, + "the applied step SEQUENCE differed" + ); + assert_eq!(first.probes, second.probes, "the probe count differed"); + assert_eq!(first.is_minimal, second.is_minimal); + assert_eq!(first.remaining_mutations, second.remaining_mutations); + + // The probe SEQUENCE is identical too, not merely its final answer: a shrinker + // that reached the same fixed point by a different route would still be + // non-deterministic in the thing FR-020 bounds — the work a reviewer's + // reproduction has to repeat. + let mut left = Synthetic::requiring(&["image", "remoteUser"]); + let mut right = Synthetic::requiring(&["image", "remoteUser"]); + reduce_now(&mutated(SEED), 512, &mut left).expect("ok"); + reduce_now(&mutated(SEED), 512, &mut right).expect("ok"); + assert_eq!(left.seen, right.seen); + assert!( + !left.seen.is_empty(), + "no probe was ever made, so equality of the sequences is vacuous" + ); + } + + // ----------------------------------------------------------------------- + // T044 — budget exhaustion (FR-022) + // ----------------------------------------------------------------------- + + #[test] + fn an_exhausted_budget_emits_the_best_reduction_marked_not_minimal_with_a_reason() { + let mut probe = Synthetic::requiring(&["image", "remoteUser"]); + let start = input(rich()); + let reduction = reduce_now(&start, 2, &mut probe).expect("infallible"); + + assert!( + !reduction.is_minimal, + "a two-probe budget cannot exhaust the catalogue on this input, so claiming \ + minimality would be presenting a partially reduced input as minimal — exactly \ + what FR-022 forbids" + ); + let reason = reduction + .not_minimal_reason + .as_deref() + .expect("not-minimal is never reported without a reason (FR-022)"); + assert!( + reason.contains("budget") && reason.contains('2'), + "the reason must name the budget that ran out: {reason}" + ); + assert!( + reduction.probes <= 2, + "the budget bounds PROBES, and {} were spent against a budget of 2", + reduction.probes + ); + + // "Best reduction found" — not the untouched input, and not nothing. + assert!( + reduction.reduced_size < reduction.original_size, + "an exhausted budget must still emit the best reduction it reached" + ); + assert_eq!( + probe.verdict(&reduction.document), + Reproduction::Preserved, + "even a partial reduction must still reproduce the signature" + ); + + // And the same input with a generous budget IS minimal, so the flag tracks the + // budget rather than being permanently false. + let mut generous = Synthetic::requiring(&["image", "remoteUser"]); + let full = reduce_now(&start, 512, &mut generous).expect("infallible"); + assert!(full.is_minimal); + assert!(full.reduced_size <= reduction.reduced_size); + } + + // ----------------------------------------------------------------------- + // T045 — signature drift (FR-023) + // ----------------------------------------------------------------------- + + #[test] + fn a_step_that_changes_the_signature_is_rejected_and_the_new_one_is_captured() { + // Dropping `name` produces a DIFFERENT signature rather than removing the + // difference: the step must be rejected for the finding under reduction, and the + // signature it produced instead must survive as a candidate finding. + let mut probe = + Synthetic::requiring(&["image", "remoteUser"]).drifting_when_missing("name"); + let start = input(rich()); + let reduction = reduce_now(&start, 512, &mut probe).expect("infallible"); + + assert!( + reduction.document.get("name").is_some(), + "the drifting step was ACCEPTED: the reduced input no longer reproduces the \ + finding it claims to be about. Document: {}", + reduction.document + ); + assert_eq!( + reduction.drifted.len(), + 1, + "the new signature must be captured as a separate candidate finding (FR-023), \ + got {:?}", + reduction.drifted + ); + let captured = &reduction.drifted[0]; + assert_eq!(captured.signature.path, "configuration.name"); + assert_eq!( + captured.signature.id, + signature("configuration.name").id, + "the captured signature must be the one the probe reported, verbatim" + ); + assert_ne!( + captured.signature.id, + target().id, + "a drifted signature that equalled the target would not be drift at all" + ); + + // It carries the input it was SEEN on, not the document this reduction settled + // on. The rejected proposal is what reproduces the drift; the reduction went + // elsewhere precisely because that proposal did not preserve the target. + assert_eq!( + probe.verdict(&captured.document), + Reproduction::Drifted(vec![signature("configuration.name")]), + "the captured input must reproduce the drifted signature, or the new finding \ + names an input nobody can re-examine" + ); + assert_ne!( + captured.document, reduction.document, + "the drifted input is the REJECTED proposal, which is not where the reduction \ + ended up" + ); + + // Deduplicated: the same drift observed by several rejected proposals is one + // candidate finding, not one per probe. + assert_eq!( + reduction + .drifted + .iter() + .map(|d| d.signature.id.clone()) + .collect::>() + .len(), + reduction.drifted.len() + ); + + // The reduction still did its job around the rejected step. + assert_eq!(probe.verdict(&reduction.document), Reproduction::Preserved); + assert!(reduction.reduced_size < reduction.original_size); + } + + // ----------------------------------------------------------------------- + // Step behaviors, individually + // ----------------------------------------------------------------------- + + #[test] + fn un_apply_mutation_reverses_exactly_one_recorded_operator() { + let mut prng = Prng::from_seed(0x5EED_0047); + let applied = mutate::apply(MutationCategory::UnknownField, &rich(), &mut prng) + .expect("the operator has a target"); + let start = ReductionInput { + document: applied.document.clone(), + mutations: vec![applied.mutation.clone()], + required_keys: vec!["image".to_string()], + }; + + let mut probe = Synthetic::requiring(&["image", "remoteUser"]); + let reduction = reduce_now(&start, 512, &mut probe).expect("infallible"); + + assert!( + reduction.remaining_mutations.is_empty(), + "the mutation was not needed to reproduce, so it must have been un-applied: {:?}", + reduction.remaining_mutations + ); + assert!( + reduction.steps.iter().any(|s| s == "un-apply-mutation"), + "the reversal must be attributed to its catalogue step: {:?}", + reduction.steps + ); + for (key, _) in &applied.mutation.reversal.keys { + assert!( + reduction.document.get(key).is_none() + || rich().get(key) == reduction.document.get(key), + "`{key}` still carries the mutation's value after un-applying it" + ); + } + } + + #[test] + fn collapse_extends_level_shortens_the_chain_and_then_removes_the_link() { + let document = json!({ "image": "alpine:3.19", "extends": ["./a.json", "./b.json"] }); + let shortened = collapse_extends(&document); + assert_eq!( + shortened, + vec![ + json!({ "image": "alpine:3.19", "extends": ["./a.json"] }), + json!({ "image": "alpine:3.19" }), + ], + "a chain of two loses one level first, and the link second" + ); + + let single = json!({ "image": "alpine:3.19", "extends": "./a.json" }); + assert_eq!( + collapse_extends(&single), + vec![json!({ "image": "alpine:3.19" })], + "a chain of one has no level to inline, so the collapse IS removing the link" + ); + + assert!(collapse_extends(&json!({ "image": "alpine:3.19" })).is_empty()); + } + + #[test] + fn drop_compose_service_never_drops_the_service_the_document_names() { + let document = json!({ + "dockerComposeFile": "docker-compose.yml", + "service": "app", + "runServices": ["app", "db", "cache"] + }); + let proposed = drop_compose_service(&document); + assert_eq!( + proposed.len(), + 2, + "two droppable services, got {proposed:?}" + ); + for candidate in &proposed { + let services = candidate["runServices"] + .as_array() + .expect("still an array") + .clone(); + assert!( + services.contains(&json!("app")), + "the referenced service was dropped: {candidate}" + ); + assert_eq!(services.len(), 2); + } + + // Nothing to do without a `runServices` list. + assert!(drop_compose_service(&json!({ "service": "app" })).is_empty()); + } + + #[test] + fn minimize_scalar_preserves_the_json_type() { + for (before, after) in [ + (json!("something"), json!("")), + (json!(4242), json!(0)), + (json!(true), json!(false)), + ] { + assert_eq!(minimal_scalar(&before), Some(after)); + } + assert_eq!( + minimal_scalar(&json!(null)), + None, + "null is already minimal" + ); + assert_eq!( + minimal_scalar(&json!([1])), + None, + "an array is not a scalar" + ); + assert_eq!( + minimal_scalar(&json!({})), + None, + "an object is not a scalar" + ); + } + + #[test] + fn drop_feature_removes_one_entry_at_a_time() { + let document = json!({ + "image": "alpine:3.19", + "features": { "a": {}, "b": {}, "c": {} } + }); + let ids = feature_ids(&document); + assert_eq!( + ids, + vec!["a", "b", "c"], + "features are visited in map order" + ); + for id in &ids { + let reduced = remove_feature(&document, id).expect("removable"); + let features = reduced["features"].as_object().expect("object"); + assert_eq!(features.len(), 2); + assert!(!features.contains_key(id)); + } + assert!(remove_feature(&document, "not-there").is_none()); + } + + #[test] + fn a_required_key_is_never_dropped() { + let start = ReductionInput { + document: json!({ "image": "alpine:3.19", "name": "x", "remoteUser": "vscode" }), + mutations: Vec::new(), + // Everything is "required" here, so a reduction that respects the grammar has + // nothing to drop and the loop must terminate immediately as minimal. + required_keys: vec![ + "image".to_string(), + "name".to_string(), + "remoteUser".to_string(), + ], + }; + let mut probe = Synthetic::requiring(&[]); + let reduction = reduce_now(&start, 512, &mut probe).expect("infallible"); + for key in ["image", "name", "remoteUser"] { + assert!( + reduction.document.get(key).is_some(), + "the grammar marks `{key}` required, and a reduction that violates the \ + grammar produces an input the finding was never found on" + ); + } + assert!(reduction.is_minimal); + } + + // ----------------------------------------------------------------------- + // Termination, the property the whole loop rests on + // ----------------------------------------------------------------------- + + #[test] + fn every_accepted_step_strictly_decreases_the_objective() { + // The guard against an emptying/un-applying oscillation. `un-apply-mutation` can + // GROW the document, so a size-only objective would let the loop cycle forever, + // spending the expensive step on a ring it never leaves. + let mut prng = Prng::from_seed(0x5EED_0048); + let applied = mutate::apply(MutationCategory::EmptyValue, &rich(), &mut prng) + .expect("the operator has a target"); + let before = complexity(&applied.document, 1); + let reversed = applied.mutation.reversal.apply(&applied.document); + let after = complexity(&reversed, 0); + + assert!( + after < before, + "un-applying must count as progress even when it grows the document: \ + {before:?} → {after:?}" + ); + assert!( + node_count(&reversed) >= node_count(&applied.document), + "this assertion is vacuous unless the reversal really did grow the document" + ); + } + + #[test] + fn a_probe_failure_aborts_the_reduction_rather_than_reading_as_a_rejection() { + struct Failing; + impl ReproductionProbe for Failing { + type Error = &'static str; + async fn probe(&mut self, _document: &Value) -> Result { + Err("the comparison could not be run") + } + } + let start = input(rich()); + let error = + reduce_now(&start, 512, &mut Failing).expect_err("a probe failure must propagate"); + assert_eq!(error, "the comparison could not be run"); + } + + #[test] + fn a_skipped_reduction_is_never_presented_as_minimal() { + let start = input(rich()); + let skipped = Reduction::not_attempted(&start, "already carried by the standing queue"); + assert!( + !skipped.is_minimal, + "an UNREDUCED input presented as minimal is the same lie FR-022 forbids about a \ + partially reduced one" + ); + assert_eq!( + skipped.not_minimal_reason.as_deref(), + Some("already carried by the standing queue") + ); + assert_eq!(skipped.document, start.document); + assert_eq!(skipped.probes, 0); + assert!(skipped.steps.is_empty()); + assert_eq!(skipped.original_size, skipped.reduced_size); + assert_eq!(skipped.size_reduction_fraction(), 0.0); + } + + #[test] + fn an_empty_document_reduces_to_itself_and_reports_no_division_by_zero() { + let start = ReductionInput { + document: json!({}), + mutations: Vec::new(), + required_keys: Vec::new(), + }; + let mut probe = Synthetic::requiring(&[]); + let reduction = reduce_now(&start, 512, &mut probe).expect("infallible"); + assert_eq!(reduction.document, json!({})); + assert!(reduction.is_minimal); + assert_eq!(reduction.probes, 0); + assert!(reduction.size_reduction_fraction().is_finite()); + assert_eq!(reduction.size_reduction_fraction(), 0.0); + } +} diff --git a/crates/conformance/src/discovery/signature.rs b/crates/conformance/src/discovery/signature.rs new file mode 100644 index 00000000..a99fdead --- /dev/null +++ b/crates/conformance/src/discovery/signature.rs @@ -0,0 +1,707 @@ +//! The normalized signature — the deduplication key +//! (025-exploratory-parity-discovery, research D3, data-model.md § 2, T014/T015). +//! +//! A signature is `(channel, observable path, difference kind, value-shape class)`, +//! hashed into a substance-anchored `sig-` id. It is **derived** from the +//! comparison's own diff output and never re-computed from the two documents. +//! +//! ## Why derivation, not a second diff +//! +//! FR-015 permits exactly one normalization definition. A signature computed by +//! independently re-diffing the two sides would be a second opinion on *what differs*, +//! able to disagree with the one the comparison actually used — the identical defect +//! class the single-normalizer rule exists to prevent. Deriving from the comparison's +//! output makes disagreement structurally impossible: there is nothing to disagree with. +//! +//! [`Divergence`] is therefore an **input shape**, not a differ. It carries exactly the +//! four fields `parity_harness::normalize::ConfigDivergence` already produces — kind, +//! path, and the two `Option`s — and this module never inspects anything else. +//! +//! ## Why this type lives here rather than importing `ConfigDivergence` +//! +//! `parity-harness` depends on `deacon-conformance`, not the other way round (the +//! hermetic half must stay loadable without the live half), so importing +//! `ConfigDivergence` here would be a dependency cycle. The live side adapts its +//! `ConfigDivergence` into [`Divergence`] at the single call site in +//! `parity_harness::discovery::differential` (T035) — a field-for-field move with no +//! recomputation, which is what keeps "derive only, never re-diff" true in the code and +//! not merely in the comment. +//! +//! ## Why concrete values are not in the signature +//! +//! Structure alone (channel + path + kind) merges a *missing* `remoteUser` with a +//! *wrongly-typed* `remoteUser`; including concrete values splits one defect across +//! every generated value and makes deduplication do nothing, so the queue would grow +//! with campaign volume. The value-*shape* class is the level at which "same defect" is +//! true. The concrete observed values are retained on the witness, where they are +//! evidence rather than identity. + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use super::hash8; + +/// The difference kind, mirroring `parity_harness::normalize::DiffKind` **exactly**, +/// including its wire spellings. +/// +/// Kept as a closed enum rather than a bare string so an adapter cannot quietly +/// introduce a fourth kind: the diff produces three, the signature recognizes three, and +/// a fourth would have to be added here deliberately. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum DivergenceKind { + /// Present on the reference side only. + RefOnly, + /// Present on the deacon side only. + DeaconOnly, + /// Present on both sides with different values. + Value, +} + +impl DivergenceKind { + /// The stable wire spelling — byte-identical to `DiffKind::as_str()`. + pub fn as_str(self) -> &'static str { + match self { + DivergenceKind::RefOnly => "ref-only", + DivergenceKind::DeaconOnly => "deacon-only", + DivergenceKind::Value => "value", + } + } + + /// Parse a wire spelling produced by `DiffKind::as_str()`. + /// + /// Returns `None` on anything else rather than defaulting to a kind: a silently + /// mis-parsed kind would merge two genuinely different defect families under one + /// signature (constitution IV). + pub fn parse(s: &str) -> Option { + match s { + "ref-only" => Some(DivergenceKind::RefOnly), + "deacon-only" => Some(DivergenceKind::DeaconOnly), + "value" => Some(DivergenceKind::Value), + _ => None, + } + } +} + +/// The value-shape class — the one derivation this feature adds +/// (data-model.md § 2). +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ValueShapeClass { + /// One side has the value and the other does not. + PresentAbsent, + /// Both sides have a value, of different JSON types. + TypeChanged, + /// Both sides have an array, and one is a permutation of the other. + OrderingChanged, + /// Both sides have a same-typed value that differs some other way. + ValueChanged, +} + +impl ValueShapeClass { + /// The stable wire spelling. + pub fn as_str(self) -> &'static str { + match self { + ValueShapeClass::PresentAbsent => "present-absent", + ValueShapeClass::TypeChanged => "type-changed", + ValueShapeClass::OrderingChanged => "ordering-changed", + ValueShapeClass::ValueChanged => "value-changed", + } + } +} + +/// The input shape a signature is derived from: exactly the fields +/// `parity_harness::normalize::ConfigDivergence` produces. +/// +/// Borrowed rather than owned so the adapter is a zero-copy field-for-field move and no +/// caller is tempted to build one from anything other than the diff's own output. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Divergence<'a> { + /// `ConfigDivergence::kind`. + pub kind: DivergenceKind, + /// `ConfigDivergence::path`, verbatim. + pub path: &'a str, + /// `ConfigDivergence::deacon`. + pub deacon: Option<&'a Value>, + /// `ConfigDivergence::reference`. + pub reference: Option<&'a Value>, +} + +/// The deduplication key (data-model.md § 2). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Signature { + /// `sig-` over `channel ‖ path ‖ kind ‖ valueShapeClass`. + pub id: String, + /// One of the declared observable channels (`chan-…`). Supplied by the caller: the + /// observers already partition evidence this way, and a channel a signature invents + /// is **D1**. + pub channel: String, + /// The observable path within that channel, verbatim from the diff. + pub path: String, + /// The difference kind. + pub kind: DivergenceKind, + /// The value-shape class. + pub value_shape_class: ValueShapeClass, +} + +impl Signature { + /// Derive a signature from a channel and one divergence. + /// + /// Pure and total: every divergence classifies, so there is no "unclassifiable" + /// escape hatch that could quietly drop a real difference. + pub fn derive(channel: &str, divergence: &Divergence<'_>) -> Signature { + let value_shape_class = classify(divergence); + let id = signature_id(channel, divergence.path, divergence.kind, value_shape_class); + Signature { + id, + channel: channel.to_string(), + path: divergence.path.to_string(), + kind: divergence.kind, + value_shape_class, + } + } + + /// Recompute this signature's id from its own fields. + /// + /// The identity check: a hand-edited queue record whose `id` no longer matches its + /// substance is **D1**, and this is what detects it. + pub fn derived_id(&self) -> String { + signature_id(&self.channel, &self.path, self.kind, self.value_shape_class) + } + + /// The `fnd-` id of the finding this signature keys. + /// + /// A finding's id is *derived* from its signature rather than independently + /// assigned, which makes duplicate findings unrepresentable: FR-030 says two + /// findings with the same signature **are** one finding, and an independently + /// assigned id would leave that invariant to be maintained by the merge logic — and + /// therefore violable by a bad merge (data-model.md § 1). + /// + /// The `fnd-` hash is taken over the signature's `id` rather than re-hashing its + /// four fields, so the 1:1 correspondence is visible in the derivation itself. + pub fn finding_id(&self) -> String { + format!("fnd-{}", hash8(&[&self.id])) + } +} + +/// Classify a divergence's value shape (data-model.md § 2's table). +/// +/// `ordering-changed` is tested **before** `value-changed` and is a distinct class +/// rather than a subcase, because declaration-order defects are a known recurring family +/// in this codebase (`BTreeMap` where the spec requires declaration order). Folding them +/// into `value-changed` would merge an order defect with an unrelated value defect at +/// the same path, and a merged finding cannot be split back into its causes because the +/// distinguishing information was never recorded. +pub fn classify(divergence: &Divergence<'_>) -> ValueShapeClass { + match divergence.kind { + // The diff only emits these when exactly one side has the value, so presence is + // the whole difference regardless of what the present value happens to be. + DivergenceKind::RefOnly | DivergenceKind::DeaconOnly => ValueShapeClass::PresentAbsent, + DivergenceKind::Value => match (divergence.deacon, divergence.reference) { + (Some(d), Some(r)) => classify_values(d, r), + // A `Value` divergence missing a side is a malformed input rather than a + // shape: treat it as the presence difference it describes rather than + // inventing a value comparison out of a `None`. + _ => ValueShapeClass::PresentAbsent, + }, + } +} + +/// Classify two present, differing values. +fn classify_values(deacon: &Value, reference: &Value) -> ValueShapeClass { + if json_type(deacon) != json_type(reference) { + return ValueShapeClass::TypeChanged; + } + if let (Value::Array(d), Value::Array(r)) = (deacon, reference) + && is_permutation(d, r) + { + return ValueShapeClass::OrderingChanged; + } + ValueShapeClass::ValueChanged +} + +/// The JSON type name, for the type-changed test. +/// +/// Numbers are one type: `1` versus `1.0` is a serialization detail of the producer, not +/// a type difference either implementation chose, and classifying it as `type-changed` +/// would split a value defect into a phantom type defect. +fn json_type(v: &Value) -> &'static str { + match v { + Value::Null => "null", + Value::Bool(_) => "boolean", + Value::Number(_) => "number", + Value::String(_) => "string", + Value::Array(_) => "array", + Value::Object(_) => "object", + } +} + +/// Whether `a` and `b` hold the same multiset of elements in a different order. +/// +/// A **multiset** comparison, not a set comparison: `[1, 1, 2]` and `[1, 2, 2]` are the +/// same *set* but are not a permutation of each other, and calling them an ordering +/// difference would misattribute a genuine content change to element order. +/// +/// `Value` is not `Ord`, so the multiset test is done by canonical-string counting — +/// `serde_json`'s `to_string` is deterministic for a given `Value` (this crate enables +/// `preserve_order`, so object key order is the value's own and two equal `Value`s +/// always render identically). +fn is_permutation(a: &[Value], b: &[Value]) -> bool { + if a.len() != b.len() { + return false; + } + // An empty (or single-element) array cannot be a *re*-ordering of anything: with + // fewer than two elements there is only one order, so a difference at that path is + // never about order. `diff` never emits equal values, so this only guards the + // degenerate shapes. + if a.len() < 2 { + return false; + } + let mut counts: std::collections::BTreeMap = std::collections::BTreeMap::new(); + for v in a { + *counts.entry(canonical_key(v)).or_insert(0) += 1; + } + for v in b { + *counts.entry(canonical_key(v)).or_insert(0) -= 1; + } + counts.values().all(|&n| n == 0) +} + +/// A canonical string form of a value, with **object keys sorted**, used as the +/// multiset key in [`is_permutation`]. +/// +/// Sorting is not cosmetic. This crate enables `serde_json`'s `preserve_order`, so a +/// `Value::Object` is an insertion-ordered `IndexMap` whose `PartialEq` compares as a +/// *map* — `{"a":1,"b":2}` and `{"b":2,"a":1}` are `==`. Their default renderings are +/// not. Keying the multiset on the raw rendering would therefore call two elements +/// different that `Value` itself calls equal, and an array reordering whose objects +/// happened to be serialized with different key order would fall out of +/// `ordering-changed` into `value-changed` — silently splitting one declaration-order +/// defect across two signatures and defeating the deduplication that class exists for. +fn canonical_key(value: &Value) -> String { + fn write(value: &Value, out: &mut String) { + use std::fmt::Write as _; + match value { + Value::Object(map) => { + let mut keys: Vec<&String> = map.keys().collect(); + keys.sort_unstable(); + out.push('{'); + for (i, k) in keys.iter().enumerate() { + if i > 0 { + out.push(','); + } + // `Value::String`'s Display-via-to_string is the JSON-escaped form, + // so key and value alike stay unambiguous. + let _ = write!(out, "{}:", Value::String((*k).clone())); + write(&map[k.as_str()], out); + } + out.push('}'); + } + Value::Array(items) => { + out.push('['); + for (i, item) in items.iter().enumerate() { + if i > 0 { + out.push(','); + } + write(item, out); + } + out.push(']'); + } + // Scalars render unambiguously and carry no ordering of their own. + other => { + let _ = write!(out, "{other}"); + } + } + } + let mut out = String::new(); + write(value, &mut out); + out +} + +/// `sig-` over `channel ‖ path ‖ kind ‖ valueShapeClass`. +fn signature_id( + channel: &str, + path: &str, + kind: DivergenceKind, + value_shape_class: ValueShapeClass, +) -> String { + format!( + "sig-{}", + hash8(&[channel, path, kind.as_str(), value_shape_class.as_str()]) + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn div<'a>( + kind: DivergenceKind, + path: &'a str, + deacon: Option<&'a Value>, + reference: Option<&'a Value>, + ) -> Divergence<'a> { + Divergence { + kind, + path, + deacon, + reference, + } + } + + #[test] + fn present_absent_covers_both_one_sided_kinds() { + let v = json!("vscode"); + assert_eq!( + classify(&div( + DivergenceKind::RefOnly, + "configuration.remoteUser", + None, + Some(&v) + )), + ValueShapeClass::PresentAbsent + ); + assert_eq!( + classify(&div( + DivergenceKind::DeaconOnly, + "configuration.remoteUser", + Some(&v), + None + )), + ValueShapeClass::PresentAbsent + ); + } + + #[test] + fn type_changed_when_the_two_json_types_differ() { + let d = json!("3000"); + let r = json!(3000); + assert_eq!( + classify(&div( + DivergenceKind::Value, + "configuration.forwardPorts.0", + Some(&d), + Some(&r) + )), + ValueShapeClass::TypeChanged + ); + + // null-vs-value is a type change, not a presence change: both sides emitted + // something, and `null` is a value the spec distinguishes from omission. + let null = json!(null); + let some = json!("x"); + assert_eq!( + classify(&div(DivergenceKind::Value, "p", Some(&null), Some(&some))), + ValueShapeClass::TypeChanged + ); + + // An array against an object is a type change, never an ordering change. + let arr = json!([1, 2]); + let obj = json!({ "a": 1 }); + assert_eq!( + classify(&div(DivergenceKind::Value, "p", Some(&arr), Some(&obj))), + ValueShapeClass::TypeChanged + ); + } + + #[test] + fn integers_and_floats_are_one_type() { + // `1` vs `1.0` is the producer's serialization detail, not a type either + // implementation chose; classifying it as `type-changed` would manufacture a + // phantom type defect out of a value difference. + let d = json!(1); + let r = json!(1.5); + assert_eq!( + classify(&div(DivergenceKind::Value, "p", Some(&d), Some(&r))), + ValueShapeClass::ValueChanged + ); + } + + #[test] + fn ordering_changed_detects_an_array_permutation() { + let d = json!(["a", "b", "c"]); + let r = json!(["c", "a", "b"]); + assert_eq!( + classify(&div( + DivergenceKind::Value, + "configuration.features", + Some(&d), + Some(&r) + )), + ValueShapeClass::OrderingChanged, + "declaration-order defects are a known family here and must not fold into \ + value-changed" + ); + + // Permutation of non-scalars too — the class is about order, not element type. + let d = json!([{ "id": "a" }, { "id": "b" }]); + let r = json!([{ "id": "b" }, { "id": "a" }]); + assert_eq!( + classify(&div(DivergenceKind::Value, "p", Some(&d), Some(&r))), + ValueShapeClass::OrderingChanged + ); + } + + #[test] + fn object_key_order_within_an_element_does_not_defeat_permutation_detection() { + // `preserve_order` makes a JSON object an insertion-ordered map whose equality is + // still map equality: these two elements are `==` as `Value`s but render + // differently. Keying the multiset on the raw rendering would call them different + // and demote a genuine reordering to `value-changed`, splitting one + // declaration-order defect across two signatures. + let left = json!({ "id": "a", "version": "1" }); + let right = json!({ "version": "1", "id": "a" }); + assert_eq!(left, right, "serde_json compares objects as maps"); + assert_eq!(canonical_key(&left), canonical_key(&right)); + + let d = json!([{ "id": "a", "version": "1" }, { "id": "b", "version": "2" }]); + let r = json!([{ "version": "2", "id": "b" }, { "version": "1", "id": "a" }]); + assert_eq!( + classify(&div(DivergenceKind::Value, "p", Some(&d), Some(&r))), + ValueShapeClass::OrderingChanged + ); + + // Nested objects and arrays canonicalize too, and genuinely different values + // still key differently. + assert_eq!( + canonical_key(&json!({ "a": [{ "y": 1, "x": 2 }] })), + canonical_key(&json!({ "a": [{ "x": 2, "y": 1 }] })) + ); + assert_ne!( + canonical_key(&json!({ "a": 1 })), + canonical_key(&json!({ "a": 2 })) + ); + // A key/value boundary cannot be blurred: `{"a:1":null}` must not key the same + // as `{"a":1}`. + assert_ne!( + canonical_key(&json!({ "a:1": null })), + canonical_key(&json!({ "a": 1 })) + ); + } + + #[test] + fn a_multiset_difference_is_not_an_ordering_change() { + // Same SET, different multiset: calling this an ordering difference would + // misattribute a genuine content change to element order. + let d = json!([1, 1, 2]); + let r = json!([1, 2, 2]); + assert_eq!( + classify(&div(DivergenceKind::Value, "p", Some(&d), Some(&r))), + ValueShapeClass::ValueChanged + ); + + // Different lengths, and a strict subsequence, are both value changes. + let d = json!([1, 2, 3]); + let r = json!([1, 2]); + assert_eq!( + classify(&div(DivergenceKind::Value, "p", Some(&d), Some(&r))), + ValueShapeClass::ValueChanged + ); + + // A one-element array has only one order, so a difference there is never + // about ordering. + let d = json!(["a"]); + let r = json!(["b"]); + assert_eq!( + classify(&div(DivergenceKind::Value, "p", Some(&d), Some(&r))), + ValueShapeClass::ValueChanged + ); + } + + #[test] + fn value_changed_is_the_residual_class() { + let d = json!("vscode"); + let r = json!("root"); + assert_eq!( + classify(&div( + DivergenceKind::Value, + "configuration.remoteUser", + Some(&d), + Some(&r) + )), + ValueShapeClass::ValueChanged + ); + let d = json!({ "a": 1 }); + let r = json!({ "a": 2 }); + assert_eq!( + classify(&div(DivergenceKind::Value, "p", Some(&d), Some(&r))), + ValueShapeClass::ValueChanged + ); + } + + #[test] + fn a_value_divergence_missing_a_side_degrades_to_present_absent() { + // The diff never produces this, but a malformed adapter could. Classifying it + // as the presence difference it describes is honest; inventing a comparison + // against `None` would not be. + let d = json!("x"); + assert_eq!( + classify(&div(DivergenceKind::Value, "p", Some(&d), None)), + ValueShapeClass::PresentAbsent + ); + } + + #[test] + fn the_id_is_substance_anchored_and_stable() { + let d = json!("vscode"); + let r = json!("root"); + let a = Signature::derive( + "chan-structured-output", + &div( + DivergenceKind::Value, + "configuration.remoteUser", + Some(&d), + Some(&r), + ), + ); + // Same structure, DIFFERENT concrete values → same signature. This is the whole + // point: concrete values are evidence on the witness, not identity. + let d2 = json!("node"); + let r2 = json!("ubuntu"); + let b = Signature::derive( + "chan-structured-output", + &div( + DivergenceKind::Value, + "configuration.remoteUser", + Some(&d2), + Some(&r2), + ), + ); + assert_eq!(a.id, b.id, "concrete values must not enter the signature"); + assert_eq!(a.finding_id(), b.finding_id()); + + assert!(a.id.starts_with("sig-")); + assert_eq!(a.id.len(), "sig-".len() + 8); + assert_eq!( + a.derived_id(), + a.id, + "the id must recompute from its fields" + ); + assert!(a.finding_id().starts_with("fnd-")); + assert_eq!(a.finding_id().len(), "fnd-".len() + 8); + } + + #[test] + fn every_signature_component_changes_the_id() { + let d = json!("a"); + let r = json!("b"); + let base = Signature::derive( + "chan-structured-output", + &div( + DivergenceKind::Value, + "configuration.remoteUser", + Some(&d), + Some(&r), + ), + ); + // channel + let other_channel = Signature::derive( + "chan-stdout", + &div( + DivergenceKind::Value, + "configuration.remoteUser", + Some(&d), + Some(&r), + ), + ); + assert_ne!(base.id, other_channel.id); + // path + let other_path = Signature::derive( + "chan-structured-output", + &div( + DivergenceKind::Value, + "configuration.remoteEnv", + Some(&d), + Some(&r), + ), + ); + assert_ne!(base.id, other_path.id); + // kind + let other_kind = Signature::derive( + "chan-structured-output", + &div( + DivergenceKind::RefOnly, + "configuration.remoteUser", + None, + Some(&r), + ), + ); + assert_ne!(base.id, other_kind.id); + // value-shape class (same channel/path/kind, different shape) + let arr_d = json!(["a", "b"]); + let arr_r = json!(["b", "a"]); + let other_shape = Signature::derive( + "chan-structured-output", + &div( + DivergenceKind::Value, + "configuration.remoteUser", + Some(&arr_d), + Some(&arr_r), + ), + ); + assert_ne!(base.id, other_shape.id); + } + + #[test] + fn a_path_containing_the_field_separator_cannot_collide() { + // A signature's `path` comes verbatim from the diff and is built from + // user-controlled configuration keys, so it can contain any byte. The shared + // `hash8` length-prefixes rather than separating for exactly this reason; assert + // it at the signature level, where the consequence of a collision would be two + // distinct defects merging into one finding. + let v = json!(1); + let a = Signature::derive( + "chan-structured-output", + &div(DivergenceKind::Value, "a\u{1f}b", Some(&v), Some(&v)), + ); + let b = Signature::derive( + "chan-structured-output\u{1f}a", + &div(DivergenceKind::Value, "b", Some(&v), Some(&v)), + ); + assert_ne!(a.id, b.id); + assert_ne!(hash8(&["ab", "c"]), hash8(&["a", "bc"])); + } + + #[test] + fn kind_wire_spellings_round_trip() { + // These must stay byte-identical to `DiffKind::as_str()`: the adapter in + // parity-harness maps between them, and a spelling drift would silently + // re-key every signature. + for kind in [ + DivergenceKind::RefOnly, + DivergenceKind::DeaconOnly, + DivergenceKind::Value, + ] { + assert_eq!(DivergenceKind::parse(kind.as_str()), Some(kind)); + } + assert_eq!(DivergenceKind::parse("value-ish"), None); + } + + #[test] + fn signature_round_trips_through_strict_json() { + let d = json!(["a", "b"]); + let r = json!(["b", "a"]); + let sig = Signature::derive( + "chan-structured-output", + &div( + DivergenceKind::Value, + "configuration.features", + Some(&d), + Some(&r), + ), + ); + let raw = serde_json::to_string(&sig).expect("serializes"); + assert!(raw.contains("\"valueShapeClass\":\"ordering-changed\"")); + let back: Signature = serde_json::from_str(&raw).expect("round-trips"); + assert_eq!(back, sig); + + let err = serde_json::from_str::( + r#"{"id":"sig-1","channel":"c","path":"p","kind":"value", + "valueShapeClass":"value-changed","extra":1}"#, + ) + .expect_err("unknown fields are rejected at load"); + assert!(err.to_string().contains("extra")); + } +} diff --git a/crates/conformance/src/lib.rs b/crates/conformance/src/lib.rs index 70a2f82c..3f01fe08 100644 --- a/crates/conformance/src/lib.rs +++ b/crates/conformance/src/lib.rs @@ -26,6 +26,7 @@ pub mod conservation; pub mod coverage; pub mod coverage_report; pub mod diff; +pub mod discovery; pub mod inventory; pub mod load; pub mod mapping; @@ -191,6 +192,192 @@ pub fn obligations_file_for(registry_dir: &std::path::Path) -> std::path::PathBu base.join("obligations").join("obligations.json") } +/// The default discovery data root: `/conformance/discovery` — the +/// findings queue, the campaign history, and the real-world corpus manifest +/// (025-exploratory-parity-discovery, research D6). +/// +/// **A sibling of `registry/`, deliberately not inside it.** [`load::Registry::load`] +/// enumerates *named* subdirectories under `conformance/registry/` and has no wildcard +/// walk at the registry root, so nothing here can be picked up by the registry loader — +/// not by convention, but because there is no code path that would reach it. That is +/// what makes "an unreviewed finding can never influence a release gate" a property of +/// the directory layout rather than a rule someone must remember. +pub fn default_discovery_dir() -> std::path::PathBuf { + workspace_root().join("conformance").join("discovery") +} + +/// Resolve the discovery data root belonging to a registry, as a sibling under the same +/// `conformance/` tree: `/../discovery`. Mirrors [`clause_paths_for`] / +/// [`migration_paths_for`] / [`obligations_file_for`], so `--registry ` picks up +/// the fixture's own discovery root rather than the workspace's. +pub fn discovery_dir_for(registry_dir: &std::path::Path) -> std::path::PathBuf { + let base = registry_dir.parent().unwrap_or(registry_dir); + base.join("discovery") +} + +/// The discovery-side (**D-class**) domain error taxonomy +/// (025-exploratory-parity-discovery, contracts/findings-queue.md). +/// +/// Each variant *is* a violation: [`DiscoveryError::class`] names the D-class it belongs +/// to and [`DiscoveryError::record`] names the offending record, so `discovery check` +/// renders `{class} {record}: {message}` without a parallel violation type. Every +/// message names the cause precisely (constitution IV). +/// +/// **Numbered separately from the registry's V-series on purpose.** These are emitted by +/// a different command over a different data root; folding them into V-numbering would +/// imply the registry validator can see the queue, which is exactly what the discovery +/// root's placement exists to prevent (research D6/D11). +/// +/// **D3** landed with US5 as [`DiscoveryError::PromotionUnresolved`]; **D4** landed with +/// US7 as [`DiscoveryError::CorpusIntegrity`]. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum DiscoveryError { + /// **D1** — a record that does not parse, or that parses but is structurally + /// impossible (empty `witnesses`, a derived id that disagrees with its substance). + /// `record` is the record id when one could be read, else the file path. + #[error( + "malformed discovery record `{record}`: {cause}. Remedy: fix the record — the \ + discovery data root is strict JSON and rejects unknown fields at load." + )] + MalformedRecord { record: String, cause: String }, + + /// **D1** — a reference that names something absent: a `firstObserved` / + /// `lastObserved` campaign missing from `campaigns.json`, a witness naming an + /// unresolvable campaign, or a `splitFrom` naming a finding that is gone. + /// + /// An unresolvable **`promotedTo`** is deliberately *not* here: it is + /// [`DiscoveryError::PromotionUnresolved`] (**D3**), because it is a claim about + /// *coverage* rather than about provenance, and folding it in would report a + /// finding that reads as covered while nothing executes it under the same code as a + /// stale campaign pointer. + #[error( + "discovery record `{record}` references {kind} `{reference}`, which does not \ + resolve. Remedy: restore the referenced record or correct the reference — a \ + dangling reference lets the queue claim provenance it does not have." + )] + UnresolvableReference { + record: String, + kind: String, + reference: String, + }, + + /// **D1** — a signature naming a channel absent from `channels.json`. The channel + /// set is closed: a signature over an undeclared channel is a signature nothing + /// observes. + #[error( + "discovery record `{record}` names undeclared channel `{channel}`. Remedy: use \ + one of the channels declared in `conformance/registry/channels.json`, or declare \ + the new channel there first (a new channel also needs a `reg-` regression record)." + )] + UnknownChannel { record: String, channel: String }, + + /// **D2** — a classification that is absent, present too early, or present where it + /// cannot lead anywhere. + /// + /// Three shapes, all one class because all three are the same defect — the queue + /// asserting a judgement nobody made, or making a judgement nobody can act on: + /// + /// - a finding in `triaged` / `promoted` / `no-longer-reproducing` with **no** + /// classification (FR-028's "exactly one" reduced to zero); + /// - a finding in `untriaged` or `split` **carrying** one (an untriaged finding by + /// definition has no judgement, and a split parent surrendered its judgement to its + /// children — a parent that kept one would assert exactly what the split rejected); + /// - a `promoted` finding classified `normalizer-defect` or `fixture-defect`, which + /// describe a defect in the discovery machinery rather than a behavior of either + /// implementation and are therefore non-promotable (FR-035). + #[error( + "discovery finding `{record}` has a classification problem: {cause}. Remedy: \ + record exactly one classification with `discovery triage` once the finding is \ + triaged or later, and none while it is untriaged or a split ancestor — a queue \ + that claims a judgement nobody made is worse than one that admits it has none." + )] + ClassificationArity { record: String, cause: String }, + + /// **D3** — a promotion the registry cannot back: a `promoted` finding with no + /// `promotedTo`, one naming a case the registry does not declare, or a `promotedTo` + /// carried in any state other than `promoted`. + /// + /// One class because all three are the same defect — **the queue claiming coverage + /// that does not exist**. That is worse than an uncovered finding: an uncovered + /// finding is visible in the untriaged bucket and gets reviewed, whereas a promotion + /// nothing executes reads as done and is never looked at again (FR-042's whole + /// purpose is that a promoted finding is not rediscovered and re-triaged). + /// + /// Deliberately checked against the *loaded registry* on every run rather than only + /// at the moment of promotion: a case deleted or renamed afterwards produces exactly + /// this shape, and nothing in the registry can notice, because the registry never + /// reads the queue (research D6). + #[error( + "discovery finding `{record}` has an unresolvable promotion: {cause}. Remedy: \ + author the case with `discovery scaffold`'s skeleton, commit it, and point \ + `promotedTo` at its real id — discovery never writes a case, so a promotion the \ + registry cannot back is a claim nothing executes." + )] + PromotionUnresolved { record: String, cause: String }, + + /// **D4** — a corpus entry that does not name a retrievable, verifiable snapshot: a + /// `commit` that is not a 40-hex object name (a branch, a tag, `HEAD`, `latest`, an + /// abbreviated SHA), a malformed `contentDigest`, an id that does not derive from the + /// entry's own substance, a duplicate id or name, or a digest that was recorded and + /// then removed. + /// + /// The first four are answerable from the manifest alone and are checked + /// hermetically on every pull request — which is the entire reason the manifest is + /// Rust-owned strict JSON rather than a Python tuple (research D8). A validation that + /// only runs when the network is up is a validation that does not run. + #[error( + "corpus entry `{record}`: {cause}. Remedy: pin the entry to a 40-hex commit and \ + leave `contentDigest` to the fetch — the manifest records provenance, and a \ + provenance record that names moving content proves nothing about what was \ + compared." + )] + CorpusIntegrity { record: String, cause: String }, + + /// **D5** — a pinned-input-set element naming a revision absent from + /// `revisions.json`. A finding is a claim about a specific pinned pair of + /// implementations; a pin nothing records is a claim nothing can be checked against. + #[error( + "discovery record `{record}`: pinned input `{element}` names revision `{value}`, \ + which is absent from `conformance/registry/revisions.json`. Remedy: record the \ + revision, or re-evaluate the finding under the current pins — a finding is never \ + carried forward across a pin change unverified." + )] + StalePin { + record: String, + element: String, + value: String, + }, +} + +impl DiscoveryError { + /// The D-class this error belongs to (`"D1"` … `"D5"`). + pub fn class(&self) -> &'static str { + match self { + DiscoveryError::MalformedRecord { .. } + | DiscoveryError::UnresolvableReference { .. } + | DiscoveryError::UnknownChannel { .. } => "D1", + DiscoveryError::ClassificationArity { .. } => "D2", + DiscoveryError::PromotionUnresolved { .. } => "D3", + DiscoveryError::CorpusIntegrity { .. } => "D4", + DiscoveryError::StalePin { .. } => "D5", + } + } + + /// The offending record's id (or the file path, when the record id could not be + /// read because the file itself did not parse). + pub fn record(&self) -> &str { + match self { + DiscoveryError::MalformedRecord { record, .. } + | DiscoveryError::UnresolvableReference { record, .. } + | DiscoveryError::UnknownChannel { record, .. } + | DiscoveryError::ClassificationArity { record, .. } + | DiscoveryError::PromotionUnresolved { record, .. } + | DiscoveryError::CorpusIntegrity { record, .. } + | DiscoveryError::StalePin { record, .. } => record, + } + } +} + /// Atomically write `contents` to `path` (unique temp file + `fs::rename`), creating /// the parent directory if needed. Never leaves a partial file, and a shorter payload /// over a longer one can never leave trailing bytes (the failure mode a plain diff --git a/crates/conformance/src/load.rs b/crates/conformance/src/load.rs index 37f41393..b4ffb9d3 100644 --- a/crates/conformance/src/load.rs +++ b/crates/conformance/src/load.rs @@ -5,7 +5,8 @@ //! //! - collection files (`{ schemaVersion, records }`): `revisions.json`, //! `dimensions.json`, `channels.json`, `profiles.json`, `gaps.json`, -//! `extensions.json`, and `sources/{schema,spec,cli,observed}.json`; +//! `extensions.json`, `metamorphic.json`, and +//! `sources/{schema,spec,cli,observed}.json`; //! - per-area behavior files: `behaviors/*.json` (each a collection); //! - per-area case files: `cases/*.json` (each a collection); //! - per-waiver files: `waivers/*.json` (each a single record object). @@ -24,6 +25,7 @@ use serde::de::DeserializeOwned; use sha2::{Digest, Sha256}; use crate::baseline::BaselineFile; +use crate::discovery::metamorphic::{MetamorphicFile, MetamorphicRelation}; use crate::mapping::{ExceptionMapping, MappingFile, MigrationMapping}; use crate::model::{ BehaviorUnit, CertificationProfile, Classification, ClauseClassification, ClauseInventory, @@ -256,6 +258,14 @@ pub struct Registry { /// scenario model, because "no regression declared" and "the channel is live" are /// not the same claim. pub regressions: Vec, + /// Hand-authored metamorphic relations — `metamorphic.json` + /// (025-exploratory-parity-discovery, US6). The ONE piece of discovery data that lives + /// inside the registry, because a relation is an assertion the project makes and names + /// `clu-`/`bhv-` ids only this loader resolves (research D11). The findings queue — + /// machine-produced, unreviewed — stays a sibling of the registry root, structurally + /// unreachable from here. A missing file is empty; **V32** is what refuses an + /// incomplete set. + pub metamorphic: Vec, } impl Registry { @@ -339,6 +349,10 @@ impl Registry { // regressions.json — the injected-regression records that prove each channel // is live (024 US6). Single-file collection at the registry root. regressions: load_regressions_collection(&root.join("regressions.json"), &mut errors), + // metamorphic.json — the hand-authored `mrl-` relation catalogue (025 US6). + // Single-file collection at the registry root, read exactly like + // `regressions.json`: same envelope, same duplicate-id rejection. + metamorphic: load_metamorphic_collection(&root.join("metamorphic.json"), &mut errors), }; // 022-conformance-runner (T007, FR-003): every case record MUST be exactly one @@ -512,6 +526,30 @@ fn load_regressions_collection( records } +/// Load `metamorphic.json` (025 US6). A missing file yields an empty vector; a malformed +/// one pushes a located [`SchemaError`]. Duplicate ids are reported per record so two +/// records can never silently claim the same identity. +fn load_metamorphic_collection( + path: &Path, + errors: &mut Vec, +) -> Vec { + if !path.exists() { + return Vec::new(); + } + let records = + match read_file(path).and_then(|raw| deserialize_located::(path, &raw)) { + Ok(file) => file.records, + Err(err) => { + errors.push(err); + return Vec::new(); + } + }; + errors.extend(crate::discovery::metamorphic::duplicate_id_errors( + path, &records, + )); + records +} + /// Load the committed baseline at `path` (a sibling of the registry dir, 023). A /// missing file yields `None`; a malformed one pushes a located [`SchemaError`] and /// yields `None` — never a silently empty baseline, which would make every @@ -1282,6 +1320,56 @@ mod tests { ); } + #[test] + fn loads_metamorphic_relations_and_rejects_duplicates() { + let dir = tempfile::tempdir().unwrap(); + write( + dir.path(), + "metamorphic.json", + r#"{ "schemaVersion": 1, "records": [ + { "id": "mrl-key-order-invariance", + "transformation": "permute the key order within an unordered JSON object", + "effect": "invariance", + "ground": "clu-a1b2c3d4", + "channels": ["chan-structured-output"], + "rationale": "Object member order carries no meaning in JSON." } + ] }"#, + ); + let reg = Registry::load(dir.path()).expect("loads"); + assert_eq!(reg.metamorphic.len(), 1); + assert_eq!(reg.metamorphic[0].ground, "clu-a1b2c3d4"); + + // A missing file is empty, never an error (a fixture registry ships none). + let empty = tempfile::tempdir().unwrap(); + assert!( + Registry::load(empty.path()) + .expect("loads") + .metamorphic + .is_empty() + ); + + // Two records claiming one identity is a located schema error, not a silent + // first-match-wins. + write( + dir.path(), + "metamorphic.json", + r#"{ "schemaVersion": 1, "records": [ + { "id": "mrl-x", "transformation": "t1", "effect": "invariance", + "ground": "bhv-a", "channels": ["chan-stdout"], "rationale": "r" }, + { "id": "mrl-x", "transformation": "t2", "effect": "sensitivity", + "ground": "bhv-a", "channels": ["chan-stdout"], "rationale": "r" } + ] }"#, + ); + let err = Registry::load(dir.path()).unwrap_err(); + match err { + LoadError::Schema(errors) => assert!( + errors.iter().any(|e| e.message.contains("mrl-x")), + "the duplicate must be named, got {errors:?}" + ), + other => panic!("expected schema errors, got {other:?}"), + } + } + #[test] fn collects_all_schema_errors_in_one_pass() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/conformance/src/model.rs b/crates/conformance/src/model.rs index bde2ad7e..b3e1d852 100644 --- a/crates/conformance/src/model.rs +++ b/crates/conformance/src/model.rs @@ -76,6 +76,11 @@ pub enum RecordType { /// A hand-authored injected-regression record (`reg-`) — the declarative perturbation /// that proves one observable channel can actually fail (024 US6). Regression, + /// A hand-authored metamorphic relation (`mrl-`) — 025-exploratory-parity-discovery, + /// US6. It lives INSIDE the registry, unlike a discovery finding, because a relation + /// is an assertion the project makes and references `clu-`/`bhv-` ids only the + /// registry loader resolves (research D11). + MetamorphicRelation, } impl RecordType { @@ -102,6 +107,7 @@ impl RecordType { RecordType::Obligation => "obl", RecordType::ObligationDisposition => "odp", RecordType::Regression => "reg", + RecordType::MetamorphicRelation => "mrl", } } @@ -128,6 +134,7 @@ impl RecordType { "obl" => RecordType::Obligation, "odp" => RecordType::ObligationDisposition, "reg" => RecordType::Regression, + "mrl" => RecordType::MetamorphicRelation, _ => return None, }) } diff --git a/crates/conformance/src/parity_corpus.rs b/crates/conformance/src/parity_corpus.rs index 3c92926a..d3c406bd 100644 --- a/crates/conformance/src/parity_corpus.rs +++ b/crates/conformance/src/parity_corpus.rs @@ -72,6 +72,57 @@ pub struct LiveBinary { pub corpus: Option, } +/// What a discovery test binary is *for*, which decides which lanes may select it +/// (025-exploratory-parity-discovery, T060; FR-055/FR-057). +/// +/// The two roles have **opposite** selection requirements, which is exactly why the role +/// is recorded rather than inferred from the `discovery_` name prefix. Inferring from the +/// prefix is the `discovery_*` glob mistake research D9 exists to prevent, one level up: +/// it would make a hermetic guard and a stochastic campaign indistinguishable to every +/// check that reads this file. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum DiscoveryRole { + /// A stochastic campaign binary. Selected ONLY by `[profile.discovery]`; excluded + /// from the `default-filter` of every pull-request profile, so a green PR run never + /// implies a campaign ran. + Live, + /// A hermetic guard. It MUST run in the fast lane (`default` / `dev-fast`) and MUST + /// NOT be captured by `[profile.discovery]`'s allow-list — a guard nobody runs is a + /// guard nobody notices going stale. + Guard, +} + +impl DiscoveryRole { + /// The wire spelling, for diagnostics. + pub fn as_str(self) -> &'static str { + match self { + DiscoveryRole::Live => "live", + DiscoveryRole::Guard => "guard", + } + } +} + +/// One discovery test binary and where its source lives. +/// +/// `tests_dir` is explicit rather than assumed: the discovery binaries are split across +/// two crates on purpose (`discovery_cli` drives the `deacon-conformance` bin and so must +/// live in that crate's test tree, while the campaign binaries and the repository-wiring +/// guard live in `crates/deacon/tests`). A checker that assumed one directory would +/// silently stop covering whichever binary moved. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DiscoveryBinary { + /// The nextest binary name (`.rs` in `tests_dir`). + pub name: String, + /// Live campaign or hermetic guard. + pub role: DiscoveryRole, + /// Workspace-root-relative directory holding `.rs`. + pub tests_dir: String, + /// Whether the binary needs a Docker daemon for any of its tiers. + pub docker_required: bool, +} + /// A case corpus with its minimum expected case count. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -89,6 +140,15 @@ pub struct ParityRegistry { pub live_binaries: Vec, pub internal_consistency_binaries: Vec, pub corpora: Vec, + /// The discovery lane's binaries (025-exploratory-parity-discovery, T060). + /// + /// `#[serde(default)]` so the many synthetic registries in unit tests stay terse; + /// the *real* file's completeness is enforced structurally by + /// `parity_registry_check`, which knows the expected set and fails loudly on an + /// empty or partial one. Defaulting here therefore never hides a missing + /// registration — it only keeps a data-model concern out of the test fixtures. + #[serde(default)] + pub discovery_binaries: Vec, } /// The (symbolic) location of the embedded registry, used in error messages. @@ -157,9 +217,57 @@ impl ParityRegistry { return Err(format!("duplicate corpus id `{}`", c.id)); } } + // Discovery names are unique and disjoint from the parity namespaces. The two + // lanes have contradictory selection rules, so a name claimed by both would make + // the truthfulness invariant unsatisfiable rather than merely ambiguous. + let mut discovery_names = std::collections::HashSet::new(); + for d in &self.discovery_binaries { + if !discovery_names.insert(d.name.as_str()) { + return Err(format!("duplicate discovery binary `{}`", d.name)); + } + if seen.contains(d.name.as_str()) { + return Err(format!( + "`{}` is both a live parity binary and a discovery binary", + d.name + )); + } + if self + .internal_consistency_binaries + .iter() + .any(|n| n == &d.name) + { + return Err(format!( + "`{}` is both an internal-consistency binary and a discovery binary", + d.name + )); + } + if d.tests_dir.trim().is_empty() { + return Err(format!( + "discovery binary `{}` declares no tests_dir — a checker cannot find \ + a source file it is not told where to look for", + d.name + )); + } + } Ok(()) } + /// Every discovery binary of `role`. + pub fn discovery_of_role(&self, role: DiscoveryRole) -> Vec<&DiscoveryBinary> { + self.discovery_binaries + .iter() + .filter(|b| b.role == role) + .collect() + } + + /// The names of every registered discovery binary. + pub fn discovery_names(&self) -> Vec<&str> { + self.discovery_binaries + .iter() + .map(|b| b.name.as_str()) + .collect() + } + /// Look up a corpus by id. pub fn corpus(&self, id: &str) -> Option<&Corpus> { self.corpora.iter().find(|c| c.id == id) @@ -305,6 +413,69 @@ mod tests { ); } + #[test] + fn embedded_registry_enumerates_the_discovery_lane() { + // 025 T060. The *content* of the expected set is asserted by + // `parity_registry_check` (which also cross-checks it against the test tree and + // `.config/nextest.toml`); here we only assert the registry can carry the lane + // and that both roles are represented — a registry with live campaigns and no + // guards, or guards and no campaigns, is a half-wired lane. + let reg = ParityRegistry::load().expect("embedded registry must parse"); + assert!( + !reg.discovery_of_role(DiscoveryRole::Live).is_empty(), + "the discovery lane must register at least one live campaign binary" + ); + assert!( + !reg.discovery_of_role(DiscoveryRole::Guard).is_empty(), + "the discovery lane must register at least one hermetic guard — the guards \ + are what keep the campaigns out of the pull-request lanes" + ); + } + + #[test] + fn rejects_a_discovery_binary_that_collides_with_a_parity_namespace() { + // The two lanes' selection rules are contradictory (a live parity binary must be + // selected by [profile.parity]; a live discovery binary must not be selected by + // it), so one name in both namespaces is unsatisfiable, not merely ambiguous. + let with_live = r#"{ + "live_binaries": [ { "name": "dup", "kind": "scenario", "docker_required": false } ], + "internal_consistency_binaries": [], + "corpora": [], + "discovery_binaries": [ { "name": "dup", "role": "live", "tests_dir": "crates/deacon/tests", "docker_required": false } ] + }"#; + assert!(ParityRegistry::parse(with_live).is_err()); + + let with_consistency = r#"{ + "live_binaries": [], + "internal_consistency_binaries": [ "dup" ], + "corpora": [], + "discovery_binaries": [ { "name": "dup", "role": "guard", "tests_dir": "crates/deacon/tests", "docker_required": false } ] + }"#; + assert!(ParityRegistry::parse(with_consistency).is_err()); + + let duplicated = r#"{ + "live_binaries": [], + "internal_consistency_binaries": [], + "corpora": [], + "discovery_binaries": [ + { "name": "d", "role": "live", "tests_dir": "crates/deacon/tests", "docker_required": false }, + { "name": "d", "role": "guard", "tests_dir": "crates/deacon/tests", "docker_required": false } + ] + }"#; + assert!( + ParityRegistry::parse(duplicated).is_err(), + "one name may not carry two roles — the roles' lane requirements are opposites" + ); + + let no_dir = r#"{ + "live_binaries": [], + "internal_consistency_binaries": [], + "corpora": [], + "discovery_binaries": [ { "name": "d", "role": "live", "tests_dir": " ", "docker_required": false } ] + }"#; + assert!(ParityRegistry::parse(no_dir).is_err()); + } + #[test] fn rejects_overlapping_live_and_consistency() { let bad = r#"{ diff --git a/crates/conformance/src/validate.rs b/crates/conformance/src/validate.rs index f272e37f..0c7c57f2 100644 --- a/crates/conformance/src/validate.rs +++ b/crates/conformance/src/validate.rs @@ -43,7 +43,11 @@ //! on a `non-testable` / `not-applicable` record; //! - **V14** provenance breakage: schemas manifest fingerprint mismatch, an //! inventory `revision` that does not name the registry's `schema`-kind revision, -//! or a committed inventory that no longer byte-matches a fresh regeneration. +//! or a committed inventory that no longer byte-matches a fresh regeneration; +//! - **V31** metamorphic-relation integrity: a missing or unresolvable `ground`, an +//! empty/undeclared `channels` entry, a duplicated `transformation`, or a filler +//! `rationale` (025-exploratory-parity-discovery, US6); +//! - **V32** a mandated metamorphic relation family (FR-044) with no record. //! //! Pure sync file IO only (V1 executable existence, V7 pin file); no Unix-only APIs //! and no path-string parsing, so the crate compiles and validates identically on @@ -56,6 +60,7 @@ use serde::Serialize; use crate::baseline::UnitCategory; use crate::clause::{generate_clauses, render as render_clauses}; +use crate::discovery::metamorphic::MANDATED_RELATIONS; use crate::inventory::{generate_inventory, render}; use crate::load::{ LoadError, Registry, SchemaError, load_clause_inventory, load_inventory, load_spec_manifest, @@ -1652,6 +1657,10 @@ pub fn run(registry: &Registry, today: &str, repo_root: &Path) -> Vec // V30: injected-regression coverage (024, US6). Same scoping as V27–V29 — a registry // that has not opted into the scenario model has not opted into this regime either. out.extend(check_regressions(registry)); + // V31/V32: metamorphic relation integrity + mandated family coverage (025, US6). + // Registry-only, so `report` and `certify` see it too: a relation whose ground stopped + // resolving must not survive until someone happens to run `validate`. + out.extend(check_metamorphic(registry)); sort_violations(&mut out); out } @@ -1878,6 +1887,16 @@ impl<'a> Checker<'a> { .iter() .map(|x| (x.id.as_str(), RecordType::Regression)), ); + // Metamorphic relations are hand-authored registry records too (025 US6) — the one + // piece of discovery data that lives inside the registry, so it obeys the same + // grammar, prefix↔type agreement, and cross-namespace uniqueness. The findings + // queue is deliberately absent: it is a sibling of the registry root, and this + // loader has no path that reaches it. + out.extend( + r.metamorphic + .iter() + .map(|x| (x.id.as_str(), RecordType::MetamorphicRelation)), + ); out } @@ -3865,6 +3884,220 @@ fn kind_target_mismatch(record: &RegressionRecord) -> Option { )) } +// --------------------------------------------------------------------------- +// V31 / V32 — metamorphic relation catalogue (025-exploratory-parity-discovery, US6) +// --------------------------------------------------------------------------- + +/// **V31 — relation integrity** and **V32 — mandated family coverage** +/// (025 US6; data-model.md § 7, contracts/metamorphic-catalogue.md). +/// +/// | Class | What it refuses | Why | +/// |---|---|---| +/// | **V31** | a missing / malformed / unresolvable `ground` | without it a relation records an author's intuition about what *ought* to be irrelevant, and an ungrounded invariance relation that happens to be wrong does not fail — it **passes**, silently, asserting something the specification never said (FR-045) | +/// | **V31** | an empty `channels`, or an entry not in `channels.json` | a relation asserting over no channel, or over one nothing observes, can never fail | +/// | **V31** | a duplicate `transformation` | two records claiming the same transformation are one relation asserted twice, and a failure could not be attributed to either | +/// | **V31** | a filler `rationale` | the ground is only checkable if the argument connecting it to the assertion is written down | +/// | **V32** | a mandated family (FR-044) with no record | a family may not be dropped quietly; the six invariance families and the one sensitivity family are the declared floor | +/// +/// An **unknown `effect`** is listed under V31 by the contract but is refused strictly +/// earlier, at **load**, by the closed [`RelationEffect`] enum — same outcome, better +/// diagnosis, and no record with an unrecognised effect can reach evaluation where +/// "unknown" would have to be resolved into one of the two answers by a default. +/// +/// ## How a ground resolves without the clause inventory +/// +/// A `bhv-` ground resolves against `behaviors/*.json`. A `clu-` ground resolves against +/// the registry's own **clause classifications**: V12 already requires every clause unit +/// to carry exactly one `clc-` record, so "named by a classification" and "is a clause +/// unit" are the same set, and V11 separately reports a classification naming a clause +/// that no longer exists. Resolving this way keeps V31 a pure function of the loaded +/// registry — so `run`, and therefore `report` and `certify`, all see it — instead of +/// scoping it to the one entry point that happens to load the committed inventory. +/// +/// ## Scope +/// +/// Silent for a registry that declares **neither** a metamorphic catalogue nor a scenario +/// model: a fixture predating this feature has not opted into the regime, and reporting +/// seven missing families for each one would say nothing true about them. A registry that +/// declares **either** is checked — in particular, deleting `metamorphic.json` from the +/// real registry is V32 seven times over, not a quiet pass. +pub fn check_metamorphic(registry: &Registry) -> Vec { + let mut out = Vec::new(); + if registry.metamorphic.is_empty() && registry.scenario.is_empty() { + return out; + } + + let behaviors: HashSet<&str> = registry.behaviors.iter().map(|b| b.id.as_str()).collect(); + let clauses: HashSet<&str> = registry + .clause_classifications + .iter() + .filter_map(|c| c.clause.as_deref()) + .collect(); + let channels: HashSet<&str> = registry.channels.iter().map(|c| c.id.as_str()).collect(); + + // -- V31: per-record integrity ----------------------------------------- + let mut transformations: BTreeMap = BTreeMap::new(); + for relation in ®istry.metamorphic { + let ground = relation.ground.trim(); + if ground.is_empty() { + out.push(Violation::new( + "V31", + &relation.id, + "names no `ground`. Every relation must cite a normative clause (`clu-`) or a \ + recorded behavior (`bhv-`): without one the relation asserts an intuition \ + about what ought to be irrelevant, and an ungrounded invariance relation \ + that happens to be wrong does not fail — it passes (FR-045).", + )); + } else { + match parse_id(ground) { + Ok(RecordType::Behavior) if !behaviors.contains(ground) => { + out.push(Violation::new( + "V31", + &relation.id, + format!( + "`ground` {ground:?} names no behavior in the registry — a ground \ + that does not resolve cannot be read, so the assertion cannot be \ + judged" + ), + )) + } + Ok(RecordType::ClauseUnit) if !clauses.contains(ground) => { + out.push(Violation::new( + "V31", + &relation.id, + format!( + "`ground` {ground:?} names no clause unit in the committed clause \ + inventory (no `clc-` record classifies it) — a ground that does \ + not resolve cannot be read, so the assertion cannot be judged" + ), + )) + } + Ok(RecordType::Behavior) | Ok(RecordType::ClauseUnit) => {} + Ok(other) => out.push(Violation::new( + "V31", + &relation.id, + format!( + "`ground` {ground:?} is a {} record; a relation may only be grounded \ + in a normative clause (`clu-`) or a recorded behavior (`bhv-`) \ + (FR-045)", + record_type_name(other) + ), + )), + Err(err) => out.push(Violation::new( + "V31", + &relation.id, + format!("`ground` {ground:?} is not a well-formed record id: {err}"), + )), + } + } + + if relation.channels.is_empty() { + out.push(Violation::new( + "V31", + &relation.id, + "declares no `channels`. A relation that asserts over no observable channel \ + observes nothing, so it can never fail — which is indistinguishable from a \ + relation that always holds.", + )); + } + for channel in &relation.channels { + if !channels.contains(channel.as_str()) { + out.push(Violation::new( + "V31", + &relation.id, + format!( + "asserts over channel {channel:?}, which is not declared in \ + `channels.json` — nothing observes it, so the assertion could never \ + be evaluated" + ), + )); + } + } + + let key = normalize_transformation(&relation.transformation); + if key.is_empty() { + out.push(Violation::new( + "V31", + &relation.id, + "declares an empty `transformation`. The transformation is what a reviewer \ + reproduces and what a failure names; an unnamed one makes the failure \ + unattributable.", + )); + } else if let Some(first) = transformations.insert(key, relation.id.as_str()) { + out.push(Violation::new( + "V31", + &relation.id, + format!( + "declares the same `transformation` as `{first}`. Two records claiming one \ + transformation are one relation asserted twice, and a failure could not \ + be attributed to either." + ), + )); + } + + if is_filler_rationale(&relation.rationale) { + out.push(Violation::new( + "V31", + &relation.id, + format!( + "`rationale` {:?} does not connect the ground to the assertion — say why \ + the cited clause or behavior makes this transformation irrelevant (or \ + significant). A ground nobody argued from is a citation, not a reason.", + relation.rationale + ), + )); + } + } + + // -- V32: mandated family coverage -------------------------------------- + let declared: HashSet<&str> = registry.metamorphic.iter().map(|r| r.id.as_str()).collect(); + for family in MANDATED_RELATIONS { + if !declared.contains(family) { + out.push(Violation::new( + "V32", + *family, + "mandated metamorphic relation family has no record (FR-044). The declared \ + set is a floor, not a suggestion: dropping a family removes an assertion \ + nothing else makes, and the loss is invisible because every remaining \ + relation still passes.", + )); + } + } + + out +} + +/// Whether a relation's `rationale` is a label rather than an argument. +/// +/// A rationale has to carry two claims and the connective between them — *what the cited +/// ground says*, and *why that makes this transformation irrelevant (or significant)*. +/// Twenty words is the shortest such an argument plausibly is; below it the field is a +/// caption on a citation, and FR-045's whole point is that a citation nobody argued from +/// leaves an ungrounded assertion looking grounded. +/// +/// Deliberately **not** [`is_filler_ground`], and the reason is a defect that check would +/// have introduced here: its `VAGUE_CAPABILITY_MARKERS` list is tuned for one-line +/// capability statements, where "later" and "unknown" are evasions. In a paragraph of +/// prose they are ordinary words — the declaration-order rationale legitimately says +/// "later files override earlier ones", quoting the clause it cites, and the shared list +/// rejected it. A marker list tuned for a different field rejects correct text, which is +/// the same reasoning that kept [`is_filler_ground`] from reusing +/// [`names_an_exclusion_ground`]. +fn is_filler_rationale(rationale: &str) -> bool { + rationale.split_whitespace().count() < 20 +} + +/// The comparison key for a relation's `transformation` — case-folded with runs of +/// whitespace collapsed, so two records that differ only in capitalisation or line +/// wrapping still collide as the single relation they are. +fn normalize_transformation(transformation: &str) -> String { + transformation + .split_whitespace() + .map(|w| w.to_ascii_lowercase()) + .collect::>() + .join(" ") +} + /// Whether a behavior applies in a profile's context: every applicability condition /// must be satisfied by the profile's assignment (empty applicability = everywhere). /// A condition on a dimension the profile does not assign is treated as unsatisfied. @@ -3976,6 +4209,7 @@ fn record_type_name(ty: RecordType) -> &'static str { RecordType::Obligation => "obligation", RecordType::ObligationDisposition => "obligation disposition", RecordType::Regression => "injected-regression", + RecordType::MetamorphicRelation => "metamorphic-relation", } } @@ -4670,4 +4904,261 @@ mod tests { "a legacy case cannot join the error-path tier, got {out:?}" ); } + + // ----------------------------------------------------------------------- + // V31 / V32 — metamorphic relation catalogue (025 US6, T088/T089) + // ----------------------------------------------------------------------- + + use crate::discovery::metamorphic::{MetamorphicRelation, RelationEffect}; + + /// A relation record that would validate cleanly, so each test can break exactly one + /// thing and attribute the resulting violation to it. + fn relation(id: &str, ground: &str) -> MetamorphicRelation { + MetamorphicRelation { + id: id.to_string(), + transformation: format!("apply the {id} transformation to the configuration"), + effect: RelationEffect::Invariance, + ground: ground.to_string(), + channels: vec!["chan-structured-output".to_string()], + rationale: "The cited ground fixes the meaning the document carries, so it \ + already decides that this transformation rewrites the spelling and \ + not the value, and the resolved configuration therefore cannot \ + change under it." + .to_string(), + } + } + + /// A registry carrying every mandated family (so V32 is quiet) plus the records the + /// grounds resolve against. + fn metamorphic_registry() -> Registry { + let mut reg = Registry::default(); + reg.channels.push(crate::model::ObservableChannel { + id: "chan-structured-output".to_string(), + description: "structured output".to_string(), + }); + reg.behaviors.push(behavior( + "bhv-ground", + SpecStatus::Conformant, + ReferenceStatus::Aligned, + Decision::FollowSpec, + vec![], + )); + reg.clause_classifications.push(ClauseClassification { + id: "clc-a1b2c3d4".to_string(), + clause: Some("clu-a1b2c3d4".to_string()), + document: None, + disposition: Disposition::BehaviorMapped, + behaviors: vec!["bhv-ground".to_string()], + rationale: None, + notes: None, + }); + for family in MANDATED_RELATIONS { + reg.metamorphic.push(relation(family, "bhv-ground")); + } + reg + } + + /// **T088 / SC-010** — a relation with a missing or unresolvable `ground` is V31. + #[test] + fn v31_flags_a_relation_whose_ground_is_missing_or_unresolvable() { + // Clean baseline: every family present, every ground resolving. + let clean = metamorphic_registry(); + assert!( + check_metamorphic(&clean).is_empty(), + "the baseline catalogue must validate cleanly, got {:?}", + check_metamorphic(&clean) + ); + + // 1. No ground at all. + let mut reg = metamorphic_registry(); + reg.metamorphic[0].ground = String::new(); + let out = check_metamorphic(®); + assert!( + out.iter() + .any(|v| v.code == "V31" && v.record == MANDATED_RELATIONS[0]), + "a groundless relation must be V31, got {out:?}" + ); + + // 2. A `bhv-` ground naming no behavior. + let mut reg = metamorphic_registry(); + reg.metamorphic[1].ground = "bhv-does-not-exist".to_string(); + let out = check_metamorphic(®); + assert!( + out.iter().any(|v| v.code == "V31" + && v.record == MANDATED_RELATIONS[1] + && v.message.contains("bhv-does-not-exist")), + "an unresolvable behavior ground must be V31 naming it, got {out:?}" + ); + + // 3. A `clu-` ground no clause classification names. The registry has exactly one + // classified clause, so a second `clu-` id is genuinely absent. + let mut reg = metamorphic_registry(); + reg.metamorphic[2].ground = "clu-a1b2c3d4".to_string(); + assert!( + check_metamorphic(®).is_empty(), + "a classified clause resolves as a ground" + ); + reg.metamorphic[2].ground = "clu-deadbeef".to_string(); + let out = check_metamorphic(®); + assert!( + out.iter().any(|v| v.code == "V31" + && v.record == MANDATED_RELATIONS[2] + && v.message.contains("clu-deadbeef")), + "an unresolvable clause ground must be V31 naming it, got {out:?}" + ); + + // 4. A ground of the wrong KIND resolves as an id but is not a clause or a + // behavior, so it cannot be read as a justification. + let mut reg = metamorphic_registry(); + reg.metamorphic[3].ground = "wvr-something".to_string(); + let out = check_metamorphic(®); + assert!( + out.iter() + .any(|v| v.code == "V31" && v.record == MANDATED_RELATIONS[3]), + "only a clause or a behavior may ground a relation, got {out:?}" + ); + + // 5. A ground that is not a well-formed id at all. + let mut reg = metamorphic_registry(); + reg.metamorphic[4].ground = "see the spec".to_string(); + let out = check_metamorphic(®); + assert!( + out.iter() + .any(|v| v.code == "V31" && v.record == MANDATED_RELATIONS[4]), + "a free-prose ground must be V31, got {out:?}" + ); + } + + /// **T089** — a mandated family (data-model.md § 7) with no record is V32. + #[test] + fn v32_flags_a_mandated_family_with_no_record() { + for missing in MANDATED_RELATIONS { + let mut reg = metamorphic_registry(); + reg.metamorphic.retain(|r| &r.id.as_str() != missing); + let out = check_metamorphic(®); + assert!( + out.iter().any(|v| v.code == "V32" && &v.record == missing), + "dropping `{missing}` must be V32 naming it — the loss is otherwise \ + invisible, because every remaining relation still passes. Got {out:?}" + ); + assert_eq!( + out.iter().filter(|v| v.code == "V32").count(), + 1, + "exactly the dropped family is reported, got {out:?}" + ); + } + } + + /// The remaining V31 clauses: channels, transformation uniqueness, and rationale. + #[test] + fn v31_flags_undeclared_channels_duplicate_transformations_and_filler_rationale() { + // An empty channel set: the relation observes nothing, so it can never fail. + let mut reg = metamorphic_registry(); + reg.metamorphic[0].channels.clear(); + let out = check_metamorphic(®); + assert!( + out.iter().any(|v| v.code == "V31" + && v.record == MANDATED_RELATIONS[0] + && v.message.contains("no `channels`")), + "a channel-less relation must be V31, got {out:?}" + ); + + // A channel `channels.json` does not declare. + let mut reg = metamorphic_registry(); + reg.metamorphic[0].channels = vec!["chan-invented".to_string()]; + let out = check_metamorphic(®); + assert!( + out.iter() + .any(|v| v.code == "V31" && v.message.contains("chan-invented")), + "an undeclared channel must be V31 naming it, got {out:?}" + ); + + // Two records claiming one transformation — differing only in case and wrapping, + // which is precisely the near-duplicate a byte comparison would miss. + let mut reg = metamorphic_registry(); + reg.metamorphic[1].transformation = reg.metamorphic[0] + .transformation + .to_uppercase() + .replace(' ', "\n "); + let out = check_metamorphic(®); + assert!( + out.iter().any(|v| v.code == "V31" + && v.record == MANDATED_RELATIONS[1] + && v.message.contains(MANDATED_RELATIONS[0])), + "a duplicated transformation must be V31 naming the first claimant, got {out:?}" + ); + + // A rationale that cites without arguing. + let mut reg = metamorphic_registry(); + reg.metamorphic[2].rationale = "obviously fine".to_string(); + let out = check_metamorphic(®); + assert!( + out.iter() + .any(|v| v.code == "V31" && v.record == MANDATED_RELATIONS[2]), + "a filler rationale must be V31, got {out:?}" + ); + } + + /// The rationale test must not inherit V26's capability-marker vocabulary. Those + /// words are evasions in a one-line capability statement and ordinary prose in a + /// paragraph — the committed declaration-order rationale quotes its own clause + /// ("later files override earlier ones") and the shared list rejected it. + #[test] + fn a_rationale_is_judged_on_length_not_on_a_marker_list() { + let quoting_the_clause = "The cited clause states outright that the order of this \ + array matters, because later files override earlier ones, \ + so reversing it describes a different configuration and \ + the result must change."; + assert!( + !is_filler_rationale(quoting_the_clause), + "a rationale that quotes its own clause must not be rejected for the words the \ + clause happens to use" + ); + assert!(is_filler_ground(quoting_the_clause), "…as V26's test would"); + assert!(is_filler_rationale("obviously fine")); + assert!(is_filler_rationale("clu-a1b2c3d4")); + } + + /// A registry that opted into neither regime is silent; one that declares a scenario + /// model but ships no catalogue is not. + #[test] + fn check_metamorphic_is_silent_only_for_a_registry_that_opted_out() { + let empty = Registry::default(); + assert!( + check_metamorphic(&empty).is_empty(), + "a fixture registry predating this feature declares no relations and no \ + scenario model; reporting seven missing families would say nothing true" + ); + + // Opting into the scenario model opts into the catalogue: deleting + // `metamorphic.json` from the real registry is seven V32s, not a quiet pass. + let mut opted_in = Registry::default(); + opted_in.scenario.push(crate::scenario::ScenarioDimension { + id: "sdim-operation".to_string(), + kind: crate::scenario::ScenarioDimensionKind::Scenario, + description: "the consumer operation".to_string(), + values: vec!["up".to_string()], + }); + let out = check_metamorphic(&opted_in); + assert_eq!( + out.iter().filter(|v| v.code == "V32").count(), + MANDATED_RELATIONS.len(), + "every mandated family must be reported missing, got {out:?}" + ); + } + + /// V31/V32 travel with `run`, so `report` and `certify` see them too — not only the + /// `validate` command. + #[test] + fn run_reports_the_metamorphic_classes() { + let mut reg = metamorphic_registry(); + reg.metamorphic[0].ground = String::new(); + reg.metamorphic.pop(); + let out = run(®, "2026-07-19", Path::new("/nonexistent-root")); + assert!(out.iter().any(|v| v.code == "V31"), "got {out:?}"); + assert!(out.iter().any(|v| v.code == "V32"), "got {out:?}"); + // The sort is numeric, so V31/V32 land after V30 rather than between V3 and V4. + assert!(code_rank("V30") < code_rank("V31")); + assert!(code_rank("V31") < code_rank("V32")); + } } diff --git a/crates/conformance/tests/discovery_cli.rs b/crates/conformance/tests/discovery_cli.rs new file mode 100644 index 00000000..cb587f5b --- /dev/null +++ b/crates/conformance/tests/discovery_cli.rs @@ -0,0 +1,800 @@ +//! Exit-status guard for the `discovery` command group +//! (025-exploratory-parity-discovery, contracts/discovery-cli.md). +//! +//! Hermetic: builds a scratch `conformance/` tree pointing at the real registry and +//! drives the `conformance` binary against it. No Docker, no network, no oracle — so it +//! carries no nextest override and runs in every profile, like every other conformance +//! guard. +//! +//! ## Why the exit statuses need their own test +//! +//! The contract's first rule is that a discovery command's status reflects **whether it +//! ran**, never **what it found**. That rule is the entire reason the discovery lane can +//! be scheduled in CI at all: the moment a status depends on findings, the lane becomes a +//! stochastic gate and green stops being reproducible. +//! +//! Nothing else asserts it. The library tests exercise the model and the wiring test +//! parses `.config/nextest.toml`, but the *process contract* — `report` exits `0` on a +//! queue full of untriaged findings, `check` exits `1` on a violation, `scaffold` refuses +//! a non-promotable finding — is only observable from outside the process. + +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +/// The workspace root, derived from this crate's manifest directory. +fn workspace_root() -> PathBuf { + let manifest = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + manifest + .parent() + .and_then(Path::parent) + .map(Path::to_path_buf) + .unwrap_or(manifest) +} + +/// A scratch `conformance/`-shaped tree: the real registry (so channels and revisions +/// resolve) beside a writable discovery root the test owns. +struct Scratch { + dir: tempfile::TempDir, +} + +impl Scratch { + fn new() -> Scratch { + let dir = tempfile::tempdir().expect("tempdir"); + let registry = dir.path().join("registry"); + // A symlink rather than a copy: the registry is large, read-only for every + // command here, and copying it would make this guard a slow test for no gain. + #[cfg(unix)] + std::os::unix::fs::symlink( + workspace_root().join("conformance").join("registry"), + ®istry, + ) + .expect("link the real registry"); + #[cfg(windows)] + std::os::windows::fs::symlink_dir( + workspace_root().join("conformance").join("registry"), + ®istry, + ) + .expect("link the real registry"); + + std::fs::create_dir_all(dir.path().join("discovery")).expect("discovery root"); + let scratch = Scratch { dir }; + scratch.write("findings.json", EMPTY_COLLECTION); + scratch.write("campaigns.json", EMPTY_COLLECTION); + scratch.write("corpus.json", EMPTY_COLLECTION); + scratch + } + + fn registry(&self) -> PathBuf { + self.dir.path().join("registry") + } + + fn write(&self, name: &str, contents: &str) { + std::fs::write(self.dir.path().join("discovery").join(name), contents) + .unwrap_or_else(|e| panic!("write {name}: {e}")); + } + + fn run(&self, args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_conformance")) + .arg("--registry") + .arg(self.registry()) + .args(args) + .output() + .expect("the conformance binary runs") + } +} + +const EMPTY_COLLECTION: &str = "{\n \"schemaVersion\": 1,\n \"records\": []\n}\n"; + +/// A queue holding one untriaged finding and the campaign that admitted it. Ids are the +/// real derived ones, so the fixture passes `check` — the point of several tests below is +/// that a *populated* queue still exits `0` from `report`. +fn populated_queue() -> (String, String, String) { + use deacon_conformance::discovery::queue::{ + Campaign, CampaignLane, CampaignTier, PinnedInputSet, Witness, + }; + use deacon_conformance::discovery::signature::{Divergence, DivergenceKind, Signature}; + + let deacon = serde_json::json!("vscode"); + let reference = serde_json::json!("root"); + let signature = Signature::derive( + "chan-structured-output", + &Divergence { + kind: DivergenceKind::Value, + path: "configuration.remoteUser", + deacon: Some(&deacon), + reference: Some(&reference), + }, + ); + let finding_id = signature.finding_id(); + + // The campaign id is DERIVED, not chosen: `check` recomputes it from the record's own + // substance, so a hand-picked id would (correctly) fail the D1 identity clause and + // this fixture would stop being the clean queue the tests below need. + let seed = "0x5eed1234"; + let profile = "prof-linux-amd64-docker-0870"; + let lane = CampaignLane::Scheduled; + let tier = CampaignTier::ConfigDifferential; + let pins = PinnedInputSet { + schema_pin: deacon_conformance::CURRENT_SCHEMA_PIN.to_string(), + prose_pin: deacon_conformance::CURRENT_SPEC_PIN.to_string(), + oracle_version: "0.87.0".to_string(), + normalizer_version: "6".to_string(), + grammar_version: "rev-schema-113500f4".to_string(), + mutation_catalog_version: "v1".to_string(), + generator_version: "splitmix64-seed+xoshiro256starstar/v1".to_string(), + }; + let campaign_id = Campaign::derive_id(seed, &pins, lane, profile, tier); + let witness_id = Witness::derived_id(&campaign_id, "cnd-11111111"); + + let findings = serde_json::json!({ + "schemaVersion": 1, + "records": [{ + "id": finding_id, + "signature": signature, + "witnesses": [{ + "id": witness_id, + "campaignId": campaign_id, + "candidateId": "cnd-11111111", + "minimalInput": { "image": "alpine:3.18" }, + "isMinimal": true, + "reductionSteps": ["drop-optional-key"], + "observedValues": { "deacon": "vscode", "reference": "root" }, + "mutationOperators": ["mop-wrong-type"] + }], + "classification": null, + "state": "untriaged", + "firstObserved": campaign_id, + "lastObserved": campaign_id, + "promotedTo": null, + "splitFrom": null, + "notes": "" + }] + }); + let campaigns = serde_json::json!({ + "schemaVersion": 1, + "records": [{ + "id": campaign_id, + "seed": seed, + "lane": lane.as_str(), + "tier": tier.as_str(), + "profile": profile, + "pinnedInputSet": pins, + "budget": { + "wallClockSeconds": 1800, + "perCandidateSeconds": 60, + "shrinkStepsPerFinding": 64, + "admissionCap": 25 + }, + "outcome": { + "candidatesGenerated": 4820, + "candidatesExecuted": 4629, + "candidatesDiscardedUnsafe": 0, + "parseStageFailures": 191, + "budgetExhausted": false, + "spaceCoveredFraction": 0.0, + "mutationApplications": { "unknown-field": 512 }, + "signaturesObserved": 1, + "signaturesAdmitted": 1, + "signaturesSuppressed": 0 + } + }] + }); + ( + finding_id, + serde_json::to_string_pretty(&findings).expect("render findings"), + serde_json::to_string_pretty(&campaigns).expect("render campaigns"), + ) +} + +/// A queue holding one finding witnessed by **two** campaigns — the shape a split acts on. +/// +/// Returns `(finding id, findings.json, campaigns.json)`, all ids derived so the fixture +/// passes `check` before the command under test touches it. Built through the library's own +/// `upsert_finding` rather than by hand-writing two witnesses: that is the function which +/// decides a second campaign appends rather than inserts, and a fixture that bypassed it +/// could assert a merge the real deduplication never performs. +fn two_witness_queue() -> (String, String, String) { + use deacon_conformance::discovery::queue::{ + self, Campaign, CampaignLane, CampaignTier, PinnedInputSet, Witness, + }; + use deacon_conformance::discovery::signature::{Divergence, DivergenceKind, Signature}; + + let deacon = serde_json::json!("vscode"); + let reference = serde_json::json!("root"); + let signature = Signature::derive( + "chan-structured-output", + &Divergence { + kind: DivergenceKind::Value, + path: "configuration.remoteUser", + deacon: Some(&deacon), + reference: Some(&reference), + }, + ); + let finding_id = signature.finding_id(); + + let profile = "prof-linux-amd64-docker-0870"; + let lane = CampaignLane::Scheduled; + let tier = CampaignTier::ConfigDifferential; + let pins = PinnedInputSet { + schema_pin: deacon_conformance::CURRENT_SCHEMA_PIN.to_string(), + prose_pin: deacon_conformance::CURRENT_SPEC_PIN.to_string(), + oracle_version: "0.87.0".to_string(), + normalizer_version: "6".to_string(), + grammar_version: "rev-schema-113500f4".to_string(), + mutation_catalog_version: "v1".to_string(), + generator_version: "splitmix64-seed+xoshiro256starstar/v1".to_string(), + }; + + let mut findings = Vec::new(); + let mut campaign_records = Vec::new(); + for (seed, candidate) in [ + ("0x5eed1234", "cnd-11111111"), + ("0x5eed5678", "cnd-22222222"), + ] { + let campaign_id = Campaign::derive_id(seed, &pins, lane, profile, tier); + let witness: Witness = serde_json::from_value(serde_json::json!({ + "id": Witness::derived_id(&campaign_id, candidate), + "campaignId": campaign_id, + "candidateId": candidate, + "minimalInput": { "image": "alpine:3.18" }, + "isMinimal": true, + "reductionSteps": ["drop-optional-key"], + "observedValues": { "deacon": "vscode", "reference": "root" }, + "mutationOperators": ["mop-wrong-type"] + })) + .expect("the synthetic witness must satisfy the strict loader"); + queue::upsert_finding(&mut findings, signature.clone(), witness, &campaign_id); + campaign_records.push(serde_json::json!({ + "id": campaign_id, + "seed": seed, + "lane": lane.as_str(), + "tier": tier.as_str(), + "profile": profile, + "pinnedInputSet": pins, + "budget": { + "wallClockSeconds": 1800, + "perCandidateSeconds": 60, + "shrinkStepsPerFinding": 64, + "admissionCap": 25 + }, + "outcome": { + "candidatesGenerated": 4820, + "candidatesExecuted": 4629, + "candidatesDiscardedUnsafe": 0, + "parseStageFailures": 191, + "budgetExhausted": false, + "spaceCoveredFraction": 0.0, + "mutationApplications": { "unknown-field": 512 }, + "signaturesObserved": 1, + "signaturesAdmitted": 1, + "signaturesSuppressed": 0 + } + })); + } + assert_eq!(findings.len(), 1, "equal signatures are one finding"); + assert_eq!(findings[0].witnesses.len(), 2, "with two witnesses"); + + ( + finding_id, + serde_json::to_string_pretty(&serde_json::json!({ + "schemaVersion": 1, + "records": findings, + })) + .expect("render findings"), + serde_json::to_string_pretty(&serde_json::json!({ + "schemaVersion": 1, + "records": campaign_records, + })) + .expect("render campaigns"), + ) +} + +fn code(output: &Output) -> i32 { + output.status.code().unwrap_or(-1) +} + +fn stdout(output: &Output) -> String { + String::from_utf8_lossy(&output.stdout).into_owned() +} + +fn stderr(output: &Output) -> String { + String::from_utf8_lossy(&output.stderr).into_owned() +} + +#[test] +fn check_exits_zero_on_a_clean_data_root() { + let scratch = Scratch::new(); + let out = scratch.run(&["discovery", "check"]); + assert_eq!(code(&out), 0, "stderr: {}", stderr(&out)); +} + +#[test] +fn check_exits_one_and_reports_every_violation_in_one_pass() { + let scratch = Scratch::new(); + let (_, findings, campaigns) = populated_queue(); + // Two independent violations: an undeclared channel (D1) and an unrecorded oracle + // pin (D5). A checker that stopped at the first would make fixing a batch an + // iterative guessing game. + scratch.write( + "findings.json", + &findings.replace("chan-structured-output", "chan-invented"), + ); + scratch.write("campaigns.json", &campaigns.replace("0.87.0", "9.9.9")); + + let out = scratch.run(&["discovery", "check"]); + assert_eq!(code(&out), 1, "stderr: {}", stderr(&out)); + let report = stdout(&out); + assert!(report.contains("D1 "), "violations go to stdout: {report}"); + assert!(report.contains("D5 "), "both classes reported: {report}"); +} + +#[test] +fn check_json_always_emits_a_document_on_stdout() { + // Including when the data root itself will not load. A consumer doing + // `discovery check --json | jq .ok` must get `false`, never a parse error — that is + // the difference between "the check says no" and "the check crashed". + let scratch = Scratch::new(); + + let clean = scratch.run(&["discovery", "check", "--json"]); + assert_eq!(code(&clean), 0); + let doc: serde_json::Value = serde_json::from_str(&stdout(&clean)).expect("clean run is JSON"); + assert_eq!(doc["ok"], serde_json::json!(true)); + + scratch.write("findings.json", "{ this is not json"); + let broken = scratch.run(&["discovery", "check", "--json"]); + assert_eq!(code(&broken), 1, "stderr: {}", stderr(&broken)); + let doc: serde_json::Value = + serde_json::from_str(&stdout(&broken)).expect("a malformed root still emits JSON"); + assert_eq!(doc["ok"], serde_json::json!(false)); + assert!( + !doc["violations"] + .as_array() + .expect("violations array") + .is_empty(), + "the document must name what failed: {doc}" + ); +} + +#[test] +fn check_refuses_a_truncated_file_rather_than_reading_it_as_empty() { + let scratch = Scratch::new(); + scratch.write("findings.json", "{\n \"schemaVersion\": 1\n}\n"); + let out = scratch.run(&["discovery", "check"]); + assert_eq!( + code(&out), + 1, + "a records-less file is damage, not an empty queue; stderr: {}", + stderr(&out) + ); +} + +#[test] +fn report_exits_zero_whether_the_queue_is_empty_or_full() { + // The never-gates rule (FR-058). Both of these are `0`, and that is the entire + // reason the discovery lane is safe to schedule. + let scratch = Scratch::new(); + let out_dir = scratch.dir.path().join("out"); + + let empty = scratch.run(&[ + "discovery", + "report", + "--out-dir", + &out_dir.to_string_lossy(), + ]); + assert_eq!(code(&empty), 0, "stderr: {}", stderr(&empty)); + assert!(out_dir.join("queue.json").is_file()); + assert!(out_dir.join("queue.md").is_file()); + + let (_, findings, campaigns) = populated_queue(); + scratch.write("findings.json", &findings); + scratch.write("campaigns.json", &campaigns); + let full = scratch.run(&[ + "discovery", + "report", + "--out-dir", + &out_dir.to_string_lossy(), + ]); + assert_eq!( + code(&full), + 0, + "a queue holding findings must still exit 0 — status reflects whether the command \ + ran, never what it found; stderr: {}", + stderr(&full) + ); + let rendered = std::fs::read_to_string(out_dir.join("queue.md")).expect("queue.md"); + assert!(rendered.contains("| untriaged | 1 |"), "{rendered}"); +} + +#[test] +fn report_exits_one_when_it_cannot_write() { + let scratch = Scratch::new(); + // A path whose parent is a regular file cannot be created as a directory. + let blocker = scratch.dir.path().join("blocker"); + std::fs::write(&blocker, "not a directory").expect("write"); + let out = scratch.run(&[ + "discovery", + "report", + "--out-dir", + &blocker.join("nested").to_string_lossy(), + ]); + assert_eq!(code(&out), 1, "stderr: {}", stderr(&out)); +} + +#[test] +fn triage_records_a_classification_and_rejects_an_invalid_one() { + let scratch = Scratch::new(); + let (finding_id, findings, campaigns) = populated_queue(); + scratch.write("findings.json", &findings); + scratch.write("campaigns.json", &campaigns); + + let bad = scratch.run(&[ + "discovery", + "triage", + &finding_id, + "--classification", + "probably-fine", + ]); + assert_eq!(code(&bad), 1); + assert!( + stderr(&bad).contains("deacon-regression"), + "the diagnosis must list the closed set: {}", + stderr(&bad) + ); + + let unknown = scratch.run(&[ + "discovery", + "triage", + "fnd-nowhere", + "--classification", + "deacon-regression", + ]); + assert_eq!(code(&unknown), 1); + + let ok = scratch.run(&[ + "discovery", + "triage", + &finding_id, + "--classification", + "deacon-regression", + "--notes", + "extends child overrides parent remoteUser", + ]); + assert_eq!(code(&ok), 0, "stderr: {}", stderr(&ok)); + + // The write landed and still validates. + assert_eq!(code(&scratch.run(&["discovery", "check"])), 0); + let written = + std::fs::read_to_string(scratch.dir.path().join("discovery").join("findings.json")) + .expect("read back"); + assert!(written.contains("\"classification\": \"deacon-regression\"")); + assert!(written.contains("\"state\": \"triaged\"")); +} + +#[test] +fn triage_does_not_revive_a_no_longer_reproducing_finding() { + // Only a campaign that actually reproduces it may move it back to `triaged` + // (contracts/findings-queue.md, "Reproduction lifecycle"). Setting the state here + // would assert an observation nothing made and silently empty the FR-033 bucket. + let scratch = Scratch::new(); + let (finding_id, findings, campaigns) = populated_queue(); + scratch.write( + "findings.json", + &findings.replace( + "\"state\": \"untriaged\"", + "\"state\": \"no-longer-reproducing\"", + ), + ); + scratch.write("campaigns.json", &campaigns); + + let out = scratch.run(&[ + "discovery", + "triage", + &finding_id, + "--classification", + "deacon-regression", + ]); + assert_eq!(code(&out), 0, "stderr: {}", stderr(&out)); + let written = + std::fs::read_to_string(scratch.dir.path().join("discovery").join("findings.json")) + .expect("read back"); + assert!( + written.contains("\"state\": \"no-longer-reproducing\""), + "the state must survive triage: {written}" + ); + assert!(written.contains("\"classification\": \"deacon-regression\"")); +} + +#[test] +fn scaffold_writes_nothing_and_refuses_a_non_promotable_finding() { + let scratch = Scratch::new(); + let (finding_id, findings, campaigns) = populated_queue(); + scratch.write("findings.json", &findings); + scratch.write("campaigns.json", &campaigns); + + let before = + std::fs::read_to_string(scratch.dir.path().join("discovery").join("findings.json")) + .expect("read"); + let out = scratch.run(&["discovery", "scaffold", &finding_id]); + assert_eq!(code(&out), 0, "stderr: {}", stderr(&out)); + let document: serde_json::Value = + serde_json::from_str(&stdout(&out)).expect("the skeleton is JSON on stdout"); + assert_eq!( + document["behavior"]["id"], + serde_json::json!("bhv-UNREVIEWED") + ); + assert_eq!( + std::fs::read_to_string(scratch.dir.path().join("discovery").join("findings.json")) + .expect("read"), + before, + "scaffold writes NOTHING — there is no code path from a finding to a registry write" + ); + + // `normalizer-defect` describes a defect in the discovery machinery, not a behavior + // of either implementation, so it can never be promoted (FR-035). + scratch.write( + "findings.json", + &findings + .replace( + "\"classification\": null", + "\"classification\": \"normalizer-defect\"", + ) + .replace("\"state\": \"untriaged\"", "\"state\": \"triaged\""), + ); + let refused = scratch.run(&["discovery", "scaffold", &finding_id]); + assert_eq!(code(&refused), 1); + assert!( + stderr(&refused).contains("not promotable"), + "stderr: {}", + stderr(&refused) + ); +} + +#[test] +fn split_requires_something_to_separate() { + let scratch = Scratch::new(); + let (finding_id, findings, campaigns) = populated_queue(); + scratch.write("findings.json", &findings); + scratch.write("campaigns.json", &campaigns); + + // One witness: a split separates witnesses with different causes, so there is + // nothing to separate. + let out = scratch.run(&["discovery", "split", &finding_id]); + assert_eq!(code(&out), 1, "stderr: {}", stderr(&out)); + assert!( + stderr(&out).contains("nothing to separate"), + "{}", + stderr(&out) + ); + + let unknown = scratch.run(&["discovery", "split", "fnd-nowhere"]); + assert_eq!(code(&unknown), 1); +} + +#[test] +fn split_writes_a_lineage_the_checker_accepts_and_the_deduplication_never_re_merges() { + // The whole T070/T072 round trip through the real binary: triage, split, and the queue + // still validates. Asserted from outside the process because the lineage is only worth + // anything if it survives being *written* — an in-memory split that produced records + // the strict loader or `check` rejects would be a split nobody could commit. + let scratch = Scratch::new(); + let (finding_id, findings, campaigns) = two_witness_queue(); + scratch.write("findings.json", &findings); + scratch.write("campaigns.json", &campaigns); + assert_eq!( + code(&scratch.run(&["discovery", "check"])), + 0, + "the fixture must start clean" + ); + + // A split leaves `triaged`, so the finding is classified first. That ordering is the + // lifecycle's, not this test's: a split separates witnesses one CLASSIFICATION could + // not describe, so there has to be one to find inadequate. + let triaged = scratch.run(&[ + "discovery", + "triage", + &finding_id, + "--classification", + "deacon-regression", + ]); + assert_eq!(code(&triaged), 0, "stderr: {}", stderr(&triaged)); + + let split = scratch.run(&["discovery", "split", &finding_id]); + assert_eq!(code(&split), 0, "stderr: {}", stderr(&split)); + assert!( + stderr(&split).contains("2 child finding(s)"), + "one child per witness: {}", + stderr(&split) + ); + assert!( + stderr(&split).contains("never re-merges"), + "the reviewer must be told the split is permanent: {}", + stderr(&split) + ); + + // The written lineage validates: parent inert and unclassified, two children keyed to + // it, each untriaged with its own witness. + assert_eq!( + code(&scratch.run(&["discovery", "check"])), + 0, + "stderr: {}", + stderr(&scratch.run(&["discovery", "check"])) + ); + let written = + std::fs::read_to_string(scratch.dir.path().join("discovery").join("findings.json")) + .expect("read back"); + let document: serde_json::Value = serde_json::from_str(&written).expect("valid JSON"); + let records = document["records"].as_array().expect("records"); + assert_eq!( + records.len(), + 3, + "the ancestor plus two children: {written}" + ); + + let parent = &records[0]; + assert_eq!(parent["id"], serde_json::json!(finding_id)); + assert_eq!(parent["state"], serde_json::json!("split")); + assert_eq!( + parent["classification"], + serde_json::Value::Null, + "a parent that kept a classification would assert the judgement the split rejected" + ); + assert_eq!( + parent["witnesses"].as_array().map(Vec::len), + Some(2), + "the ancestor keeps its witnesses as historical record" + ); + + for child in &records[1..] { + assert_eq!(child["splitFrom"], serde_json::json!(finding_id)); + assert_eq!(child["state"], serde_json::json!("untriaged")); + assert_eq!(child["classification"], serde_json::Value::Null); + assert_eq!(child["witnesses"].as_array().map(Vec::len), Some(1)); + assert_ne!(child["id"], serde_json::json!(finding_id)); + } + assert_ne!(records[1]["id"], records[2]["id"]); + + // A second split is refused: the ancestor is terminal. + let again = scratch.run(&["discovery", "split", &finding_id]); + assert_eq!(code(&again), 1); + + // And the ancestor is no longer the reviewer's to classify — its judgement belongs to + // its children now. + let retriage = scratch.run(&[ + "discovery", + "triage", + &finding_id, + "--classification", + "reference-quirk", + ]); + assert_eq!(code(&retriage), 1, "stderr: {}", stderr(&retriage)); +} + +#[test] +fn the_discovery_group_never_writes_into_the_registry() { + // FR-036 as an observable property: every discovery command runs, and the registry + // tree is byte-identical afterwards. The symlinked registry means a write would land + // in the real one, so this also protects the repository from the test itself. + let scratch = Scratch::new(); + let (finding_id, findings, campaigns) = populated_queue(); + scratch.write("findings.json", &findings); + scratch.write("campaigns.json", &campaigns); + + let registry_dir = workspace_root().join("conformance").join("registry"); + let fingerprint = |dir: &Path| -> Vec<(PathBuf, u64, String)> { + let mut out = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + while let Some(next) = stack.pop() { + for entry in std::fs::read_dir(&next) + .expect("read registry dir") + .flatten() + { + let path = entry.path(); + let meta = entry.metadata().expect("metadata"); + if meta.is_dir() { + stack.push(path); + } else { + let contents = std::fs::read_to_string(&path).unwrap_or_default(); + out.push((path, meta.len(), contents)); + } + } + } + out.sort(); + out + }; + + let before = fingerprint(®istry_dir); + for args in [ + vec!["discovery", "check"], + vec!["discovery", "scaffold", finding_id.as_str()], + vec![ + "discovery", + "triage", + finding_id.as_str(), + "--classification", + "deacon-regression", + ], + ] { + scratch.run(&args); + } + assert_eq!( + before, + fingerprint(®istry_dir), + "no discovery command may alter the conformance registry (FR-036)" + ); +} + +/// **T126 / FR-041**, at the process boundary: `discovery scaffold --tolerate` emits a +/// scoped waiver plus the allowed-difference entry that references it, to **stdout only**. +/// +/// Asserted from outside the process because that is where the contract lives: the library +/// guard in `discovery_hermetic` proves the skeleton is scoped, and this proves the command +/// exits `0`, writes nothing, and puts the document on stdout rather than mixing it into +/// the diagnostics a reviewer pipes past. +#[test] +fn scaffold_tolerate_emits_a_scoped_waiver_and_writes_nothing() { + let scratch = Scratch::new(); + let (finding_id, findings, campaigns) = populated_queue(); + scratch.write("findings.json", &findings); + scratch.write("campaigns.json", &campaigns); + + let before = + std::fs::read_to_string(scratch.dir.path().join("discovery").join("findings.json")) + .expect("read"); + let out = scratch.run(&["discovery", "scaffold", &finding_id, "--tolerate"]); + assert_eq!(code(&out), 0, "stderr: {}", stderr(&out)); + + let document: serde_json::Value = + serde_json::from_str(&stdout(&out)).expect("the tolerance skeleton is JSON on stdout"); + + // Scoped WITHIN a channel, never to one: a bare-channel `observablePath` is a global + // ignore list wearing a waiver id, and the registry refuses one at load (V19). + let path = document["allowedDifference"]["observablePath"] + .as_str() + .expect("an observable path"); + let (channel, rest) = path + .split_once('.') + .unwrap_or_else(|| panic!("`{path}` is a bare channel")); + assert_eq!(channel, "chan-structured-output"); + assert!(!rest.is_empty(), "`{path}` scopes to nothing"); + + // Self-invalidating rather than permanent, and unusable until a human edits it. + assert_eq!( + document["waiver"]["rationale"], + serde_json::json!("UNREVIEWED") + ); + assert_eq!( + document["waiver"]["expires"], + serde_json::json!("UNREVIEWED") + ); + assert_eq!( + document["allowedDifference"]["waiverId"], + document["waiver"]["id"] + ); + + assert_eq!( + std::fs::read_to_string(scratch.dir.path().join("discovery").join("findings.json")) + .expect("read"), + before, + "`--tolerate` writes NOTHING either — the tolerate path is the same review-only \ + discipline as the promotion path, not an exception to it" + ); + + // And the same FR-035 refusal: a defect in the discovery machinery is not a difference + // anyone can legitimately tolerate. + scratch.write( + "findings.json", + &findings + .replace( + "\"classification\": null", + "\"classification\": \"fixture-defect\"", + ) + .replace("\"state\": \"untriaged\"", "\"state\": \"triaged\""), + ); + let refused = scratch.run(&["discovery", "scaffold", &finding_id, "--tolerate"]); + assert_eq!(code(&refused), 1); + assert!( + stderr(&refused).contains("not promotable"), + "stderr: {}", + stderr(&refused) + ); +} diff --git a/crates/deacon/tests/discovery_campaign.rs b/crates/deacon/tests/discovery_campaign.rs new file mode 100644 index 00000000..39b54fc4 --- /dev/null +++ b/crates/deacon/tests/discovery_campaign.rs @@ -0,0 +1,2101 @@ +//! Live discovery campaign binary (025-exploratory-parity-discovery, US1). +//! +//! **Selected ONLY by `[profile.discovery]`.** Every other profile — `default`, +//! `dev-fast`, `full`, `ci`, `mvp-integration`, `parity` — excludes it in its +//! `default-filter`, so those lanes are truthful by *non-selection*: a green pull-request +//! run never implies a campaign ran (FR-055/FR-057). +//! +//! ## What runs here, and what it needs +//! +//! Every test below drives a **real** campaign against the **verified pinned oracle**. +//! There is no mock and no skip: a missing or wrong-version oracle fails the test naming +//! the cause (FR-003), which is the whole point — a harness that quietly passed without +//! the reference would certify nothing while looking green. +//! +//! Each test owns an isolated temporary discovery root, so a campaign here never touches +//! the committed `conformance/discovery/` tree. That is not merely hygiene: the committed +//! queue is a reviewed artifact, and a test that appended to it would make `git status` +//! the review surface for machine output nobody looked at. +//! +//! ## Why this file exists at Phase 2 rather than at US1 +//! +//! The lane wiring (T006/T007/T121) and this binary (T040) are mutually dependent: +//! nextest **hard-fails** a whole config when a `binary(=NAME)` predicate names a binary +//! that does not exist, so wiring the allow-list before the file exists would break every +//! lane in the workspace, not just this one. The file therefore landed with the wiring, +//! carrying the selection guard; US1 fills in the campaign acceptance tests. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use deacon_conformance::discovery::corpus::{self, CorpusEntry}; +use deacon_conformance::discovery::generate::Generator; +use deacon_conformance::discovery::grammar::Grammar; +use deacon_conformance::discovery::mutate; +use deacon_conformance::discovery::queue::{ + self, Budget, CampaignLane, CampaignTier, Classification, DEFAULT_ADMISSION_CAP, + DEFAULT_PER_CANDIDATE_SECONDS_HERMETIC, DiscoveryData, FindingState, +}; +use deacon_conformance::discovery::report::{ + TRIVIAL_FAILURE_CEILING, build_campaign_outcome_report, +}; +use deacon_conformance::load::Registry; +use deacon_conformance::{default_discovery_dir, default_registry_dir}; + +use parity_harness::HarnessError; +use parity_harness::discovery::campaign::{self, CampaignRequest, CampaignRun}; +use parity_harness::discovery::candidate; +use parity_harness::discovery::corpus_fetch::EntryStatus; +use parity_harness::discovery::differential::{ + self, Characterization, DifferentialInput, OutcomeClass, +}; +use parity_harness::oracle::{ORACLE_OVERRIDE_ENV, Oracle, VerifiedOracle}; +use parity_harness::prereq; + +/// The environment variable nextest sets to the profile it selected the run under. +const NEXTEST_PROFILE: &str = "NEXTEST_PROFILE"; + +/// The one profile permitted to select this binary. +const DISCOVERY_PROFILE: &str = "discovery"; + +/// The certification profile these campaigns record themselves under. +const TEST_PROFILE: &str = "prof-linux-amd64-docker-0870"; + +/// If this binary is running under a nextest profile at all, that profile must be +/// `discovery`. +/// +/// This is the lane-isolation invariant enforced from **inside** the thing being isolated, +/// which is a stronger claim than the config cross-check in `discovery_hermetic`: that one +/// asserts the filters *say* the right thing, this one asserts the binary *was not +/// actually selected* by anything else. A future edit that widened a pull-request lane's +/// filter would fail here even if the config still parsed and the cross-check still passed +/// against some other reading of it. +/// +/// An absent `NEXTEST_PROFILE` means the binary was run directly (`cargo test --test +/// discovery_campaign`), which is a deliberate developer act rather than a lane, and is +/// allowed. It is **not** a silent skip: the assertion below has nothing to assert about a +/// run that no profile selected. +#[test] +fn this_binary_runs_only_under_the_discovery_profile() { + let Some(profile) = std::env::var_os(NEXTEST_PROFILE) else { + // Run outside nextest: no profile selected it, so there is no selection to check. + return; + }; + let profile = profile.to_string_lossy().into_owned(); + assert_eq!( + profile, DISCOVERY_PROFILE, + "a live discovery campaign binary was selected by [profile.{profile}]. Only \ + [profile.{DISCOVERY_PROFILE}] may select it — every other lane must be truthful by \ + non-selection, so that a green pull-request run never implies a campaign ran \ + (FR-055/FR-057). Fix the profile's `default-filter` in .config/nextest.toml." + ); +} + +// --------------------------------------------------------------------------- +// Shared scaffolding +// --------------------------------------------------------------------------- + +/// An isolated discovery data root: the three empty collection files a campaign expects. +struct Scratch { + dir: tempfile::TempDir, +} + +impl Scratch { + fn new() -> Scratch { + let dir = tempfile::tempdir().expect("tempdir"); + for name in ["findings.json", "campaigns.json", "corpus.json"] { + std::fs::write( + dir.path().join(name), + "{\n \"schemaVersion\": 1,\n \"records\": []\n}\n", + ) + .expect("seed an empty collection"); + } + Scratch { dir } + } + + /// Replace the scratch root's corpus manifest with `entries` (US7). + /// + /// Routed through the production writer rather than a hand-rolled `serde_json` dump, + /// so a fixture can never hold a manifest the real writer would refuse — and so a + /// digest recorded by the campaign lands in the byte-identical rendering a reviewer + /// would commit. + fn with_corpus(self, entries: &[CorpusEntry]) -> Scratch { + corpus::write(self.dir.path(), entries).expect("seed the scratch corpus manifest"); + self + } + + fn path(&self) -> &Path { + self.dir.path() + } +} + +/// The committed 33-entry manifest, as the campaign would load it. +fn committed_corpus() -> Vec { + DiscoveryData::load(&default_discovery_dir()) + .expect("the committed discovery data root must load") + .corpus +} + +/// The first `n` committed entries. +/// +/// A prefix rather than a hand-picked selection: any subset exercises the same code path, +/// and hand-picking would invite choosing the entries that behave, which is how a canary +/// stops being one. Kept small because each entry costs a network round trip plus two CLI +/// invocations, and a test that takes four minutes is a test people start skipping. +fn corpus_prefix(n: usize) -> Vec { + let mut entries = committed_corpus(); + entries.truncate(n); + assert_eq!( + entries.len(), + n, + "the committed manifest must hold at least {n} entries" + ); + entries +} + +/// A `corpus`-tier campaign request over an isolated data root. +fn corpus_request(seed_hex: &str, seed: u64, scratch: &Scratch) -> CampaignRequest { + CampaignRequest { + seed_hex: seed_hex.to_string(), + seed, + tier: CampaignTier::Corpus, + lane: CampaignLane::Invoked, + profile: TEST_PROFILE.to_string(), + budget: Budget { + wall_clock_seconds: 1800, + per_candidate_seconds: DEFAULT_PER_CANDIDATE_SECONDS_HERMETIC, + shrink_steps_per_finding: 64, + admission_cap: DEFAULT_ADMISSION_CAP, + }, + // Ignored by the corpus driver — the plan is the manifest's own size, because a + // generator denominator has nothing to denominate for a tier that draws nothing. + planned_candidates: 0, + registry_dir: default_registry_dir(), + discovery_dir: scratch.path().to_path_buf(), + report_root: scratch.path().join("artifacts"), + deacon_binary: deacon_binary(), + oracle_override: None, + persist: true, + } +} + +/// The status the run recorded for `entry_id`. +fn status_of<'a>(run: &'a CampaignRun, entry_id: &str) -> &'a EntryStatus { + run.corpus_statuses + .iter() + .find(|s| s.entry_id() == entry_id) + .unwrap_or_else(|| { + panic!( + "no status recorded for corpus entry `{entry_id}`; got {:?}", + run.corpus_statuses + .iter() + .map(EntryStatus::summary) + .collect::>() + ) + }) +} + +/// The deacon binary under test — the artifact cargo just built for this test binary, +/// never a path guessed from `target/`. +fn deacon_binary() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_deacon")) +} + +/// The compiled `discovery-campaign` bin, built and taken from cargo's own artifact report. +/// +/// `env!("CARGO_BIN_EXE_discovery-campaign")` is **not** available here: that macro is +/// defined only for bins of the package the test belongs to, and this bin belongs to +/// `parity-harness` while this test belongs to `deacon`. The alternative — looking under +/// `target/{debug,release}` for whichever file exists — is the guess that 023 T115 caught +/// judging a three-day-old artifact, so the shared [`prereq::cargo_bin`] rule is reused +/// rather than re-derived here. +fn discovery_campaign_binary() -> PathBuf { + runtime() + .block_on(prereq::cargo_bin("parity-harness", "discovery-campaign")) + .expect("the discovery-campaign bin must build; its exit status is the contract") +} + +/// A campaign request over an isolated data root. +fn campaign_request( + seed_hex: &str, + seed: u64, + candidates: u64, + scratch: &Scratch, +) -> CampaignRequest { + CampaignRequest { + seed_hex: seed_hex.to_string(), + seed, + tier: CampaignTier::ConfigDifferential, + lane: CampaignLane::Invoked, + profile: TEST_PROFILE.to_string(), + budget: Budget { + // Generous: these tests bound themselves by candidate count, not by the clock, + // so a slow machine produces a slower run rather than a different one. + wall_clock_seconds: 1800, + per_candidate_seconds: DEFAULT_PER_CANDIDATE_SECONDS_HERMETIC, + // Deliberately far below the bin's 64 (T052). A shrink probe is two CLI + // invocations, and these campaigns admit dozens of new signatures — at the + // production budget a single test would spend hours on reduction. The + // reduction *strategy* is proven exhaustively and hermetically in + // `deacon_conformance::discovery::shrink` (T041–T045); what the live lane has + // to establish is that the campaign WIRES it — that a witness carries a + // genuinely reduced input, an honest `isMinimal`, and named catalogue steps. + // A small budget establishes exactly that, and `candidate_completeness` + // raises it for the one campaign whose subject is the reduction itself. + shrink_steps_per_finding: 4, + admission_cap: DEFAULT_ADMISSION_CAP, + }, + planned_candidates: candidates, + registry_dir: default_registry_dir(), + discovery_dir: scratch.path().to_path_buf(), + report_root: scratch.path().join("artifacts"), + deacon_binary: deacon_binary(), + oracle_override: None, + persist: true, + } +} + +/// Run a campaign, failing the test with the cause on any prerequisite problem. +/// +/// `.expect` rather than a skip: a campaign that cannot verify the oracle has certified +/// nothing, and reporting that as a pass is the exact silent-vacuity failure FR-003 +/// forbids. +fn run_campaign(request: &CampaignRequest) -> CampaignRun { + runtime() + .block_on(campaign::run(request)) + .unwrap_or_else(|e| { + panic!( + "the campaign could not run: {e}\n\nThis lane requires the verified pinned \ + oracle. Install it with `npm i -g @devcontainers/cli@` (see \ + fixtures/parity-corpus/oracle.json), or point DEACON_PARITY_DEVCONTAINER at \ + it. It is never skipped." + ) + }) +} + +fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Runtime::new().expect("async runtime") +} + +fn verified_oracle() -> VerifiedOracle { + runtime() + .block_on(Oracle::acquire()) + .expect("the verified pinned oracle is required by this lane and is never skipped") +} + +// --------------------------------------------------------------------------- +// T025 — seed reproduction (SC-001) +// --------------------------------------------------------------------------- + +/// The same seed and pinned input set produce an identical ordered candidate sequence and +/// an identical finding set. +/// +/// This is the property every recorded finding's reproducibility rests on: a finding names +/// a campaign, a campaign names a seed, and if the seed does not reproduce the run then +/// the finding names nothing anyone can re-examine. +/// +/// Asserted at both levels deliberately. The candidate sequence is checked hermetically +/// (no oracle needed) because a failure there localizes instantly to the generator; the +/// finding set is checked live, because the generator reproducing while the campaign does +/// not would mean the non-determinism is downstream — in normalization, in the tolerance +/// index, or in the admission order — and only the live comparison can see that. +#[test] +fn the_same_seed_reproduces_the_candidate_sequence_and_the_finding_set() { + const SEED: u64 = 0x5EED_0025; + + // 1. The ordered candidate sequence, hermetically. + let grammar = Grammar::load_default().expect("the committed grammar loads"); + let mut left = Generator::new(&grammar, SEED); + let mut right = Generator::new(&grammar, SEED); + let left_ids: Vec = (0..60).map(|_| left.next_candidate().id).collect(); + let right_ids: Vec = (0..60).map(|_| right.next_candidate().id).collect(); + assert_eq!( + left_ids, right_ids, + "the same seed must produce the identical ORDERED candidate sequence" + ); + + // 2. The finding set, live. Two isolated roots, so both runs start from an empty queue + // — comparing a second run that inherited the first's findings would prove nothing. + let first_scratch = Scratch::new(); + let second_scratch = Scratch::new(); + let first = run_campaign(&campaign_request("0x5eed0025", SEED, 6, &first_scratch)); + let second = run_campaign(&campaign_request("0x5eed0025", SEED, 6, &second_scratch)); + + assert_eq!( + first.campaign.id, second.campaign.id, + "the campaign id is derived from the seed and the pinned input set, so two runs of \ + the same seed against the same pins are the same campaign" + ); + + let mut first_findings: Vec = first.findings.iter().map(|f| f.id.clone()).collect(); + let mut second_findings: Vec = second.findings.iter().map(|f| f.id.clone()).collect(); + first_findings.sort(); + second_findings.sort(); + assert_eq!( + first_findings, second_findings, + "the same seed must produce the identical finding set" + ); + + assert_eq!( + first.report.candidates_generated, second.report.candidates_generated, + "reproducibility covers the volume too, or the two runs explored different spaces" + ); + assert_eq!( + first.report.mutation_applications, second.report.mutation_applications, + "the per-category application counts are part of what a seed reproduces" + ); + + // A different seed must not alias, or "reproducible" would be trivially true because + // every campaign produced the same thing. + let other_scratch = Scratch::new(); + let other = run_campaign(&campaign_request("0x5eed0026", SEED + 1, 6, &other_scratch)); + assert_ne!( + other.campaign.id, first.campaign.id, + "a different seed is a different campaign" + ); + let other_ids: Vec = { + let mut g = Generator::new(&grammar, SEED + 1); + (0..60).map(|_| g.next_candidate().id).collect() + }; + assert_ne!(other_ids, left_ids, "a different seed must not alias"); +} + +// --------------------------------------------------------------------------- +// T026 — the trivial-failure ceiling (SC-002) +// --------------------------------------------------------------------------- + +/// At most 10% of generated candidates fail at the document-syntax stage. +/// +/// The ceiling exists because a campaign whose candidates die before configuration +/// resolution is exploring the *parser*, not the tool — and it spends the expensive step +/// (an oracle invocation) to learn nothing. The ratio is reported for every run whether or +/// not it breaches, so a rising trend is visible before it matters. +#[test] +fn the_trivial_failure_ratio_stays_below_the_declared_ceiling() { + let scratch = Scratch::new(); + // Forty candidates rather than a handful: the ceiling is a *proportion*, and over a + // sample of five a single unlucky candidate is 20%. A ratio measured over a sample too + // small to hold it is not a measurement. + let run = run_campaign(&campaign_request("0x5eed0026", 0x5EED_0026, 40, &scratch)); + + assert!( + run.report.candidates_generated >= 40, + "the ratio is meaningless over a sample the campaign never took: {} generated", + run.report.candidates_generated + ); + assert!( + run.report.trivial_failure_fraction <= TRIVIAL_FAILURE_CEILING, + "{} of {} candidates failed at the document-syntax stage ({:.1}%), above the {:.0}% \ + ceiling — the campaign is exploring the parser rather than the tool (SC-002)", + run.report.parse_stage_failures, + run.report.candidates_generated, + run.report.trivial_failure_fraction * 100.0, + TRIVIAL_FAILURE_CEILING * 100.0, + ); + assert!( + !run.report.trivial_failure_ceiling_breached, + "the report must agree with the ratio it computed" + ); + // FR-007 requires the proportion to be REPORTED for every run, not only when it + // breaches. A ceiling nobody can see the distance to is a ceiling nobody manages. + assert_eq!( + run.report.trivial_failure_ceiling, TRIVIAL_FAILURE_CEILING, + "the report states the ceiling it was judged against" + ); +} + +// --------------------------------------------------------------------------- +// T027 — mutation-category coverage (SC-003) +// --------------------------------------------------------------------------- + +/// Every declared mutation category is applied at least once, and **all eleven keys are +/// present** in `mutationApplications` including zeroes. +/// +/// The two halves are different claims and both matter. "Applied at least once" says the +/// campaign explored the whole catalogue. "All eleven keys present" says a category that +/// was *not* applied would be visible as an explicit zero rather than as an absence — +/// FR-010's point, and the difference between a reported generation deficiency and a +/// silent one. +#[test] +fn every_mutation_category_is_applied_and_every_key_is_reported() { + let scratch = Scratch::new(); + let run = run_campaign(&campaign_request("0x5eed0027", 0x5EED_0027, 44, &scratch)); + + let counts = &run.report.mutation_applications; + assert_eq!( + counts.len(), + mutate::CATEGORY_COUNT, + "the report must carry every declared category, got {:?}", + counts.keys().collect::>() + ); + let keys: Vec<&str> = counts.keys().map(String::as_str).collect(); + assert_eq!( + keys, + mutate::category_names(), + "the key list is the catalogue's, in its declaration order" + ); + assert!( + run.report.unapplied_categories.is_empty(), + "categories never applied in a {}-candidate campaign: {:?}. A category with zero \ + successful applications is a hole in what the campaign explored (SC-003).", + run.report.candidates_generated, + run.report.unapplied_categories + ); + for (category, applications) in counts { + assert!( + *applications > 0, + "`{category}` was reported with zero applications while the deficiency list was \ + empty — the two views of the same fact disagree" + ); + } + + // The zero case is representable and reported, not merely absent: rebuild the report + // from a record whose counts are all zero and confirm every key survives. + let mut zeroed = run.campaign.clone(); + zeroed.outcome.mutation_applications = mutate::empty_application_counts(); + let zero_report = build_campaign_outcome_report(&zeroed, &[]); + assert_eq!( + zero_report.mutation_applications.len(), + mutate::CATEGORY_COUNT + ); + assert_eq!( + zero_report.unapplied_categories.len(), + mutate::CATEGORY_COUNT, + "every category with zero applications is named as an explicit deficiency" + ); +} + +// --------------------------------------------------------------------------- +// T028 — the oracle fails loudly (FR-003) +// --------------------------------------------------------------------------- + +/// A missing or wrong-version oracle fails naming the cause, reports no findings, and +/// never skips. +/// +/// The failure mode this guards is the comfortable one: a campaign that could not verify +/// the reference, reported an empty finding set, and exited zero would look exactly like a +/// campaign that found the two implementations in perfect agreement. +/// +/// Unix-only because it drives the verification path with shell stubs. The property is not +/// platform-specific; the stubs are. +#[cfg(unix)] +#[test] +fn an_unverifiable_oracle_fails_naming_the_cause_and_reports_no_findings() { + use std::os::unix::fs::PermissionsExt; + + fn stub(dir: &Path, name: &str, body: &str) -> PathBuf { + let path = dir.join(name); + std::fs::write(&path, body).expect("write stub"); + let mut perms = std::fs::metadata(&path).expect("stat").permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&path, perms).expect("chmod"); + path + } + + let stubs = tempfile::tempdir().expect("tempdir"); + + // 1. Wrong version. + let scratch = Scratch::new(); + let mut wrong = campaign_request("0x5eed0028", 0x5EED_0028, 4, &scratch); + wrong.oracle_override = Some(stub( + stubs.path(), + "wrong-version", + "#!/bin/sh\necho 0.0.1\n", + )); + let err = runtime() + .block_on(campaign::run(&wrong)) + .expect_err("a wrong-version oracle must fail the campaign"); + let message = err.to_string(); + assert!( + matches!(err, HarnessError::OracleUnverified { .. }), + "expected OracleUnverified, got {err:?}" + ); + assert!( + message.contains("0.0.1"), + "the diagnosis must name the version it found: {message}" + ); + assert_findings_untouched(&scratch); + + // 2. Missing binary. + let scratch = Scratch::new(); + let mut missing = campaign_request("0x5eed0028", 0x5EED_0028, 4, &scratch); + missing.oracle_override = Some(stubs.path().join("definitely-not-here")); + let err = runtime() + .block_on(campaign::run(&missing)) + .expect_err("a missing oracle must fail the campaign"); + assert!( + matches!(err, HarnessError::OracleUnverified { .. }), + "expected OracleUnverified, got {err:?}" + ); + assert!( + err.to_string().contains("definitely-not-here"), + "the diagnosis must name the path it could not use: {err}" + ); + assert_findings_untouched(&scratch); + + // 3. A binary that is not the reference at all. + let scratch = Scratch::new(); + let mut garbage = campaign_request("0x5eed0028", 0x5EED_0028, 4, &scratch); + garbage.oracle_override = Some(stub( + stubs.path(), + "not-a-version", + "#!/bin/sh\necho hello there\n", + )); + let err = runtime() + .block_on(campaign::run(&garbage)) + .expect_err("an unidentifiable oracle must fail the campaign"); + assert!( + matches!(err, HarnessError::OracleUnverified { .. }), + "expected OracleUnverified, got {err:?}" + ); + assert_findings_untouched(&scratch); +} + +/// The queue must be untouched after a prerequisite failure: no findings, no campaign. +/// +/// An empty finding set written by a campaign that never compared anything would be a +/// record of an observation nobody made. +#[cfg(unix)] +fn assert_findings_untouched(scratch: &Scratch) { + let data = deacon_conformance::discovery::queue::DiscoveryData::load(scratch.path()) + .expect("the scratch root loads"); + assert!( + data.findings.is_empty(), + "a campaign that could not verify the reference reported findings: {:?}", + data.findings + .iter() + .map(|f| &f.id) + .collect::>() + ); + assert!( + data.campaigns.is_empty(), + "a campaign that never ran must not record itself as having run" + ); +} + +// --------------------------------------------------------------------------- +// T029 — budget exhaustion (FR-005) +// --------------------------------------------------------------------------- + +/// An exhausted budget stops the campaign, sets `budgetExhausted`, and reports the portion +/// of the planned space it covered. +/// +/// The alternative — presenting a truncated run as complete — is the failure FR-005 names: +/// a report that says "no findings" after covering 2% of its plan is making a much smaller +/// claim than it appears to. +#[test] +fn an_exhausted_budget_stops_the_campaign_and_reports_the_covered_fraction() { + let scratch = Scratch::new(); + let mut request = campaign_request("0x5eed0029", 0x5EED_0029, 5_000, &scratch); + // One second of wall clock against a five-thousand-candidate plan: the run cannot + // finish, so exhaustion is the outcome under test rather than a race. + request.budget.wall_clock_seconds = 1; + + let run = run_campaign(&request); + + assert!( + run.report.budget_exhausted, + "a campaign that stopped short of its plan must say so: {} of {} candidates", + run.report.candidates_generated, request.planned_candidates + ); + assert!( + run.report.candidates_generated < request.planned_candidates, + "the run reported exhaustion after covering its whole plan" + ); + assert!( + run.report.space_covered_fraction.is_finite(), + "a non-finite fraction serializes as bare `null` and never loads back" + ); + assert!( + run.report.space_covered_fraction > 0.0 && run.report.space_covered_fraction < 1.0, + "the covered fraction must be a real portion, got {}", + run.report.space_covered_fraction + ); + let expected = run.report.candidates_generated as f64 / request.planned_candidates as f64; + assert!( + (run.report.space_covered_fraction - expected).abs() < 1e-9, + "the fraction must be the portion actually covered: reported {}, computed {expected}", + run.report.space_covered_fraction + ); + + // Exhaustion is a fact about the run, never an error: the status reflects whether the + // campaign ran, not what it found or how far it got (FR-058). + assert!( + run.report.candidates_generated > 0, + "even an exhausted run reports the volume it did reach" + ); + + // And a run that completes its plan is NOT exhausted, or the flag would say nothing. + let complete_scratch = Scratch::new(); + let complete = run_campaign(&campaign_request( + "0x5eed002a", + 0x5EED_002A, + 3, + &complete_scratch, + )); + assert!(!complete.report.budget_exhausted); + assert!((complete.report.space_covered_fraction - 1.0).abs() < 1e-9); +} + +// --------------------------------------------------------------------------- +// T067 — the admission cap (FR-034b, SC-019) +// --------------------------------------------------------------------------- + +/// A campaign that exceeds its admission cap admits at most the cap, reports a **non-zero** +/// suppressed count, and still succeeds. +/// +/// All three clauses carry weight and none implies the others: +/// +/// - **At most the cap** is what keeps the queue reviewable. It is set from reviewer +/// throughput, not machine capacity (research D10): a nightly run that admits more new +/// signatures than anyone clears before the next run has produced a backlog, not +/// coverage. +/// - **A non-zero suppressed count** is what keeps the cap honest. Silent truncation would +/// render "we found 25 things" and "we found many more than we can review" identically — +/// and the second is itself a signal that something systemic is diverging, which is +/// precisely the signal a discovery lane exists to raise. +/// - **Still succeeds** is what keeps discovery from gating on its own output. A campaign +/// that failed when it found too much would make green depend on finding little, and the +/// quickest route to green would be to break the generator. +/// +/// The cap is set to `1` rather than left at the default so the boundary is *reached*: at +/// 25 this test would depend on the two implementations disagreeing two dozen ways, and a +/// run that admitted 3 findings would pass while asserting nothing at all. +#[test] +fn exceeding_the_admission_cap_suppresses_visibly_and_still_succeeds() { + let scratch = Scratch::new(); + let mut request = campaign_request("0x5eed0067", 0x5EED_0067, 60, &scratch); + request.budget.admission_cap = 1; + + // `run_campaign` panics on any prerequisite failure, so reaching the next line IS the + // "still succeeds" clause: a campaign that found more than it could admit ran to + // completion rather than erroring. + let run = run_campaign(&request); + + assert!( + run.report.signatures_admitted <= request.budget.admission_cap, + "the campaign admitted {} signature(s) against a cap of {}", + run.report.signatures_admitted, + request.budget.admission_cap + ); + assert!( + run.report.signatures_suppressed > 0, + "the campaign observed {} distinct signature(s) and admitted {}, yet reported zero \ + suppressed. Either the cap did not engage — in which case this test asserts \ + nothing — or suppression is silent, which is the failure FR-034b exists to \ + prevent. Volume: {} generated, {} executed.", + run.report.signatures_observed, + run.report.signatures_admitted, + run.report.candidates_generated, + run.report.candidates_executed + ); + assert!( + run.report.signatures_admitted + run.report.signatures_suppressed + <= run.report.signatures_observed, + "admitted ({}) + suppressed ({}) cannot exceed observed ({}) — every suppressed \ + signature was observed first", + run.report.signatures_admitted, + run.report.signatures_suppressed, + run.report.signatures_observed + ); + + // The persisted queue holds no more than the cap, so the truncation is real and not + // only reported. + let data = deacon_conformance::discovery::queue::DiscoveryData::load(scratch.path()) + .expect("the written data root must load"); + assert!( + data.findings.len() as u64 <= request.budget.admission_cap, + "the queue holds {} finding(s) against a cap of {}", + data.findings.len(), + request.budget.admission_cap + ); + + // And the suppression reaches the artifacts a reviewer actually reads — both the + // campaign's own report and the standing queue report, which totals suppression across + // every campaign so the queue is legible as a *sample*. + let json = deacon_conformance::discovery::report::render_campaign_json(&run.report); + let document: serde_json::Value = serde_json::from_str(&json).expect("valid JSON"); + assert_eq!( + document["signaturesSuppressed"], + serde_json::json!(run.report.signatures_suppressed), + "the emitted document must carry the suppressed count, not only the in-memory report" + ); + + let registry = + deacon_conformance::load::Registry::load(&default_registry_dir()).expect("registry loads"); + let pins = deacon_conformance::discovery::report::CurrentPins::from_registry(®istry); + let queue_report = deacon_conformance::discovery::report::build_queue_report(&data, &pins); + assert_eq!( + queue_report.signatures_suppressed, run.report.signatures_suppressed, + "the standing queue report must total the campaign's suppression" + ); + let markdown = deacon_conformance::discovery::report::render_md(&queue_report); + assert!( + markdown.contains("This queue is a sample"), + "the human report must say that the counts above it are not everything: {markdown}" + ); + + // The cap is a *cap*, not a fixed size: the same seed with a cap it cannot reach admits + // strictly more, so the numbers above are the cap engaging rather than the run simply + // finding one thing. + // + // The comparison cap is set far above the default, not left at it: the default of 25 is + // reviewer throughput, and this seed genuinely exceeds it (the first attempt at this + // test suppressed 4 signatures in the comparison run), which would have made the + // comparison itself truncated and the conclusion unsound. + let uncapped_scratch = Scratch::new(); + let mut uncapped_request = campaign_request("0x5eed0067", 0x5EED_0067, 60, &uncapped_scratch); + uncapped_request.budget.admission_cap = 10_000; + assert!( + uncapped_request.budget.admission_cap > DEFAULT_ADMISSION_CAP, + "the comparison run must not be capped at reviewer throughput" + ); + let uncapped = run_campaign(&uncapped_request); + assert!( + uncapped.report.signatures_admitted > run.report.signatures_admitted, + "the same seed uncapped admitted {} and capped admitted {} — if they are equal the \ + cap never engaged", + uncapped.report.signatures_admitted, + run.report.signatures_admitted + ); + assert_eq!( + uncapped.report.signatures_suppressed, 0, + "the comparison run must not itself be truncated, or it says nothing about what the \ + capped run left out" + ); +} + +// --------------------------------------------------------------------------- +// T122 — outcomes and structured content, never message wording (FR-016) +// --------------------------------------------------------------------------- + +/// Two rejections that differ only in wording produce **no finding**. +/// +/// Both implementations refuse a malformed document, and both explain themselves in their +/// own words — different words, in different formats, on different streams. If diagnostic +/// prose were part of the comparison, every one of those would be a finding, and the queue +/// would be a list of the two projects' writing styles. +/// +/// The property is structural rather than filtered: there is no code path from a captured +/// stderr into a comparison. This test observes the consequence, and additionally asserts +/// that the two messages really did differ — otherwise it would pass vacuously on a day +/// when both sides happened to say the same thing. +#[test] +fn two_rejections_that_differ_only_in_wording_are_not_a_finding() { + let oracle = verified_oracle(); + let scratch = Scratch::new(); + // A workspace with no configuration at all. Both implementations refuse it, and each + // explains itself in its own words on its own stream. + // + // Note what this fixture is deliberately NOT: a malformed `devcontainer.json`. That + // looks like the obvious choice and is the wrong one — the reference's + // `read-configuration` is a lenient parse-and-echo that ACCEPTS malformed JSONC where + // deacon rejects it, a divergence this project has already characterized + // (`bhv-readconfig-malformed-jsonc-rejected`). Using it here would produce a genuine + // outcome difference and this test would fail for the right reason at the wrong + // question. An absent configuration is the case both sides really do refuse. + let workspace = tempfile::tempdir().expect("tempdir"); + + let result = runtime() + .block_on(differential::compare( + DifferentialInput { + candidate_id: "cnd-wording", + workspace: workspace.path(), + deacon: &deacon_binary(), + oracle: &oracle, + bound: Duration::from_secs(60), + report_root: &scratch.path().join("artifacts"), + deliberately_invalid: true, + }, + &Characterization::default(), + )) + .expect("the comparison itself must succeed"); + + assert_eq!(result.deacon.outcome, OutcomeClass::Rejected); + assert_eq!(result.reference.outcome, OutcomeClass::Rejected); + assert!( + result.parse_stage_failure, + "neither side reached configuration resolution, which is what a document-syntax \ + failure is" + ); + assert!( + result.observations.is_empty(), + "two rejections produced {} observation(s): {:?}. Only outcomes and structured \ + content are compared — never diagnostic message wording (FR-016).", + result.observations.len(), + result + .observations + .iter() + .map(|o| (&o.signature.channel, &o.signature.path)) + .collect::>() + ); + + // The messages really did differ, so the assertion above is not vacuous. + let deacon_stderr = + std::fs::read_to_string(&result.deacon.stderr_path).expect("deacon stderr preserved"); + let reference_stderr = + std::fs::read_to_string(&result.reference.stderr_path).expect("reference stderr preserved"); + assert!( + !deacon_stderr.trim().is_empty(), + "deacon explained its rejection somewhere; if not, this test proves nothing" + ); + assert!( + !reference_stderr.trim().is_empty(), + "the reference explained its rejection somewhere; if not, this test proves nothing" + ); + assert_ne!( + deacon_stderr.trim(), + reference_stderr.trim(), + "the two rejections must actually differ in wording, or the comparison had nothing \ + to ignore" + ); + + // Raw evidence is preserved SEPARATELY from the normalized form (FR-014). Here there + // is no normalized document at all, and the raw bytes are still on disk — which is + // what lets a reviewer see the wording the comparison declined to read. + assert!(result.deacon.normalized.is_none()); + assert!(result.reference.normalized.is_none()); + assert!(result.deacon.stdout_path.is_file()); + assert!(result.reference.stdout_path.is_file()); +} + +// --------------------------------------------------------------------------- +// T123 — zero-finding volume reporting (FR-062) +// --------------------------------------------------------------------------- + +/// A campaign that finds nothing still reports `candidatesGenerated` and +/// `candidatesExecuted`, so "nothing found" is distinguishable from "nothing ran". +/// +/// Without the volume, a broken pipeline is the most comfortable possible state for this +/// machinery to be in: it reports exactly what a clean run reports. +/// +/// Constructed from a **real** campaign rather than a synthetic record. A campaign against +/// the current pins does find differences, so the zero-finding case is produced by +/// rebuilding that run's report with an empty finding list — the volume comes from a run +/// that genuinely happened, and the question under test is whether the report keeps it when +/// there is nothing to show. A purely synthetic record would test the constructor rather +/// than the campaign. +#[test] +fn a_campaign_that_finds_nothing_still_reports_what_it_covered() { + let scratch = Scratch::new(); + let run = run_campaign(&campaign_request("0x5eed0123", 0x5EED_0123, 8, &scratch)); + + // The run really ran. + assert!(run.report.candidates_generated > 0); + assert!(run.report.candidates_executed > 0); + + let nothing_found = build_campaign_outcome_report(&run.campaign, &[]); + assert!(nothing_found.findings.is_empty()); + assert_eq!( + nothing_found.candidates_generated, run.report.candidates_generated, + "the volume must survive having nothing to report" + ); + assert_eq!( + nothing_found.candidates_executed, + run.report.candidates_executed + ); + + let json = deacon_conformance::discovery::report::render_campaign_json(¬hing_found); + let document: serde_json::Value = serde_json::from_str(&json).expect("valid JSON"); + assert_eq!( + document["candidatesGenerated"], run.report.candidates_generated, + "the emitted document carries the volume, not only the in-memory report" + ); + assert_eq!( + document["candidatesExecuted"], + run.report.candidates_executed + ); + assert_eq!(document["findings"], serde_json::json!([])); + + // And the two facts are genuinely distinguishable: a campaign that never ran reports a + // different volume, so a reader can tell them apart. + let mut never_ran = run.campaign.clone(); + never_ran.outcome.candidates_generated = 0; + never_ran.outcome.candidates_executed = 0; + let never_ran = build_campaign_outcome_report(&never_ran, &[]); + assert_ne!( + never_ran.candidates_generated, nothing_found.candidates_generated, + "\"nothing found\" and \"nothing ran\" must not render identically — that \ + indistinguishability is the whole of FR-062" + ); + + let markdown = deacon_conformance::discovery::report::render_campaign_md(¬hing_found); + assert!( + markdown.contains("admitted no findings"), + "the human report must say what the absence does not mean" + ); + assert!(markdown.contains(&run.report.candidates_generated.to_string())); +} + +// --------------------------------------------------------------------------- +// T096 — the metamorphic tier, through the real campaign driver (US6) +// --------------------------------------------------------------------------- + +/// The committed relation catalogue — the metamorphic tier's whole plan. +fn metamorphic_catalogue() -> Vec { + let path = default_registry_dir().join("metamorphic.json"); + let records = deacon_conformance::discovery::metamorphic::load_metamorphic(&path) + .unwrap_or_else(|e| panic!("the committed catalogue at {path:?} must load: {e}")); + assert!( + !records.is_empty(), + "an empty catalogue would make every assertion below vacuous" + ); + records +} + +/// A path that is guaranteed not to be an oracle. +/// +/// `DEACON_PARITY_DEVCONTAINER` (and `CampaignRequest::oracle_override`) pointing at a +/// non-existent file is a hard `HarnessError::OracleMissing` with **no** `PATH` fallback, so +/// it is a deterministic, instant prerequisite failure rather than a slow one. +const ABSENT_ORACLE: &str = "/definitely/not/here/devcontainer"; + +/// The `metamorphic` tier runs through the real driver with **no oracle**, evaluates the +/// whole catalogue, and records itself like any other campaign. +/// +/// The no-prerequisite claim is made non-vacuously: the same request is pointed at an oracle +/// that cannot exist, which any differential tier refuses outright (asserted below). The +/// metamorphic tier must not so much as consult it — research D12's point, and what makes +/// this the one complete vertical slice a contributor with nothing installed can run. +#[test] +fn the_metamorphic_tier_runs_with_no_oracle_and_evaluates_the_whole_catalogue() { + let catalogue = metamorphic_catalogue(); + let scratch = Scratch::new(); + + // `planned_candidates` is deliberately 1 and deliberately wrong: there is nothing to + // generate for this tier, so the plan IS the catalogue. A driver that honored the + // request's figure would stop after one relation and report full coverage of a plan it + // never had. + let mut request = campaign_request("0x5eed0096", 0x5EED_0096, 1, &scratch); + request.tier = CampaignTier::Metamorphic; + request.oracle_override = Some(PathBuf::from(ABSENT_ORACLE)); + + let run = runtime() + .block_on(campaign::run(&request)) + .unwrap_or_else(|e| { + panic!( + "the metamorphic tier must run with no oracle, no Docker, and no network \ + (research D12), but it failed: {e}" + ) + }); + + assert_eq!(run.campaign.tier, CampaignTier::Metamorphic); + assert_eq!( + run.report.candidates_generated as usize, + catalogue.len(), + "every declared relation must be reached — a skipped relation reports nothing, and \ + reporting nothing is indistinguishable from holding" + ); + assert_eq!( + run.report.candidates_executed as usize, + catalogue.len(), + "every reached relation must be evaluated" + ); + assert!( + catalogue.len() > 1, + "the assertions above would be satisfied by the request's planned_candidates of 1 \ + if the catalogue held a single relation, and would prove nothing" + ); + + // The plan is the catalogue, so a complete run covers all of it and is NOT exhausted. + assert!( + !run.report.budget_exhausted, + "a run that reached every relation stopped short of nothing" + ); + assert!( + (run.report.space_covered_fraction - 1.0).abs() < 1e-9, + "the whole catalogue was evaluated, so the covered fraction is 1.0, got {}", + run.report.space_covered_fraction + ); + + // No mutation occurs in this tier — and FR-010 still requires every category to be + // present as an explicit zero rather than absent, so an unapplied category is reported + // rather than inferred from a map a reader would have to cross-check. + assert_eq!( + run.report.mutation_applications.len(), + mutate::CATEGORY_COUNT + ); + assert!(run.report.mutation_applications.values().all(|&n| n == 0)); + assert_eq!( + run.report.unapplied_categories.len(), + mutate::CATEGORY_COUNT, + "no mutation occurs here, so every category is an explicit zero" + ); + // There is no document-syntax stage: the fixtures are authored, not drawn. + assert_eq!(run.report.parse_stage_failures, 0); + + // The pinned input set has no optional elements (FR-002): the tier never invokes the + // reference, but the pin it WOULD have been compared against is still recorded. + assert!( + !run.report.pinned_input_set.oracle_version.trim().is_empty(), + "the oracle pin is part of what makes a metamorphic finding checkable, even though \ + the tier never invokes the reference" + ); + + // Admission bookkeeping and the queue agree. Asserted structurally rather than as a + // finding count: the tier's status must not depend on what it found (FR-058), and the + // relations' own verdicts are `discovery_metamorphic`'s subject, not this test's. + assert_eq!(run.report.signatures_admitted as usize, run.admitted.len()); + assert_eq!( + run.findings.len(), + run.admitted.len(), + "the scratch queue started empty, so every finding in it is one this run admitted" + ); + + // It really was persisted, so the write path is exercised rather than assumed. + let data = deacon_conformance::discovery::queue::DiscoveryData::load(scratch.path()) + .expect("the scratch root loads"); + assert_eq!(data.campaigns.len(), 1); + assert_eq!(data.campaigns[0].id, run.campaign.id); + assert_eq!(data.campaigns[0].tier, CampaignTier::Metamorphic); + assert_eq!(data.findings.len(), run.findings.len()); + + // The no-oracle claim is not vacuous: the SAME override fails a differential tier + // outright, so the metamorphic run above genuinely never consulted it. + let differential_scratch = Scratch::new(); + let mut differential = campaign_request("0x5eed0096", 0x5EED_0096, 1, &differential_scratch); + differential.oracle_override = Some(PathBuf::from(ABSENT_ORACLE)); + let err = runtime() + .block_on(campaign::run(&differential)) + .expect_err("a differential tier must refuse an oracle that cannot exist"); + assert!( + matches!(err, HarnessError::OracleUnverified { .. }), + "expected OracleUnverified, got {err:?}" + ); +} + +// --------------------------------------------------------------------------- +// T059 — the bin's exit-status contract, observed from OUTSIDE the process +// --------------------------------------------------------------------------- + +/// Run the compiled `discovery-campaign` bin. +/// +/// The child is pointed at this test binary's own deacon through the documented +/// `prereq::DEACON_BIN_ENV` seam, so the bin under test judges the same artifact every other +/// test here does — and does not spend a nested cargo build discovering it. +fn run_discovery_campaign( + bin: &Path, + args: &[&str], + envs: &[(&str, &str)], +) -> std::process::Output { + let mut command = std::process::Command::new(bin); + command + .args(args) + .env(prereq::DEACON_BIN_ENV, deacon_binary()) + .stdin(std::process::Stdio::null()); + for (key, value) in envs { + command.env(key, value); + } + command + .output() + .unwrap_or_else(|e| panic!("could not run {bin:?}: {e}")) +} + +fn exit_code(output: &std::process::Output) -> i32 { + output + .status + .code() + .unwrap_or_else(|| panic!("the bin was killed by a signal: {:?}", output.status)) +} + +/// A campaign that **ran** exits `0`, whatever it found. +/// +/// Driven through the compiled binary rather than through `campaign::run`, on purpose. The +/// library-level half of this contract is already covered (`discovery_hermetic`'s T055, and +/// `discovery_cli`'s process-level guard for the hermetic commands); what is unproven +/// without this is that `main`'s own translation from a `Result` into an `ExitCode` matches +/// the contract. That translation is where the contract is actually enforced, and a doc +/// comment describing it is not evidence. +/// +/// The `metamorphic` tier is used because it needs no oracle, no Docker, and no network +/// (research D12), so the test states a property of the *status* rather than of the +/// environment. `--dry-run` keeps the committed `conformance/discovery/` tree untouched. +#[test] +fn the_bin_exits_zero_when_the_campaign_ran_whatever_it_found() { + let bin = discovery_campaign_binary(); + let output = run_discovery_campaign( + &bin, + &["--seed", "0x5eed0059", "--tier", "metamorphic", "--dry-run"], + &[], + ); + + let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); + assert_eq!( + exit_code(&output), + 0, + "a campaign that ran must exit 0 regardless of what it found (FR-058). Any command \ + whose status depends on its findings becomes a gate the moment someone wires it \ + into CI, and a stochastic gate makes green non-reproducible.\nstderr:\n{stderr}" + ); + + // stdout carries the single JSON outcome and nothing else, so a caller can pipe it into + // `jq` without the diagnostics landing in it. + let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); + let document: serde_json::Value = serde_json::from_str(&stdout) + .unwrap_or_else(|e| panic!("stdout must be a single JSON document ({e}): {stdout}")); + assert_eq!(document["tier"], "metamorphic"); + assert_eq!(document["seed"], "0x5eed0059"); + assert!( + document["candidatesGenerated"].as_u64().unwrap_or(0) > 0, + "the outcome must report the volume it covered, or \"nothing found\" and \"nothing \ + ran\" are indistinguishable (FR-062): {document}" + ); + assert!( + document["findings"].is_array(), + "the finding list is reported in the document, never in the exit status: {document}" + ); + assert!( + !stderr.trim().is_empty(), + "diagnostics belong on stderr; an empty stderr means the run said nothing about \ + itself" + ); +} + +/// A malformed invocation exits `2` — never `1`, which is reserved for machinery failure, +/// and never `0`. +/// +/// The three statuses are three different facts, and collapsing any two of them leaves a +/// scheduled lane unable to tell "the campaign could not run" from "the invocation was +/// wrong", which are fixed in entirely different places. +#[test] +fn a_malformed_invocation_exits_two() { + let bin = discovery_campaign_binary(); + + // `--seed` is required and never defaulted (FR-001). + let missing_seed = run_discovery_campaign(&bin, &["--tier", "metamorphic", "--dry-run"], &[]); + assert_eq!( + exit_code(&missing_seed), + 2, + "a missing `--seed` is a usage error, not a machinery failure" + ); + let stderr = String::from_utf8_lossy(&missing_seed.stderr).into_owned(); + assert!( + stderr.contains("--seed"), + "the diagnosis must name the missing flag: {stderr}" + ); + assert!( + stderr.contains("usage:"), + "a usage error must print the usage line: {stderr}" + ); + assert!( + missing_seed.stdout.is_empty(), + "a run that never happened must not emit a campaign outcome document" + ); + + // Every other malformed form takes the same status, so a caller can branch on it. + for (label, args) in [ + ("a missing --tier", vec!["--seed", "0x1"]), + ( + "an unknown tier", + vec!["--seed", "0x1", "--tier", "not-a-tier"], + ), + ( + "a non-hex seed", + vec!["--seed", "zzz", "--tier", "metamorphic"], + ), + ( + "a zero candidate plan", + vec![ + "--seed", + "0x1", + "--tier", + "config-differential", + "--candidates", + "0", + ], + ), + ( + "an unknown argument", + vec!["--seed", "0x1", "--tier", "metamorphic", "--nope"], + ), + ( + "a flag with no value", + vec!["--tier", "metamorphic", "--seed"], + ), + ] { + let output = run_discovery_campaign(&bin, &args, &[]); + assert_eq!( + exit_code(&output), + 2, + "{label} must exit 2: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert!( + output.stdout.is_empty(), + "{label} produced a campaign outcome on stdout for a run that never happened" + ); + } +} + +// --------------------------------------------------------------------------- +// T046 — candidate completeness (SC-005, FR-024 – FR-027) +// --------------------------------------------------------------------------- + +/// Every emitted candidate carries all six parts and is reproducible from the candidate +/// plus the pins it names. +/// +/// The six parts are not six nice-to-haves; each answers a question a reviewer cannot +/// otherwise answer, and the whole point of FR-027's self-containment is that answering +/// them must not require re-running the campaign that found the difference. So the +/// assertions below go past "the file exists": the fixture is *materialized and re-run* +/// against both implementations, and the difference the candidate claims has to still be +/// there. +/// +/// The reproduction is the load-bearing half. A candidate whose six files were all present +/// but whose fixture no longer produced the difference would pass a file-existence check +/// while being worthless — and that is the failure mode minimization introduces, because +/// minimization is precisely the step that rewrites the fixture. +#[test] +fn every_emitted_candidate_has_all_six_parts_and_reproduces_from_them() { + let scratch = Scratch::new(); + // A small plan with a real shrink budget: this is the one campaign whose subject IS + // the reduction, so it pays for it rather than borrowing the shared helper's floor. + let mut request = campaign_request("0x5eed0046", 0x5EED_0046, 4, &scratch); + request.budget.shrink_steps_per_finding = 24; + let run = run_campaign(&request); + + assert!( + !run.candidates.is_empty(), + "this campaign emitted no candidates, so the test asserts nothing. Volume: {} \ + generated, {} executed, {} finding(s) admitted.", + run.report.candidates_generated, + run.report.candidates_executed, + run.findings.len() + ); + + let oracle = verified_oracle(); + let registry = + deacon_conformance::load::Registry::load(&default_registry_dir()).expect("registry loads"); + let behaviors: Vec<&str> = registry.behaviors.iter().map(|b| b.id.as_str()).collect(); + let mut reproduced = 0usize; + + for finding_id in &run.candidates { + let finding = run + .findings + .iter() + .find(|f| &f.id == finding_id) + .unwrap_or_else(|| { + panic!( + "a candidate was emitted for `{finding_id}`, which the queue does not \ + carry — a reviewable artifact for a finding nobody can find" + ) + }); + let dir = candidate::candidate_dir(&run.candidates_root, &finding.id); + assert!( + dir.is_dir(), + "no reviewable candidate was emitted for admitted finding `{}` at {}", + finding.id, + dir.display() + ); + assert!( + candidate::missing_parts(&dir).is_empty(), + "candidate `{}` is missing {:?}. SC-005 is 100% of six parts, not most of \ + them — a candidate short a part sends the reviewer back to the campaign.", + finding.id, + candidate::missing_parts(&dir) + ); + + let parts = candidate::read_parts(&dir); + assert_eq!(parts.len(), 5, "every JSON part must parse: {parts:?}"); + + // (2) Context: the operations, the campaign, the seed, and the FULL pinned input + // set — FR-026's "the pins it names", which is what makes FR-027 checkable. + let context = &parts["context.json"]; + assert_eq!(context["findingId"], finding.id.as_str()); + assert_eq!(context["campaignId"], run.campaign.id.as_str()); + assert_eq!(context["seed"], run.campaign.seed.as_str()); + let operations = context["operations"] + .as_array() + .expect("the operations are an array"); + assert!( + !operations.is_empty(), + "a candidate with no operation names no run" + ); + assert_eq!(operations[0]["subcommand"], "read-configuration"); + assert!( + operations[0]["argv"] + .as_array() + .is_some_and(|a| a.iter().any(|v| v == "${WORKSPACE}")), + "operations must be ${{WORKSPACE}}-tokenized, the way a case's are: {}", + operations[0] + ); + for pin in [ + "schemaPin", + "prosePin", + "oracleVersion", + "normalizerVersion", + "grammarVersion", + "mutationCatalogVersion", + "generatorVersion", + ] { + assert!( + context["pinnedInputSet"][pin] + .as_str() + .is_some_and(|s| !s.is_empty()), + "the pinned input set has no optional elements (FR-002/FR-026); `{pin}` \ + is missing from candidate `{}`", + finding.id + ); + } + + // (3)/(4) Raw and normalized are SEPARATE documents (T050, FR-014), and neither is + // a view of the other. + let raw = &parts["raw.json"]; + let normalized = &parts["normalized.json"]; + for side in ["deacon", "reference"] { + assert!( + raw[side]["outcome"].is_string(), + "raw evidence must record each side's outcome: {raw}" + ); + assert!( + raw[side].get("stdout").is_some() && raw[side].get("stderr").is_some(), + "raw evidence must carry the verbatim streams, or the reviewer cannot see \ + the wording the comparison declined to read: {raw}" + ); + } + assert!( + raw.get("difference").is_none(), + "the raw document must not carry the comparison's conclusion; conflating the \ + two is exactly what makes a `normalizer-defect` unreachable (FR-014)" + ); + assert_eq!( + normalized["difference"]["signature"]["id"], + finding.signature.id.as_str(), + "the normalized document must name the difference the finding is about" + ); + assert_eq!( + normalized["difference"]["channel"], + finding.signature.channel.as_str() + ); + assert_eq!( + normalized["difference"]["path"], + finding.signature.path.as_str() + ); + + // (5) Reference provenance, and how the input was produced and reduced. + let provenance = &parts["provenance.json"]; + // A campaign compares against the VERIFIED pinned oracle, and the provenance says + // which of the two comparison shapes produced this candidate. The `kind` is + // asserted first because it is what a reviewer reads to tell a real divergence + // from an injected self-comparison (the FR-042a pipeline proof records the other + // variant, `injected-self-comparison`), and a candidate that claimed the wrong one + // would be a lie in the file whose only job is to say what the evidence is. + assert_eq!( + provenance["reference"]["kind"], + serde_json::json!("verified-oracle") + ); + assert_eq!(provenance["reference"]["version"], oracle.version.as_str()); + assert_eq!(provenance["reference"]["verified"], serde_json::json!(true)); + assert!(provenance["reduction"]["isMinimal"].is_boolean()); + if provenance["reduction"]["isMinimal"] == serde_json::json!(false) { + assert!( + provenance["reduction"]["notMinimalReason"] + .as_str() + .is_some_and(|r| !r.trim().is_empty()), + "a reduction that is not minimal must say WHY (FR-022): {provenance}" + ); + } + for step in provenance["reduction"]["steps"] + .as_array() + .expect("the applied steps are an array") + { + let step = step.as_str().expect("a step name is a string"); + assert!( + deacon_conformance::discovery::shrink::REDUCTION_STEPS.contains(&step), + "`{step}` is not a declared catalogue step" + ); + } + + // (6) The suggested mapping: a resolvable id, or an explicit no-match. NEVER an + // invented identity (FR-025) — a fabricated id would make the reviewer's job + // verifying a plausible-looking mapping rather than deciding one. + let mapping = &parts["mapping.json"]; + match mapping["match"].as_str() { + Some("behavior") => { + let suggested = mapping["behavior"].as_str().expect("a behavior id"); + assert!( + behaviors.contains(&suggested), + "candidate `{}` suggests `{suggested}`, which does not resolve in \ + conformance/registry/behaviors/*.json", + finding.id + ); + } + Some("none") => { + assert!( + mapping["reason"] + .as_str() + .is_some_and(|r| !r.trim().is_empty()), + "a no-match must say why it found none: {mapping}" + ); + for considered in mapping["considered"].as_array().unwrap_or(&Vec::new()) { + let id = considered.as_str().expect("a behavior id"); + assert!( + behaviors.contains(&id), + "even a REJECTED suggestion must resolve; `{id}` does not" + ); + } + } + other => panic!("mapping.json must state `behavior` or `none`, got {other:?}"), + } + + // (1) The fixture, and the reproduction claim itself. The candidate's fixture tree + // is copied into a fresh workspace and both implementations are re-run over it; the + // difference the candidate claims must still be observable there. + let workspace = tempfile::tempdir().expect("tempdir"); + copy_tree(&dir.join("fixture"), workspace.path()); + assert!( + workspace + .path() + .join(".devcontainer") + .join("devcontainer.json") + .is_file(), + "the fixture must be a materializable workspace tree, not a bare document" + ); + + let replay = runtime() + .block_on(differential::compare( + DifferentialInput { + candidate_id: &format!("replay-{}", finding.id), + workspace: workspace.path(), + deacon: &deacon_binary(), + oracle: &oracle, + bound: Duration::from_secs(60), + report_root: &scratch.path().join("replay"), + deliberately_invalid: true, + }, + &Characterization::default(), + )) + .expect("the replay comparison itself must succeed"); + assert!( + replay + .observations + .iter() + .any(|o| o.signature.id == finding.signature.id), + "candidate `{}` does not reproduce from its own fixture and pins (FR-027 / \ + SC-005). Expected signature `{}` at `{}`; the replay observed {:?}. A \ + candidate that cannot be replayed is a report about a comparison nobody can \ + repeat.", + finding.id, + finding.signature.id, + finding.signature.path, + replay + .observations + .iter() + .map(|o| (o.signature.channel.as_str(), o.signature.path.as_str())) + .collect::>() + ); + reproduced += 1; + } + + assert!( + reproduced > 0, + "no candidate was replayed, so nothing was proven" + ); + assert_eq!( + reproduced, + run.candidates.len(), + "every emitted candidate must be checked; SC-005 is 100%, not a sample" + ); + + // A finding in the queue without a candidate is permitted in exactly one case: it was + // admitted from a minimization DRIFT (FR-023), which has no two-sided evidence set + // behind it. Any other uncovered finding means the campaign admitted something it then + // gave the reviewer no way to look at. + for finding in &run.findings { + if run.candidates.contains(&finding.id) { + continue; + } + assert!( + finding + .witnesses + .iter() + .all(|w| w.reduction_steps.is_empty() && !w.is_minimal), + "finding `{}` has no reviewable candidate and does not look like a \ + minimization drift — a finding admitted from a comparison must be packaged \ + for review (FR-024)", + finding.id + ); + } + + // Minimization really ran, rather than every witness having quietly taken the + // not-attempted path. + assert!( + run.shrink_probes > 0, + "the campaign spent zero shrink probes across {} finding(s): minimization is \ + wired in name only", + run.findings.len() + ); + let witnesses: Vec<&deacon_conformance::discovery::queue::Witness> = + run.findings.iter().flat_map(|f| &f.witnesses).collect(); + assert!( + witnesses.iter().any(|w| !w.reduction_steps.is_empty()), + "no witness records a single applied reduction step, so no finding was actually \ + reduced" + ); + for witness in &witnesses { + assert!( + witness.minimal_input.is_object(), + "a witness must carry a materializable document" + ); + for step in &witness.reduction_steps { + assert!( + deacon_conformance::discovery::shrink::REDUCTION_STEPS.contains(&step.as_str()), + "witness step `{step}` is not a declared catalogue step" + ); + } + } +} + +/// Copy a directory tree, so a candidate's fixture can be materialized somewhere fresh. +/// +/// The replay must not run *in* the candidate directory: a comparison writes raw artifacts, +/// and a reviewable candidate that gained files by being reviewed is no longer the artifact +/// the campaign emitted. +fn copy_tree(from: &Path, to: &Path) { + std::fs::create_dir_all(to).expect("create the destination"); + for entry in std::fs::read_dir(from).expect("read the fixture tree") { + let entry = entry.expect("a readable entry"); + let target = to.join(entry.file_name()); + if entry.file_type().expect("file type").is_dir() { + copy_tree(&entry.path(), &target); + } else { + std::fs::copy(entry.path(), &target).expect("copy a fixture file"); + } + } +} + +// --------------------------------------------------------------------------- +// T124 — the container tier is separately selectable (FR-060) +// --------------------------------------------------------------------------- + +/// A path that is guaranteed not to be a Docker CLI. +const ABSENT_DOCKER: &str = "/definitely/not/here/docker"; + +/// The container-backed tier is selectable **independently** of the configuration-resolution +/// tier, so a campaign runs where containers are unavailable. +/// +/// Asserted as a pair, because either half alone proves nothing: +/// +/// - With Docker made unreachable, `config-differential` still runs to completion and exits +/// `0`. That is FR-060's actual promise — the tier a contributor without a working +/// container runtime can run. +/// - With the *same* environment, `container-differential` fails at its Docker prerequisite +/// and exits `1`. Without this half, the first would be satisfied by a driver that never +/// probes Docker at all, and the "separately selectable" claim would be vacuous: two tiers +/// with identical prerequisites are not separately selectable, they are the same tier +/// twice. +/// +/// Driven through the compiled bin as a subprocess rather than through `campaign::run`, +/// because the environment is the subject: `std::env::set_var` is `unsafe` under this +/// workspace's edition (and process-global, so hostile to a parallel runner), and the +/// documented `DEACON_PARITY_DOCKER` seam is read by the child. +#[test] +fn the_container_tier_is_selectable_independently_of_the_configuration_tier() { + let bin = discovery_campaign_binary(); + let no_docker = [(prereq::DOCKER_OVERRIDE_ENV, ABSENT_DOCKER)]; + + // 1. The configuration-resolution tier runs where containers are unavailable. + let hermetic = run_discovery_campaign( + &bin, + &[ + "--seed", + "0x5eed0124", + "--tier", + "config-differential", + "--candidates", + "2", + "--dry-run", + ], + &no_docker, + ); + let stderr = String::from_utf8_lossy(&hermetic.stderr).into_owned(); + assert_eq!( + exit_code(&hermetic), + 0, + "the configuration-resolution tier must run with no container runtime at all \ + (FR-060). It compares two CLIs over a workspace tree and brings nothing up, so a \ + Docker dependency here would be a dependency it does not have.\nstderr:\n{stderr}" + ); + let document: serde_json::Value = + serde_json::from_str(&String::from_utf8_lossy(&hermetic.stdout)) + .unwrap_or_else(|e| panic!("stdout must be a single JSON document ({e}):\n{stderr}")); + assert_eq!(document["tier"], "config-differential"); + assert!( + document["candidatesExecuted"].as_u64().unwrap_or(0) > 0, + "the campaign must have actually compared something; otherwise \"it ran without \ + Docker\" is a claim about a run that did nothing: {document}" + ); + + // 2. The container-backed tier, in the SAME environment, refuses at its prerequisite. + let container = run_discovery_campaign( + &bin, + &[ + "--seed", + "0x5eed0124", + "--tier", + "container-differential", + "--candidates", + "2", + "--dry-run", + ], + &no_docker, + ); + let container_stderr = String::from_utf8_lossy(&container.stderr).into_owned(); + assert_eq!( + exit_code(&container), + 1, + "the container-backed tier must fail loudly without Docker, never silently degrade \ + into the hermetic one. If it exits 0 here, the two tiers have the same \ + prerequisites and are not separately selectable at all.\nstderr:\n{container_stderr}" + ); + assert!( + container.stdout.is_empty(), + "a campaign that never compared anything must not emit an outcome document" + ); + + // The two tiers really are two tiers, and the runtime requirement is the difference. + assert_ne!( + CampaignTier::ConfigDifferential, + CampaignTier::ContainerDifferential + ); + let mut hermetic_request = campaign_request("0x5eed0124", 0x5EED_0124, 1, &Scratch::new()); + assert!( + !hermetic_request.container_backed(), + "the configuration-resolution tier must not be container-backed" + ); + hermetic_request.tier = CampaignTier::ContainerDifferential; + assert!( + hermetic_request.container_backed(), + "the container tier must be the one that declares the runtime requirement" + ); +} + +/// An unverifiable prerequisite exits `1` — a machinery failure, distinct from both a usage +/// error and a clean run. +/// +/// The failure mode this guards is the comfortable one: a campaign that could not verify the +/// reference and exited `0` would look exactly like a campaign that found the two +/// implementations in perfect agreement. +#[test] +fn an_unverifiable_oracle_exits_one() { + let bin = discovery_campaign_binary(); + let output = run_discovery_campaign( + &bin, + &[ + "--seed", + "0x5eed0059", + "--tier", + "config-differential", + "--dry-run", + ], + &[(ORACLE_OVERRIDE_ENV, ABSENT_ORACLE)], + ); + + let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); + assert_eq!( + exit_code(&output), + 1, + "an unverifiable oracle is a prerequisite failure, not a finding and not a usage \ + error.\nstderr:\n{stderr}" + ); + assert!( + stderr.contains(ABSENT_ORACLE), + "the diagnosis must name the path it could not use: {stderr}" + ); + assert!( + output.stdout.is_empty(), + "a campaign that never compared anything must not emit an outcome document" + ); +} + +// --------------------------------------------------------------------------- +// T101/T102/T103 (US7) — the network-backed real-world corpus canary +// --------------------------------------------------------------------------- +// +// These three need the network as well as the verified oracle. That is not a widening of +// this binary's requirements so much as the point of the tier: the corpus is an ecological +// canary, and the ecosystem is not in this repository. They still never skip — an +// unreachable *lane* fails loudly, exactly as a missing oracle does, because "the ecosystem +// agrees with us" and "we never reached the ecosystem" must not look alike. + +/// **T101 / FR-051**: a fetched entry whose digest disagrees fails loudly **for that +/// entry**, and is not compared. +/// +/// Two halves, and the second is the one that is easy to get wrong. Detecting the +/// disagreement is necessary; *refusing to compare* is what makes it useful. A tolerant +/// fetch would hand the unexpected content to the differential, and every difference it +/// found would be attributed to deacon-versus-reference when its real cause is that the +/// upstream workspace is not what was recorded — a wrong conclusion, arrived at +/// confidently. +/// +/// The third half, which has no assertion of its own anywhere else: the mismatch must not +/// **re-baseline** the manifest. Writing the fetched digest over the recorded one would +/// make every FR-051 failure self-healing, so the check would report a problem exactly once +/// and then never again. +#[test] +fn a_corpus_entry_whose_digest_disagrees_fails_loudly_for_that_entry() { + // A real, reachable entry — carrying a well-formed digest that is deliberately not its + // own. Well-formed matters: a malformed digest is refused hermetically by D4 and would + // never reach the fetch, so it would test the wrong rule. + let wrong_digest = format!("sha256:{}", "b".repeat(64)); + let mut entries = corpus_prefix(2); + entries[0].content_digest = Some(wrong_digest.clone()); + let mismatched_id = entries[0].id.clone(); + let honest_id = entries[1].id.clone(); + + let scratch = Scratch::new().with_corpus(&entries); + let run = run_campaign(&corpus_request("0x5eed0101", 0x5EED_0101, &scratch)); + + match status_of(&run, &mismatched_id) { + EntryStatus::DigestMismatch { + expected, actual, .. + } => { + assert_eq!( + expected, &wrong_digest, + "the diagnosis must name the digest the manifest RECORDED" + ); + assert_ne!( + actual, &wrong_digest, + "the diagnosis must name the digest that was actually materialized, or a \ + reviewer cannot tell a re-pin from a compromise" + ); + assert!( + corpus::is_well_formed_digest(actual), + "the materialized digest must be `sha256:<64 hex>`, got {actual}" + ); + } + other => panic!("expected a digest mismatch, got {}", other.summary()), + } + + // The honest entry ran. Without it this test could pass on a campaign that compared + // nothing at all, which is the failure mode it is supposed to distinguish from. + assert!( + matches!(status_of(&run, &honest_id), EntryStatus::Materialized(_)), + "the entry with no recorded digest must materialize and be compared: {}", + status_of(&run, &honest_id).summary() + ); + assert_eq!( + run.report.candidates_generated, 2, + "both entries were attempted" + ); + assert_eq!( + run.report.candidates_executed, 1, + "exactly one entry was COMPARED — the mismatched one must never reach the \ + differential" + ); + assert_eq!( + run.report.candidates_discarded_unsafe, 1, + "the entry that was attempted but deliberately not executed is counted, never \ + silently dropped" + ); + + // And the manifest was not re-baselined behind the failure. + let reloaded = DiscoveryData::load(scratch.path()).expect("the scratch manifest reloads"); + let after = reloaded + .corpus_entry(&mismatched_id) + .expect("the mismatched entry survives"); + assert_eq!( + after.content_digest.as_deref(), + Some(wrong_digest.as_str()), + "a mismatch must never overwrite the recorded digest: a self-healing check reports \ + a problem once and then never again" + ); + + // The honest entry's digest WAS recorded, because that is a first materialization — + // the behavior the mismatch path must not be confused with. + let honest = reloaded + .corpus_entry(&honest_id) + .expect("the honest entry survives"); + let honest_digest = honest + .content_digest + .as_deref() + .expect("a first materialization records its digest"); + assert!(corpus::is_well_formed_digest(honest_digest)); + assert!( + corpus::check(&reloaded.corpus).is_empty(), + "recording a digest must leave the manifest D4-clean" + ); +} + +/// **T102 / FR-052**: an unreachable entry is distinguished from one that ran and found +/// nothing. +/// +/// These two are the most confusable pair in the whole tier, and confusing them is the +/// comfortable direction: "nothing to report" is what both look like from a distance, and +/// one of them means the canary never went down the mine. The run therefore reports them as +/// different *kinds* of outcome, not as a smaller number. +#[test] +fn an_unreachable_corpus_entry_is_not_an_entry_that_found_nothing() { + let mut entries = corpus_prefix(1); + let reachable_id = entries[0].id.clone(); + + // A repository that does not exist, at a syntactically valid immutable commit. The + // commit must be well formed: an unresolvable *reference* is D4 and never reaches the + // network, so using one would test the hermetic rule instead of the fetch. + let unreachable = { + let repository = "deacon-nonexistent-org/deacon-nonexistent-repo"; + let commit = "0123456789abcdef0123456789abcdef01234567"; + CorpusEntry { + id: CorpusEntry::derive_id(repository, commit, ""), + name: "synthetic-unreachable".to_string(), + repository: repository.to_string(), + commit: commit.to_string(), + path: String::new(), + content_digest: None, + notes: "deliberately unreachable, to prove the distinction FR-052 draws".to_string(), + } + }; + let unreachable_id = unreachable.id.clone(); + entries.push(unreachable); + + let scratch = Scratch::new().with_corpus(&entries); + let run = run_campaign(&corpus_request("0x5eed0102", 0x5EED_0102, &scratch)); + + match status_of(&run, &unreachable_id) { + EntryStatus::Unreachable { cause, .. } => assert!( + !cause.trim().is_empty(), + "an unreachable entry must carry the cause it failed for; \"unreachable\" with \ + no reason is a shrug, and a reviewer cannot tell a deleted repository from a \ + dropped network" + ), + other => panic!("expected Unreachable, got {}", other.summary()), + } + + let reachable = status_of(&run, &reachable_id); + assert!( + matches!(reachable, EntryStatus::Materialized(_)), + "the reachable entry must materialize: {}", + reachable.summary() + ); + + // The distinction, stated three ways — because a single tally cannot carry it. + assert_eq!(run.report.candidates_generated, 2, "both were attempted"); + assert_eq!( + run.report.candidates_executed, 1, + "only the reachable entry was compared; an unreachable one contributes no \ + comparison, and must not be counted as one that agreed" + ); + assert_eq!( + run.corpus_statuses.len(), + 2, + "every attempted entry gets a status, including the ones that produced no finding" + ); + assert!( + !matches!(reachable, EntryStatus::Unreachable { .. }), + "an entry that ran and found nothing is NOT unreachable — that conflation is the \ + one FR-052 exists to prevent" + ); + + // The run still exits cleanly: an unreachable entry is a fact about the ecosystem, not + // a machinery failure, and the exit-status contract says a campaign reports what it + // ran, never what it found (FR-058). + assert!( + !run.campaign.outcome.budget_exhausted, + "the plan is the manifest, and both entries were reached" + ); + + // The unreachable entry recorded no digest. Recording one would claim a verification of + // content nobody retrieved. + let reloaded = DiscoveryData::load(scratch.path()).expect("the scratch manifest reloads"); + assert_eq!( + reloaded + .corpus_entry(&unreachable_id) + .expect("the entry survives") + .content_digest, + None, + "an unreachable entry must not record a digest" + ); +} + +/// **T103 / FR-054**: a corpus finding enters the same minimization, classification, +/// deduplication, and promotion pipeline as a generated one, and names its upstream +/// provenance. +/// +/// "The same pipeline" is asserted as sameness of *mechanism*, not of outcome: the corpus +/// driver calls the same `differential::compare`, the same tolerance index, the same +/// signature derivation, and the same admission queue, and this test checks the observable +/// consequences of that — a corpus finding validates under the same D-classes, deduplicates +/// against a second campaign, and is accepted by the same triage and promotion API. +/// +/// **On the provenance:** a generated finding's witness carries the document that +/// reproduces it. A corpus finding's cannot — corpus content is never vendored (FR-053) — +/// so it carries the pin instead: repository, commit, path, and the verified digest. That +/// is the same claim in the only form available, because a witness whose input nobody can +/// retrieve names nothing. +/// +/// **On vacuity, stated plainly:** the per-finding assertions below are conditional, +/// because two implementations agreeing on a real-world workspace is a legitimate and +/// welcome outcome, and a test that *failed* when they agreed would be a gate on findings — +/// the one thing FR-058 forbids this machinery to become. What is unconditional is the part +/// that would catch the pipeline being wired up wrong at all: the tier ran, entries were +/// compared, the campaign record and queue validate under the same D-classes, and a second +/// campaign over the same standing queue admits nothing new. +/// +/// **Note on minimization (US2):** reduction is not implemented at the time of writing — +/// `shrink.rs` declares the ordered catalogue and `minimize.rs` is a stub — so "the same +/// minimization" is asserted here as the same *state*: a corpus witness reports +/// `isMinimal: false` with no reduction steps, exactly as a generated one does (FR-022 +/// forbids presenting an unreduced input as minimal). When T047–T052 land, re-run this to +/// confirm a corpus witness is reduced by the same catalogue rather than exempted from it. +#[test] +fn a_corpus_finding_enters_the_same_pipeline_and_names_its_provenance() { + let entries = corpus_prefix(6); + let by_id: std::collections::BTreeMap<&str, &CorpusEntry> = + entries.iter().map(|e| (e.id.as_str(), e)).collect(); + + let scratch = Scratch::new().with_corpus(&entries); + let first = run_campaign(&corpus_request("0x5eed0103", 0x5EED_0103, &scratch)); + + // --- Unconditional: the tier ran, and its record is the ordinary record ------------ + assert_eq!( + first.campaign.tier, + CampaignTier::Corpus, + "the record names the tier that wrote it" + ); + assert!( + first.report.candidates_executed > 0, + "no corpus entry was compared, so this test would assert nothing about a \ + pipeline: {:?}", + first + .corpus_statuses + .iter() + .map(EntryStatus::summary) + .collect::>() + ); + // The plan is the manifest, not the request's generator denominator. + assert_eq!( + first.report.candidates_generated, + entries.len() as u64, + "the corpus tier plans its manifest, so a full sweep is a complete run rather than \ + a fraction of a plan it never had" + ); + + let registry = Registry::load(&default_registry_dir()).expect("the registry loads"); + let data = DiscoveryData::load(scratch.path()).expect("the written data root reloads"); + let violations = queue::check(&data, &queue::RegistryView::from_registry(®istry)); + assert!( + violations.is_empty(), + "a corpus campaign's own record must satisfy the same D-classes as a generated \ + one:\n{}", + violations + .iter() + .map(|v| format!(" {} {}: {v}", v.class(), v.record())) + .collect::>() + .join("\n") + ); + + // --- Per finding: provenance, minimization state, classification, promotion -------- + for finding in &data.findings { + for witness in &finding.witnesses { + if witness.campaign_id != first.campaign.id { + continue; + } + let entry = by_id.get(witness.candidate_id.as_str()).unwrap_or_else(|| { + panic!( + "a corpus witness's candidateId must BE the `cor-` entry id — the \ + one thing that reproduces the observation — got {:?}", + witness.candidate_id + ) + }); + + let provenance = &witness.minimal_input; + for (field, expected) in [ + ("corpusEntry", entry.id.as_str()), + ("repository", entry.repository.as_str()), + ("commit", entry.commit.as_str()), + ("path", entry.path.as_str()), + ] { + assert_eq!( + provenance.get(field).and_then(|v| v.as_str()), + Some(expected), + "the witness must name its upstream {field} (FR-054): {provenance}" + ); + } + let digest = provenance + .get("contentDigest") + .and_then(|v| v.as_str()) + .unwrap_or_else(|| { + panic!("the witness must name the verified digest: {provenance}") + }); + assert!( + corpus::is_well_formed_digest(digest), + "the recorded provenance digest must be `sha256:<64 hex>`, got {digest}" + ); + + // The same minimization STATE a generated witness carries. See the note above. + assert!( + !witness.is_minimal, + "no reduction has run, so a corpus witness must not claim minimality \ + (FR-022) — a real-world workspace is the least minimal input this feature \ + has" + ); + assert!( + witness.reduction_steps.is_empty(), + "an unreduced witness must record no reduction steps" + ); + assert!( + witness.mutation_operators.is_empty(), + "no mutation operator produced a corpus input; a third party did" + ); + } + + // Classification and promotion: the same API a generated finding goes through. + // Exercised on a clone so the written queue is left exactly as the campaign wrote + // it — a test that promoted the real record would be authoring the review it is + // supposed to be checking. + let mut candidate = finding.clone(); + assert_eq!( + candidate.state, + FindingState::Untriaged, + "a newly admitted finding is untriaged whatever tier admitted it" + ); + candidate + .triage(Classification::DeaconRegression, None) + .expect("a corpus finding is triaged by the same API as a generated one"); + candidate + .promote("case-tier1-decl-go-minimal") + .expect("a triaged corpus finding is promotable by the same API"); + } + + // --- Deduplication across campaigns ------------------------------------------------ + // A second campaign with a different seed over the SAME standing queue. The corpus tier + // draws nothing, so a different seed changes only the campaign id — which is exactly + // what makes this a deduplication test rather than a reproducibility one: a signature + // already in the queue must be re-witnessed, never admitted again. + let before: std::collections::BTreeSet = + data.findings.iter().map(|f| f.id.clone()).collect(); + let second = run_campaign(&corpus_request("0x5eed0104", 0x5EED_0104, &scratch)); + assert_ne!( + second.campaign.id, first.campaign.id, + "a different seed is a different campaign" + ); + + let after = DiscoveryData::load(scratch.path()).expect("the data root reloads"); + let after_ids: std::collections::BTreeSet = + after.findings.iter().map(|f| f.id.clone()).collect(); + assert_eq!( + before, after_ids, + "re-running the corpus tier over an unchanged manifest must admit no NEW finding — \ + the same signature is one defect observed twice, not two defects" + ); + + let violations = queue::check(&after, &queue::RegistryView::from_registry(®istry)); + assert!( + violations.is_empty(), + "the queue must stay clean after a second campaign:\n{violations:?}" + ); + + // Every entry's digest was RECORDED by the first run and VERIFIED by the second — the + // FR-051 lifecycle, end to end. A `recorded: true` on the second run would mean the + // digest went missing between them, which is D4's second clause. + for status in &second.corpus_statuses { + if let EntryStatus::Materialized(m) = status { + assert!( + !m.recorded, + "the second fetch must VERIFY the digest recorded by the first, not record \ + it again: {}", + status.summary() + ); + } + } +} diff --git a/crates/deacon/tests/discovery_hermetic.rs b/crates/deacon/tests/discovery_hermetic.rs new file mode 100644 index 00000000..f62c444b --- /dev/null +++ b/crates/deacon/tests/discovery_hermetic.rs @@ -0,0 +1,2531 @@ +//! Hermetic discovery guards (025-exploratory-parity-discovery, T023/T024). +//! +//! This binary is a **guard, not a campaign**. It loads the committed discovery data +//! root, runs the D-class validation over it, and cross-checks this repository's own lane +//! wiring — no Docker, no network, no reference oracle, no randomness. It therefore runs +//! in the `default` and `dev-fast` profiles like every other hermetic conformance guard, +//! and it must keep doing so: a guard that does not run in the fast lane is a guard +//! nobody notices going stale. +//! +//! That is exactly why `[profile.discovery]`'s `default-filter` is an explicit +//! `binary(=…)` allow-list and not a `discovery_*` glob (research D9). The glob would +//! capture *this file* and silently remove it from the fast lane — the mistake the parity +//! profile already documents having made with `parity_harness_faults` and +//! `parity_registry_check`. [`the_hermetic_guard_runs_in_the_fast_lane`] asserts the +//! allow-list actually behaves that way, so the reasoning is enforced rather than +//! merely written down. +//! +//! Later user stories add their own guards here (US3's no-network and never-gates tests, +//! US4's classification/deduplication tests, US5's no-write-path and traversal proofs, +//! US7's corpus-provenance tests). They land in this same file as independent test +//! functions. + +use std::path::{Path, PathBuf}; + +use deacon_conformance::discovery::corpus::{self, CorpusEntry}; +use deacon_conformance::discovery::generate; +use deacon_conformance::discovery::grammar::Grammar; +use deacon_conformance::discovery::queue::{self, CampaignsFile, DiscoveryData, FindingsFile}; +use deacon_conformance::discovery::report as discovery_report; +use deacon_conformance::discovery::rng::Prng; +use deacon_conformance::discovery::signature::{Divergence, DivergenceKind, Signature}; +use deacon_conformance::load::Registry; +// `PULL_REQUEST_PROFILES` is the harness's single definition of "every lane a pull +// request runs through", shared with `parity_registry_check`. Two copies of a six-element +// list is exactly the kind of thing that drifts silently: a seventh profile added to one +// copy and not the other would leave a lane nobody checks, which is indistinguishable +// from a lane that checks out. +use parity_harness::registry::{PULL_REQUEST_PROFILES, filter_selects, parse_nextest_profiles}; + +/// The workspace root, derived from this crate's manifest directory so the paths are +/// stable regardless of the per-package cargo/nextest working directory. +fn workspace_root() -> PathBuf { + let manifest = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + manifest + .parent() + .and_then(Path::parent) + .map(Path::to_path_buf) + .unwrap_or(manifest) +} + +fn load_registry() -> Registry { + Registry::load(&deacon_conformance::default_registry_dir()) + .expect("the committed conformance registry must load") +} + +fn load_discovery() -> DiscoveryData { + DiscoveryData::load_default().unwrap_or_else(|e| { + panic!( + "the committed discovery data root must load — a data root that does not load \ + is not an empty queue, it is an unreadable one: {e}" + ) + }) +} + +/// The three files the discovery data root is made of. +const DATA_ROOT_FILES: [&str; 3] = ["findings.json", "campaigns.json", "corpus.json"]; + +/// The live campaign binaries — selected ONLY by `[profile.discovery]`. +const LIVE_DISCOVERY_BINARIES: [&str; 2] = ["discovery_campaign", "discovery_metamorphic"]; + +/// The hermetic discovery guards — selected by the fast lane and by no discovery lane. +/// +/// `discovery_cli` lives in the `deacon-conformance` crate (it drives that crate's own +/// binary), but its lane requirement is identical, so both are asserted here rather than +/// splitting one invariant across two files. +const HERMETIC_DISCOVERY_BINARIES: [&str; 2] = ["discovery_hermetic", "discovery_cli"]; + +#[test] +fn the_discovery_data_root_exists_and_is_canonically_rendered() { + let dir = deacon_conformance::default_discovery_dir(); + assert!( + dir.is_dir(), + "the discovery data root {dir:?} must exist — it is version-controlled, not \ + created on demand" + ); + for name in DATA_ROOT_FILES { + let path = dir.join(name); + let raw = std::fs::read_to_string(&path) + .unwrap_or_else(|e| panic!("{path:?} must be readable: {e}")); + let parsed: serde_json::Value = serde_json::from_str(&raw) + .unwrap_or_else(|e| panic!("{path:?} must be valid JSON: {e}")); + assert_eq!( + parsed["schemaVersion"], + serde_json::json!(queue::SCHEMA_VERSION), + "{path:?} must declare the current schema version" + ); + assert!( + parsed["records"].is_array(), + "{path:?} must carry a `records` array" + ); + assert!( + raw.ends_with("}\n"), + "{path:?} must end with a trailing newline so the first campaign's write is a \ + content diff and not a whole-file reformat" + ); + } +} + +#[test] +fn the_discovery_data_root_loads_and_validates_clean() { + let registry = load_registry(); + let data = load_discovery(); + + let violations = queue::check(&data, &queue::RegistryView::from_registry(®istry)); + assert!( + violations.is_empty(), + "the committed discovery data root must have no D-class violations:\n{}", + violations + .iter() + .map(|v| format!(" {} {}: {v}", v.class(), v.record())) + .collect::>() + .join("\n") + ); +} + +#[test] +fn the_queue_report_renders_from_the_committed_data_root() { + // `discovery report` must work end to end against the real data root, including + // while it is empty. An empty queue is a legitimate state, not a degenerate one. + let registry = load_registry(); + let data = load_discovery(); + + let pins = discovery_report::CurrentPins::from_registry(®istry); + let report = discovery_report::build_queue_report(&data, &pins); + assert_eq!(report.total, data.findings.len()); + + let json = discovery_report::render_json(&report); + let md = discovery_report::render_md(&report); + assert_eq!( + ( + discovery_report::render_json(&report), + discovery_report::render_md(&report) + ), + (json.clone(), md.clone()), + "the report must be byte-stable — no timestamps, no absolute paths" + ); + assert!( + md.contains("| untriaged |"), + "the untriaged bucket is COUNTED (FR-029)" + ); + assert!( + md.contains("never gates"), + "the report must say what it is: a triage queue, not a gate" + ); + + // The pins the report compares against must be the ones the registry records, or + // every finding would read as pin-stale (or none ever would). + assert_eq!(pins.schema_pin, deacon_conformance::CURRENT_SCHEMA_PIN); + assert_eq!(pins.prose_pin, deacon_conformance::CURRENT_SPEC_PIN); + assert!( + pins.oracle_version.is_some(), + "the registry must record an oracle revision, or oracle pin-staleness is not \ + decidable and every finding's oracle claim goes unchecked" + ); +} + +/// The data root is a **sibling** of the registry, and nothing in the registry loader +/// reaches it. This is the structural half of the guarantee that an unreviewed finding +/// can never influence `certify` (research D6) — asserted rather than assumed, because +/// the failure mode is silent: a finding quietly joining the certification denominator. +#[test] +fn the_discovery_root_is_outside_the_registry_and_unreachable_from_it() { + let registry_dir = deacon_conformance::default_registry_dir(); + let discovery_dir = deacon_conformance::default_discovery_dir(); + + assert!( + !discovery_dir.starts_with(®istry_dir), + "the discovery root {discovery_dir:?} must not sit inside the registry \ + {registry_dir:?} — placing it there means either the loader rejects it or \ + someone wires it in and unreviewed findings reach `certify`" + ); + assert_eq!( + discovery_dir.parent(), + registry_dir.parent(), + "the discovery root must be a SIBLING of the registry under conformance/" + ); + + // Loading the registry must not surface any discovery record. There is no field + // that could hold one, which is the point; assert on the observable consequence. + let registry = Registry::load(®istry_dir).expect("registry loads"); + let discovery = load_discovery(); + for finding in &discovery.findings { + assert!( + !registry.cases.iter().any(|c| c.id == finding.id), + "a finding id must never appear as a registry case id" + ); + } +} + +/// The one reference that crosses the root boundary points **out** of discovery into the +/// registry (`Finding.promotedTo → case-`). Nothing in the registry points back, so +/// following references from the registry can never arrive at a finding. +#[test] +fn the_only_cross_root_reference_points_out_of_the_queue() { + let registry = load_registry(); + let discovery = load_discovery(); + + for finding in &discovery.findings { + if let Some(case_id) = &finding.promoted_to { + assert!( + registry.cases.iter().any(|c| &c.id == case_id), + "promoted finding {} names case `{case_id}`, which does not resolve — the \ + queue must never claim coverage that does not exist", + finding.id + ); + } + } +} + +/// **T024**: this guard runs in the fast lane, and the live campaign binaries do not run +/// in any pull-request lane. +/// +/// Evaluated against the real `.config/nextest.toml` with the harness's filterset +/// evaluator, so an exclusion written as `not (…)` is honored exactly rather than merely +/// token-matched. +#[test] +fn the_hermetic_guard_runs_in_the_fast_lane() { + let toml_text = std::fs::read_to_string(workspace_root().join(".config/nextest.toml")) + .expect("read .config/nextest.toml"); + let profiles = parse_nextest_profiles(&toml_text).expect("parse .config/nextest.toml"); + + for profile in ["default", "dev-fast"] { + let filter = profiles + .default_filters + .get(profile) + .unwrap_or_else(|| panic!("[profile.{profile}] must exist")); + for guard in HERMETIC_DISCOVERY_BINARIES { + match filter { + // No default-filter means "select everything", which includes the guards. + None => {} + Some(expr) => assert!( + filter_selects(expr, guard) + .unwrap_or_else(|e| panic!("[profile.{profile}] filter: {e}")), + "[profile.{profile}] must select `{guard}`: it is a guard, not a \ + campaign, and a guard that does not run in the fast lane is a guard \ + nobody notices going stale" + ), + } + } + } +} + +/// **T006/T007**: the live campaign binaries are selected by `[profile.discovery]` and by +/// no pull-request profile. +/// +/// The allow-list must also NOT capture this guard — the `discovery_*` glob mistake +/// research D9 exists to prevent. +#[test] +fn live_discovery_binaries_are_selected_by_exactly_one_lane() { + let toml_text = std::fs::read_to_string(workspace_root().join(".config/nextest.toml")) + .expect("read .config/nextest.toml"); + let profiles = parse_nextest_profiles(&toml_text).expect("parse .config/nextest.toml"); + + let discovery_filter = profiles + .default_filters + .get("discovery") + .expect("nextest.toml must declare [profile.discovery] — discovery has no lane otherwise") + .as_deref() + .expect( + "[profile.discovery] must declare a default-filter; without one it selects every \ + binary in the workspace", + ); + + for name in LIVE_DISCOVERY_BINARIES { + assert!( + filter_selects(discovery_filter, name).expect("evaluate the discovery filter"), + "[profile.discovery] must select live binary `{name}`" + ); + } + for guard in HERMETIC_DISCOVERY_BINARIES { + assert!( + !filter_selects(discovery_filter, guard).expect("evaluate the discovery filter"), + "[profile.discovery] must NOT capture the hermetic guard `{guard}` — that is \ + precisely the `discovery_*` glob mistake the explicit allow-list exists to \ + avoid (research D9)" + ); + } + + for &profile in PULL_REQUEST_PROFILES { + let filter = profiles + .default_filters + .get(profile) + .unwrap_or_else(|| panic!("[profile.{profile}] must exist")); + let expr = filter.as_deref().unwrap_or_else(|| { + panic!( + "[profile.{profile}] has no default-filter, so it selects every binary \ + including the live discovery campaigns" + ) + }); + for name in LIVE_DISCOVERY_BINARIES { + assert!( + !filter_selects(expr, name) + .unwrap_or_else(|e| panic!("[profile.{profile}] filter: {e}")), + "[profile.{profile}] selects live discovery binary `{name}` — a green \ + pull-request run must never imply a campaign ran (FR-055/FR-057)" + ); + } + } +} + +// --------------------------------------------------------------------------- +// T053 (US3, SC-013) — the hermetic surface cannot reach the network +// --------------------------------------------------------------------------- + +/// Every source file that makes up the hermetic discovery surface. +/// +/// Returned as `(relative path, contents)` so a failure names the file. The list is +/// enumerated from disk rather than hard-coded: a module added by a later user story +/// joins the scan automatically, which is the only way a guard like this keeps up with +/// the thing it guards. +fn hermetic_discovery_sources() -> Vec<(String, String)> { + let dir = workspace_root().join("crates/conformance/src/discovery"); + let mut out = Vec::new(); + let rd = std::fs::read_dir(&dir).unwrap_or_else(|e| panic!("read {dir:?}: {e}")); + for entry in rd.filter_map(Result::ok) { + let path = entry.path(); + if path.extension().and_then(|s| s.to_str()) != Some("rs") { + continue; + } + let name = path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or_default() + .to_string(); + let text = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {path:?}: {e}")); + out.push((format!("crates/conformance/src/discovery/{name}"), text)); + } + out.sort(); + out +} + +/// **T053 / SC-013**: the hermetic discovery surface completes with zero network requests. +/// +/// "Zero network requests" is asserted the strongest way available — as an **absent +/// capability**, not as an unobserved behaviour. A run that merely *happened* not to make +/// a request proves nothing about the next run with different data, and sandboxing a Rust +/// test's sockets from inside the test is not something the test can honestly do. So the +/// claim is established in three parts, and each covers a way the other two can be +/// evaded: +/// +/// 1. **The crate declares no way to speak a network protocol.** `deacon-conformance` +/// owns the entire hermetic surface (grammar, PRNG, signature, queue, report) and +/// depends on no HTTP client, no async runtime, and no git/ssh transport. This is the +/// same structural argument `clause_determinism` makes for the clause commands, taken +/// here for the discovery ones; the git/ssh entries are specific to discovery, because +/// the corpus canary is the one part of this feature that legitimately fetches — and +/// it lives in `parity-harness`, deliberately on the other side of the seam (D8). +/// 2. **No discovery module shells out.** A dependency audit is blind to +/// `Command::new("curl")` and `git fetch`, which is exactly how a hermetic module +/// would most plausibly acquire the network by accident: the corpus manifest model +/// lives here while the fetch lives in the live half, so the tempting shortcut is one +/// line away at all times. +/// 3. **The surface actually runs.** A capability argument about code nobody executes is +/// worth little, so the whole hermetic path is exercised end to end below. Together +/// with (1) and (2) that is what "completes with zero network requests" means here. +#[test] +fn the_hermetic_discovery_surface_cannot_reach_the_network() { + // --- 1. No network-capable dependency in the crate that owns the surface ---------- + let manifest_path = workspace_root().join("crates/conformance/Cargo.toml"); + let manifest = std::fs::read_to_string(&manifest_path) + .unwrap_or_else(|e| panic!("read {manifest_path:?}: {e}")); + // Inspect dependency-declaration lines only (`name = …`), so prose in a comment + // never trips the check: the guarantee is about actual dependencies. + let declared: Vec = manifest + .lines() + .map(str::trim) + .filter(|l| !l.starts_with('#')) + .filter_map(|l| l.split_once('=').map(|(name, _)| name.trim().to_string())) + .collect(); + for forbidden in [ + // HTTP clients. + "reqwest", + "hyper", + "ureq", + "curl", + "isahc", + "surf", + "attohttpc", + "http", + // Async runtimes — the substrate a socket would need. + "tokio", + "async-std", + "smol", + // Git / ssh transports: the corpus canary's fetch belongs in `parity-harness` + // (research D8), never in the hermetic half that models the manifest. + "git2", + "gix", + "ssh2", + "russh", + ] { + assert!( + !declared.iter().any(|d| d == forbidden), + "deacon-conformance must not depend on {forbidden:?}: it owns the hermetic \ + discovery surface, whose no-network guarantee is that the capability is \ + ABSENT rather than merely unused (SC-013)" + ); + } + + // --- 2. No discovery module spawns a process or opens a socket ------------------- + let sources = hermetic_discovery_sources(); + assert!( + sources.len() >= 10, + "expected to scan the whole hermetic discovery surface, only saw {} file(s) — a \ + scan that silently stopped finding files would pass by checking nothing", + sources.len() + ); + for required in [ + "queue.rs", + "grammar.rs", + "rng.rs", + "signature.rs", + "report.rs", + ] { + assert!( + sources.iter().any(|(p, _)| p.ends_with(required)), + "the scan must cover {required}; got {:?}", + sources.iter().map(|(p, _)| p).collect::>() + ); + } + + /// Tokens that would give a hermetic module a way off the machine. Matched on + /// non-comment lines only, so a doc comment may still *discuss* `git fetch`. + const FORBIDDEN: &[&str] = &[ + "std::net", + "TcpStream", + "TcpListener", + "UdpSocket", + "std::process", + "tokio::process", + "Command::new", + "reqwest", + "ureq", + "hyper::", + "git2", + "gix::", + ]; + let mut problems = Vec::new(); + for (path, text) in &sources { + for (line_no, line) in text.lines().enumerate() { + let code = line.trim_start(); + if code.starts_with("//") || code.starts_with("/*") || code.starts_with('*') { + continue; + } + for needle in FORBIDDEN { + if code.contains(needle) { + problems.push(format!( + "{path}:{}: uses `{needle}` — the hermetic half must have no way to \ + reach the network, and shelling out is the one route a dependency \ + audit cannot see. Fetching belongs in \ + `parity_harness::discovery::corpus_fetch` (research D8).", + line_no + 1 + )); + } + } + } + } + assert!( + problems.is_empty(), + "the hermetic discovery surface must not spawn processes or open sockets:\n{}", + problems.join("\n") + ); + + // --- 3. And the surface really runs ----------------------------------------------- + // Grammar: the pinned constraint inventory, read from disk. + let grammar = Grammar::load_default().expect("the committed grammar must load"); + assert!( + !grammar.is_empty(), + "an empty grammar would make every later assertion in this test vacuous" + ); + + // PRNG: a seeded draw, entirely in-process. + let mut prng = Prng::from_seed(0x5eed_1234); + let first = prng.next_u64(); + assert_eq!( + Prng::from_seed(0x5eed_1234).next_u64(), + first, + "the stream is a property of committed code, not of the environment" + ); + + // Signature: derived from an in-memory divergence, never re-diffed. + let deacon = serde_json::json!("vscode"); + let reference = serde_json::json!("root"); + let signature = Signature::derive( + "chan-structured-output", + &Divergence { + kind: DivergenceKind::Value, + path: "configuration.remoteUser", + deacon: Some(&deacon), + reference: Some(&reference), + }, + ); + assert!(signature.finding_id().starts_with("fnd-")); + + // Queue + report: load the committed data root, validate it, render the artifacts. + let registry = load_registry(); + let data = load_discovery(); + assert!(queue::check(&data, &queue::RegistryView::from_registry(®istry)).is_empty()); + let report = discovery_report::build_queue_report( + &data, + &discovery_report::CurrentPins::from_registry(®istry), + ); + assert!(!discovery_report::render_md(&report).is_empty()); +} + +// --------------------------------------------------------------------------- +// T055 (US3, SC-014) — the surface never gates on what it found +// --------------------------------------------------------------------------- + +/// The certification profile every synthetic campaign in this file records. +const SYNTHETIC_PROFILE: &str = "prof-linux-amd64-docker-0870"; + +/// The pinned input set every synthetic campaign records — the registry's **real** pins, +/// so a synthetic finding is never accidentally pin-stale and the pin-stale bucket keeps +/// meaning what it says. +fn synthetic_pins(registry: &Registry) -> queue::PinnedInputSet { + let pins = discovery_report::CurrentPins::from_registry(registry); + let oracle = pins + .oracle_version + .clone() + .expect("the registry must record an oracle revision"); + serde_json::from_value(serde_json::json!({ + "schemaPin": pins.schema_pin, + "prosePin": pins.prose_pin, + "oracleVersion": oracle, + "normalizerVersion": deacon_conformance::snapshot::NORMALIZER_VERSION, + "grammarVersion": Grammar::load_default() + .expect("grammar loads") + .revision() + .to_string(), + "mutationCatalogVersion": "v1", + "generatorVersion": deacon_conformance::discovery::rng::prng_identity() + })) + .expect("the synthetic pinned input set must parse") +} + +/// One synthetic campaign, routed through the strict loader. +/// +/// The id is **derived**, never chosen: `check` recomputes it from the record's own +/// substance, so a hand-picked id would (correctly) fail the D1 identity clause and the +/// fixture would stop being the clean queue these tests need. `seed` is what makes two +/// calls two different campaigns. +fn synthetic_campaign( + pinned_input_set: &queue::PinnedInputSet, + seed: &str, + admitted: u64, + suppressed: u64, +) -> queue::Campaign { + let lane = queue::CampaignLane::Scheduled; + let tier = queue::CampaignTier::ConfigDifferential; + let id = queue::Campaign::derive_id(seed, pinned_input_set, lane, SYNTHETIC_PROFILE, tier); + let file: CampaignsFile = serde_json::from_value(serde_json::json!({ + "schemaVersion": queue::SCHEMA_VERSION, + "records": [{ + "id": id, + "seed": seed, + "lane": "scheduled", + "tier": "config-differential", + "profile": SYNTHETIC_PROFILE, + "pinnedInputSet": pinned_input_set, + "budget": { + "wallClockSeconds": queue::DEFAULT_WALL_CLOCK_SECONDS, + "perCandidateSeconds": queue::DEFAULT_PER_CANDIDATE_SECONDS_HERMETIC, + "shrinkStepsPerFinding": 64, + "admissionCap": queue::DEFAULT_ADMISSION_CAP + }, + "outcome": { + "candidatesGenerated": 4820, + "candidatesExecuted": 4629, + "candidatesDiscardedUnsafe": 0, + "parseStageFailures": 191, + "budgetExhausted": false, + "spaceCoveredFraction": 0.0, + "mutationApplications": { "unknown-field": 512 }, + "signaturesObserved": admitted + suppressed, + "signaturesAdmitted": admitted, + "signaturesSuppressed": suppressed + } + }] + })) + .expect("the synthetic campaign must satisfy the strict loader"); + file.records + .into_iter() + .next() + .expect("one synthetic campaign") +} + +/// A value-difference signature at `path` on the structured-output channel. +fn signature_at(path: &str) -> Signature { + let deacon = serde_json::json!("vscode"); + let reference = serde_json::json!("root"); + Signature::derive( + "chan-structured-output", + &Divergence { + kind: DivergenceKind::Value, + path, + deacon: Some(&deacon), + reference: Some(&reference), + }, + ) +} + +/// A **present-versus-absent** signature at `path`: the same observable location as +/// [`signature_at`], a different kind of difference, and therefore a different signature. +fn absence_signature_at(path: &str) -> Signature { + let reference = serde_json::json!("root"); + Signature::derive( + "chan-structured-output", + &Divergence { + kind: DivergenceKind::RefOnly, + path, + deacon: None, + reference: Some(&reference), + }, + ) +} + +/// One witness of `signature`, attributed to `campaign` and `candidate`. +/// +/// Routed through the strict loader for the same reason the queue is: a witness the real +/// file could not hold would let a test assert a shape that cannot occur. +fn synthetic_witness(campaign: &queue::Campaign, candidate: &str) -> queue::Witness { + serde_json::from_value(serde_json::json!({ + "id": queue::Witness::derived_id(&campaign.id, candidate), + "campaignId": campaign.id, + "candidateId": candidate, + "minimalInput": { "image": "alpine:3.18" }, + "isMinimal": true, + "reductionSteps": ["drop-optional-key"], + "observedValues": { "deacon": "vscode", "reference": "root" }, + "mutationOperators": ["mop-wrong-type"] + })) + .expect("the synthetic witness must satisfy the strict loader") +} + +/// Build a queue holding `paths.len()` untriaged findings plus the campaign that admitted +/// them, using the real derived ids and the registry's real pins so it validates clean. +/// +/// Deliberately routed through the strict loader (`FindingsFile` / `CampaignsFile`) +/// rather than constructed as structs: the loader is part of the hermetic surface, and a +/// fixture that bypassed it could assert a shape the real data root can never hold. +fn synthetic_queue(registry: &Registry, paths: &[&str]) -> DiscoveryData { + let pinned_input_set = synthetic_pins(registry); + let campaign = synthetic_campaign( + &pinned_input_set, + "0x5eed1234", + paths.len() as u64, + /* suppressed */ 0, + ); + + let mut records = Vec::new(); + for (index, path) in paths.iter().enumerate() { + let signature = signature_at(path); + let candidate_id = format!("cnd-0000000{index}"); + let witness = synthetic_witness(&campaign, &candidate_id); + records.push(serde_json::json!({ + "id": signature.finding_id(), + "signature": signature, + "witnesses": [witness], + "classification": null, + "state": "untriaged", + "firstObserved": campaign.id, + "lastObserved": campaign.id, + "promotedTo": null, + "splitFrom": null, + "notes": "" + })); + } + + let findings: FindingsFile = serde_json::from_value(serde_json::json!({ + "schemaVersion": queue::SCHEMA_VERSION, + "records": records, + })) + .expect("the synthetic findings must satisfy the strict loader"); + + DiscoveryData { + findings: findings.records, + campaigns: vec![campaign], + corpus: Vec::new(), + } +} + +/// **T055 / SC-014**: a queue full of findings and an empty one are indistinguishable in +/// *outcome* — both validate clean and both render — while remaining entirely +/// distinguishable in *content*. +/// +/// That pairing is the whole rule. If only the first half held, "nothing found" and +/// "nothing ran" would look alike and the most comfortable way for the machinery to be +/// broken would be to report success forever (FR-062). If only the second held, discovery +/// would be a gate, and a stochastic gate makes green non-reproducible — at which point +/// somebody eventually turns the lane off. +/// +/// Asserted at the library level here because this binary lives in the `deacon` crate and +/// has no handle on the `deacon-conformance` executable. The **process**-level contract — +/// `discovery report` exiting `0` on a populated queue, `discovery check` exiting `1` only +/// on a violation — is asserted from outside the process by +/// `crates/conformance/tests/discovery_cli.rs`, in the crate that owns the binary. The +/// two halves are cross-checked below so neither can be deleted while the other keeps +/// implying it is covered. +#[test] +fn a_populated_queue_and_an_empty_one_are_equally_clean() { + let registry = load_registry(); + let view = queue::RegistryView::from_registry(®istry); + let pins = discovery_report::CurrentPins::from_registry(®istry); + + let empty = DiscoveryData::default(); + let populated = synthetic_queue( + ®istry, + &[ + "configuration.remoteUser", + "configuration.workspaceFolder", + "configuration.userEnvProbe", + ], + ); + + // Same verdict: no quantity of findings is itself a violation. + for (label, data) in [("empty", &empty), ("populated", &populated)] { + let violations = queue::check(data, &view); + assert!( + violations.is_empty(), + "the {label} queue must validate clean — a finding is a candidate for an \ + assertion, never a failure:\n{}", + violations + .iter() + .map(|v| format!(" {} {}: {v}", v.class(), v.record())) + .collect::>() + .join("\n") + ); + } + + // Same success: both render, and both write their artifacts. + let dir = tempfile::tempdir().expect("tempdir"); + for (label, data) in [("empty", &empty), ("populated", &populated)] { + let report = discovery_report::build_queue_report(data, &pins); + let out = dir.path().join(label); + std::fs::create_dir_all(&out).expect("create out dir"); + let written = discovery_report::write_queue_report(&out, &report) + .unwrap_or_else(|e| panic!("the {label} queue must render: {e}")); + assert_eq!(written.len(), 2, "queue.json + queue.md"); + assert!( + discovery_report::render_md(&report).contains("never gates"), + "the {label} report must say what it is: a triage queue, not a gate" + ); + } + + // Different content: the counted untriaged bucket distinguishes "nobody has looked + // yet" from "nothing was found", which is precisely what a status code cannot carry. + let empty_md = + discovery_report::render_md(&discovery_report::build_queue_report(&empty, &pins)); + let full_md = + discovery_report::render_md(&discovery_report::build_queue_report(&populated, &pins)); + assert!(empty_md.contains("| untriaged | 0 |"), "{empty_md}"); + assert!(full_md.contains("| untriaged | 3 |"), "{full_md}"); + assert!( + full_md.contains("4820"), + "a campaign's volume is reported whether or not it found anything (FR-062): \ + {full_md}" + ); + + // The process-level half must still exist. Without this, deleting + // `report_exits_zero_whether_the_queue_is_empty_or_full` would leave the exit-status + // contract — the rule that makes this lane safe to schedule — asserted nowhere, while + // this test kept passing and looking like coverage. + let cli_guard = workspace_root().join("crates/conformance/tests/discovery_cli.rs"); + let cli_guard_text = std::fs::read_to_string(&cli_guard) + .unwrap_or_else(|e| panic!("the process-level exit-status guard {cli_guard:?}: {e}")); + assert!( + cli_guard_text.contains("fn report_exits_zero_whether_the_queue_is_empty_or_full"), + "{cli_guard:?} must keep asserting the exit-status contract from outside the \ + process; the library-level property proven here does not imply it" + ); +} + +/// The discovery command group is **dev-only**. It must not reach the shipped consumer +/// CLI (FR-059, constitution II). +/// +/// Asserted against the **subcommand column** of `deacon --help`, not against the source +/// text: `discovery` is an ordinary English word that already appears in several flag +/// descriptions ("auto-discovery of corporate root CAs", "container identity and +/// discovery"), so a substring search over `cli.rs` would fail on prose that has nothing +/// to do with this feature. The same parsing discipline `parity_registry_check` uses, +/// and for the same reason. +#[test] +fn the_discovery_surface_never_reaches_the_shipped_cli() { + let output = std::process::Command::new(env!("CARGO_BIN_EXE_deacon")) + .arg("--help") + .output() + .expect("the deacon binary runs"); + assert!(output.status.success(), "`deacon --help` must succeed"); + let help = String::from_utf8_lossy(&output.stdout); + + let subcommands: Vec<&str> = help + .lines() + .skip_while(|l| !l.trim_start().starts_with("Commands:")) + .skip(1) + .take_while(|l| !l.trim().is_empty()) + .filter_map(|l| l.split_whitespace().next()) + .collect(); + assert!( + !subcommands.is_empty(), + "expected to parse the subcommand list from `deacon --help`" + ); + assert!( + !subcommands.contains(&"discovery"), + "`discovery` reached the shipped consumer CLI; it is contributor tooling and a \ + conformance-tracking command in the consumer surface is a scope violation that \ + ships to users and is then hard to withdraw. Subcommands found: {subcommands:?}" + ); +} + +// --------------------------------------------------------------------------- +// US4 (T061–T066, T125) — classify and deduplicate what was found +// --------------------------------------------------------------------------- + +/// Triage `finding` in place, failing the test with the refusal rather than swallowing it. +fn triage(finding: &mut queue::Finding, classification: queue::Classification) { + finding + .triage(classification, None) + .unwrap_or_else(|e| panic!("triage must be accepted: {e}")); +} + +/// Assert the queue has no D-class violation, naming every one it does have. +fn assert_clean(data: &DiscoveryData, registry: &Registry, label: &str) { + let violations = queue::check(data, &queue::RegistryView::from_registry(registry)); + assert!( + violations.is_empty(), + "the {label} queue must validate clean:\n{}", + violations + .iter() + .map(|v| format!(" {} {}: {v}", v.class(), v.record())) + .collect::>() + .join("\n") + ); +} + +/// **T061 / SC-007**: every finding either carries exactly one classification or sits in a +/// visible unclassified bucket. None is in neither, none is in both. +/// +/// The partition has **two** unclassified buckets, not one, and the second is easy to miss: +/// `untriaged` (nobody has looked) and `split` (an inert ancestor that surrendered its +/// classification to its children — Q10). Both are counted in the report, which is what +/// keeps SC-007's "no finding is in neither state" true: a split parent is unclassified on +/// purpose and is still visible, rather than being a third, silent category. +/// +/// Both failure directions are exercised, because a checker that only caught the missing +/// classification would let the opposite defect through — an untriaged finding carrying a +/// judgement, which makes the FR-029 bucket count something that has in fact been judged. +#[test] +fn every_finding_carries_exactly_one_classification_or_is_visibly_unclassified() { + let registry = load_registry(); + let view = queue::RegistryView::from_registry(®istry); + let pins = discovery_report::CurrentPins::from_registry(®istry); + + let mut data = synthetic_queue( + ®istry, + &[ + "configuration.remoteUser", + "configuration.workspaceFolder", + "configuration.userEnvProbe", + ], + ); + triage( + &mut data.findings[0], + queue::Classification::DeaconRegression, + ); + triage(&mut data.findings[1], queue::Classification::ReferenceQuirk); + // The third stays untriaged: the partition must hold across a MIXED queue, which is + // the only state a real queue is ever in. + + assert_clean(&data, ®istry, "mixed"); + + let unclassified_states = [queue::FindingState::Untriaged, queue::FindingState::Split]; + for finding in &data.findings { + let classified = finding.classification.is_some(); + let visibly_unclassified = unclassified_states.contains(&finding.state); + assert_ne!( + classified, + visibly_unclassified, + "finding {} is in {} state(s): classification={:?}, state={} — SC-007 requires \ + exactly one", + finding.id, + if classified { "both" } else { "neither" }, + finding.classification, + finding.state.as_str() + ); + } + + // The report accounts for every finding, so "exactly one" is observable from the + // artifact and not only from the in-memory records. + let report = discovery_report::build_queue_report(&data, &pins); + assert_eq!(report.total, 3); + assert_eq!(report.untriaged.len(), 1); + assert_eq!(report.triaged.len(), 2); + assert!( + report.triaged.iter().all(|f| f.classification.is_some()), + "a triaged finding without a classification would be in NEITHER state" + ); + assert!( + report.untriaged.iter().all(|f| f.classification.is_none()), + "an untriaged finding with a classification would be in BOTH" + ); + + // Direction 1: a classification that went missing. + let mut missing = data.clone(); + missing.findings[0].classification = None; + assert!( + queue::check(&missing, &view) + .iter() + .any(|v| v.class() == "D2"), + "a triaged finding with no classification must be D2" + ); + + // Direction 2: a classification that arrived too early. + let mut premature = data.clone(); + premature.findings[2].classification = Some(queue::Classification::SpecAmbiguity); + assert!( + queue::check(&premature, &view) + .iter() + .any(|v| v.class() == "D2"), + "an untriaged finding carrying a classification must be D2 — otherwise the visible \ + unclassified bucket counts something that has been judged" + ); +} + +/// **T062 / SC-006**: equal signatures from two campaigns collapse to one finding with two +/// witnesses — and repeating a campaign adds nothing at all. +/// +/// The two halves are different claims. Collapsing says the queue reflects *distinct +/// problems*; adding nothing on a repeat says it does not reflect *campaign volume*. A +/// queue that grew by one record every night would be a log, and nobody triages a log. +#[test] +fn equal_signatures_from_two_campaigns_collapse_to_one_finding_with_two_witnesses() { + let registry = load_registry(); + let pins = synthetic_pins(®istry); + let first = synthetic_campaign(&pins, "0x5eed0001", 1, 0); + let second = synthetic_campaign(&pins, "0x5eed0002", 1, 0); + assert_ne!(first.id, second.id, "two seeds are two campaigns"); + + let signature = signature_at("configuration.remoteUser"); + let mut findings: Vec = Vec::new(); + + assert_eq!( + queue::upsert_finding( + &mut findings, + signature.clone(), + synthetic_witness(&first, "cnd-00000001"), + &first.id + ), + queue::Upsert::Inserted + ); + // A DIFFERENT campaign, a DIFFERENT candidate, the SAME signature. + assert_eq!( + queue::upsert_finding( + &mut findings, + signature.clone(), + synthetic_witness(&second, "cnd-00000002"), + &second.id + ), + queue::Upsert::WitnessAppended + ); + + assert_eq!(findings.len(), 1, "equal signatures are one finding"); + assert_eq!(findings[0].witnesses.len(), 2, "both observations retained"); + assert_eq!(findings[0].first_observed, first.id); + assert_eq!( + findings[0].last_observed, second.id, + "the most recent campaign that reproduced it" + ); + + // One finding takes ONE classification, covering both witnesses. + triage(&mut findings[0], queue::Classification::DeaconRegression); + assert_eq!( + findings[0].classification, + Some(queue::Classification::DeaconRegression) + ); + + let data = DiscoveryData { + findings, + campaigns: vec![first.clone(), second.clone()], + corpus: Vec::new(), + }; + assert_clean(&data, ®istry, "merged"); + + // SC-006 proper: repeating a campaign with an unchanged seed and unchanged pins adds + // ZERO new findings — and does not even add a witness, because the same campaign + // observing the same candidate is the same observation. + let mut repeated = data.findings.clone(); + assert_eq!( + queue::upsert_finding( + &mut repeated, + signature, + synthetic_witness(&second, "cnd-00000002"), + &second.id + ), + queue::Upsert::AlreadyWitnessed + ); + assert_eq!(repeated, data.findings, "a repeat changes nothing at all"); +} + +/// **T063 / FR-031**: distinct signatures that map to the same behavior stay distinct +/// findings. They are *reported* grouped; grouping is a view, never a merge. +/// +/// Merging them would destroy the ability to tell whether a fix addressed one cause or all +/// of them — which is precisely the question a reviewer asks after landing the fix. +/// +/// Both grouping keys are exercised. The `behavior` key is a **reviewed** mapping: it +/// exists only because a human promoted each finding into a case naming that behavior, and +/// a finding never names a behavior itself (FR-025). The `observable-path` key is what +/// relates two findings *before* anyone has decided what they mean, and it is deliberately +/// not called a behavior claim. +#[test] +fn distinct_signatures_mapping_to_one_behavior_are_grouped_but_never_merged() { + let registry = load_registry(); + let pins = discovery_report::CurrentPins::from_registry(®istry); + let behaviors = discovery_report::BehaviorIndex::from_registry(®istry); + + // Two DIFFERENT signatures at the SAME observable location: one value difference and + // one present-versus-absent difference. Same channel, same path, different kind — so + // they are two signatures by construction, which is the situation FR-031 is about. + let mut data = synthetic_queue(®istry, &["configuration.remoteUser"]); + let campaign = data.campaigns[0].clone(); + let second = absence_signature_at("configuration.remoteUser"); + assert_ne!( + data.findings[0].signature.id, second.id, + "the fixture must really hold two distinct signatures" + ); + queue::upsert_finding( + &mut data.findings, + second, + synthetic_witness(&campaign, "cnd-00000009"), + &campaign.id, + ); + assert_eq!(data.findings.len(), 2, "distinct signatures stay distinct"); + assert_clean(&data, ®istry, "two-signature"); + + // Before promotion: grouped by the observable path they share, still two findings. + let report = discovery_report::build_queue_report_with_behaviors(&data, &pins, &behaviors); + assert_eq!(report.total, 2); + let path_group = report + .groups + .iter() + .find(|g| g.kind == discovery_report::GroupKind::ObservablePath) + .expect("two findings at one observable path must be grouped"); + assert_eq!(path_group.findings.len(), 2); + assert_eq!( + path_group.key, + "chan-structured-output configuration.remoteUser" + ); + + // After promotion into ONE case: grouped by the behavior that case names. + let case = registry + .cases + .iter() + .find(|c| !c.behaviors.is_empty()) + .expect("the registry must hold a case naming at least one behavior"); + for finding in &mut data.findings { + triage(finding, queue::Classification::DeaconRegression); + finding + .promote(&case.id) + .unwrap_or_else(|e| panic!("a deacon-regression finding is promotable: {e}")); + } + assert_clean(&data, ®istry, "promoted"); + + let report = discovery_report::build_queue_report_with_behaviors(&data, &pins, &behaviors); + let behavior_group = report + .groups + .iter() + .find(|g| g.kind == discovery_report::GroupKind::Behavior && g.key == case.behaviors[0]) + .unwrap_or_else(|| { + panic!( + "both findings promoted into `{}` must be grouped under `{}`; groups: {:?}", + case.id, case.behaviors[0], report.groups + ) + }); + assert_eq!(behavior_group.findings.len(), 2); + + // The grouping changed nothing about the findings themselves: two records, each with + // its own signature and its own witnesses. + assert_eq!(report.total, 2, "grouping is a view, never a merge"); + assert_eq!(report.promoted.len(), 2); + assert_ne!(report.promoted[0].id, report.promoted[1].id); + assert_ne!( + report.promoted[0].value_shape_class, report.promoted[1].value_shape_class, + "the two findings really are different differences" + ); + for summary in &report.promoted { + assert_eq!( + summary.witnesses, 1, + "witnesses stay with their own finding" + ); + } + + let md = discovery_report::render_md(&report); + assert!( + md.contains("never a merge"), + "the artifact must say what grouping is and is not: {md}" + ); + assert!(md.contains(&case.behaviors[0]), "{md}"); +} + +/// **T064 / FR-035**: `normalizer-defect` and `fixture-defect` are rejected at promotion. +/// +/// They describe a defect in the discovery or comparison machinery, not a behavior of +/// either implementation, so promoting one would record a claim about deacon or the +/// reference that the evidence does not support. Resolving them changes the normalizer or +/// the generator. +/// +/// Rejected in **two** places, and both matter. The promotion path refuses by construction, +/// so the record is never written; **D2** refuses a hand edit that bypassed the path, so a +/// record that was written anyway does not stand. A checker alone would be too late — by +/// the time it runs, the queue is already claiming coverage that cannot exist. +#[test] +fn a_normalizer_or_fixture_defect_can_never_be_promoted() { + let registry = load_registry(); + let view = queue::RegistryView::from_registry(®istry); + let case = registry + .cases + .first() + .expect("the registry must hold at least one case"); + + for non_promotable in [ + queue::Classification::NormalizerDefect, + queue::Classification::FixtureDefect, + ] { + assert!( + !non_promotable.is_promotable(), + "{} must be non-promotable", + non_promotable.as_str() + ); + + let mut data = synthetic_queue(®istry, &["configuration.remoteUser"]); + triage(&mut data.findings[0], non_promotable); + + // 1. The promotion path refuses, and leaves the record exactly as it was. + let before = data.findings[0].clone(); + let err = data.findings[0] + .promote(&case.id) + .expect_err("a machinery defect is not a behavior of either implementation"); + assert!(matches!(err, queue::TransitionError::NonPromotable { .. })); + assert!(err.to_string().contains("not promotable"), "{err}"); + assert!( + err.to_string().contains("normalizer or the generator"), + "the diagnosis must name where the fix belongs: {err}" + ); + assert_eq!( + data.findings[0], before, + "a refused promotion writes nothing" + ); + assert_clean(&data, ®istry, "refused-promotion"); + + // 2. And a hand edit that bypassed the path does not stand. + data.findings[0].state = queue::FindingState::Promoted; + data.findings[0].promoted_to = Some(case.id.clone()); + let violations = queue::check(&data, &view); + assert!( + violations + .iter() + .any(|v| v.class() == "D2" && v.to_string().contains("not promotable")), + "a hand-edited promotion of `{}` must be D2: {violations:?}", + non_promotable.as_str() + ); + } + + // The four promotable classifications really do promote, or the assertions above would + // pass equally for a promotion path that refuses everything. + for promotable in [ + queue::Classification::DeaconRegression, + queue::Classification::ReferenceQuirk, + queue::Classification::SpecAmbiguity, + queue::Classification::UnsupportedBehavior, + ] { + let mut data = synthetic_queue(®istry, &["configuration.remoteUser"]); + triage(&mut data.findings[0], promotable); + data.findings[0] + .promote(&case.id) + .unwrap_or_else(|e| panic!("{} must promote: {e}", promotable.as_str())); + assert_eq!(data.findings[0].state, queue::FindingState::Promoted); + assert_clean(&data, ®istry, "promoted"); + } + + // The process-level half must still exist: `discovery scaffold` refusing a + // non-promotable finding is what a reviewer actually meets, and deleting that test + // would leave the refusal asserted only where no reviewer runs it. + let cli_guard = workspace_root().join("crates/conformance/tests/discovery_cli.rs"); + let cli_guard_text = std::fs::read_to_string(&cli_guard) + .unwrap_or_else(|e| panic!("the process-level promotion guard {cli_guard:?}: {e}")); + assert!( + cli_guard_text.contains("fn scaffold_writes_nothing_and_refuses_a_non_promotable_finding"), + "{cli_guard:?} must keep asserting the refusal from outside the process" + ); +} + +/// **T065 / FR-033**: a finding that stops reproducing is *reported* with the campaign that +/// last observed it — never deleted. +/// +/// Deleting it would destroy the ability to distinguish two very different situations: a +/// fix landed, or the generator stopped reaching that input. The first is success; the +/// second is a coverage regression in the discovery machinery itself. Only the retained +/// record makes them separable, and only the retained *last observation* says which run to +/// go back to. +#[test] +fn a_finding_that_stops_reproducing_is_reported_with_its_last_campaign() { + let registry = load_registry(); + let pins = discovery_report::CurrentPins::from_registry(®istry); + let pinned = synthetic_pins(®istry); + let first = synthetic_campaign(&pinned, "0x5eed0011", 1, 0); + let second = synthetic_campaign(&pinned, "0x5eed0012", 1, 0); + + let signature = signature_at("configuration.remoteUser"); + let mut findings: Vec = Vec::new(); + queue::upsert_finding( + &mut findings, + signature.clone(), + synthetic_witness(&first, "cnd-00000001"), + &first.id, + ); + queue::upsert_finding( + &mut findings, + signature.clone(), + synthetic_witness(&second, "cnd-00000002"), + &second.id, + ); + triage(&mut findings[0], queue::Classification::DeaconRegression); + + // A third campaign runs and does not reproduce it. + let third = synthetic_campaign(&pinned, "0x5eed0013", 0, 0); + findings[0] + .mark_no_longer_reproducing() + .expect("a triaged finding may stop reproducing"); + + let data = DiscoveryData { + findings, + campaigns: vec![first, second.clone(), third], + corpus: Vec::new(), + }; + assert_clean(&data, ®istry, "no-longer-reproducing"); + + let report = discovery_report::build_queue_report(&data, &pins); + assert_eq!(report.total, 1, "the record is retained, not deleted"); + assert_eq!(report.no_longer_reproducing.len(), 1); + let summary = &report.no_longer_reproducing[0]; + assert_eq!( + summary.last_observed, second.id, + "the bucket must name the campaign that LAST observed it — the run a reviewer goes \ + back to" + ); + assert_eq!( + summary.classification.as_deref(), + Some("deacon-regression"), + "the reviewer's judgement survives the disappearance" + ); + + let md = discovery_report::render_md(&report); + assert!(md.contains("| no-longer-reproducing | 1 |"), "{md}"); + assert!( + md.contains(&second.id), + "the campaign that last saw it must be named in the artifact: {md}" + ); + assert!( + md.contains("Retained, not deleted"), + "the report must say why the record is still there: {md}" + ); + + // And a later campaign that reproduces it revives it to `triaged`, KEEPING the + // classification — re-triaging a finding a reviewer already judged is wasted work. + let fourth = synthetic_campaign(&pinned, "0x5eed0014", 1, 0); + let mut revived = data.findings.clone(); + assert_eq!( + queue::upsert_finding( + &mut revived, + signature, + synthetic_witness(&fourth, "cnd-00000004"), + &fourth.id + ), + queue::Upsert::WitnessAppended + ); + assert_eq!(revived[0].state, queue::FindingState::Triaged); + assert_eq!( + revived[0].classification, + Some(queue::Classification::DeaconRegression) + ); + assert_eq!(revived[0].last_observed, fourth.id); +} + +/// **T066 / FR-029**: the untriaged count is visible, so "not yet looked at" can never read +/// as "nothing found". +/// +/// A status code cannot carry this distinction and neither can a bare list — the count has +/// to be in the artifact, next to the total, where a reader who is skimming sees it. The +/// three queues below are the three states that would otherwise be conflated: nothing +/// found, nothing looked at, and everything looked at. +#[test] +fn the_untriaged_bucket_is_counted_so_nothing_looked_at_never_reads_as_nothing_found() { + let registry = load_registry(); + let pins = discovery_report::CurrentPins::from_registry(®istry); + let paths = [ + "configuration.remoteUser", + "configuration.workspaceFolder", + "configuration.userEnvProbe", + ]; + + let empty = discovery_report::build_queue_report(&DiscoveryData::default(), &pins); + let untouched = synthetic_queue(®istry, &paths); + let mut all_triaged = synthetic_queue(®istry, &paths); + for finding in &mut all_triaged.findings { + triage(finding, queue::Classification::DeaconRegression); + } + let untouched_report = discovery_report::build_queue_report(&untouched, &pins); + let triaged_report = discovery_report::build_queue_report(&all_triaged, &pins); + + // Nothing found vs nothing looked at: the same total would be a lie, and the same + // untriaged count would hide the backlog. + assert_eq!((empty.total, empty.untriaged.len()), (0, 0)); + assert_eq!( + (untouched_report.total, untouched_report.untriaged.len()), + (3, 3) + ); + // Everything looked at: zero untriaged, and the total says the queue is NOT empty. A + // report that only carried the untriaged count would render this identically to the + // empty queue. + assert_eq!( + (triaged_report.total, triaged_report.untriaged.len()), + (3, 0) + ); + + for (label, report, untriaged) in [ + ("empty", &empty, 0usize), + ("untouched", &untouched_report, 3), + ("all-triaged", &triaged_report, 0), + ] { + let md = discovery_report::render_md(report); + assert!( + md.contains(&format!("| untriaged | {untriaged} |")), + "the {label} report must COUNT the untriaged bucket, not merely list it: {md}" + ); + assert!( + md.contains(&format!("| total | {} |", report.total)), + "the {label} report must carry the total beside it: {md}" + ); + let json: serde_json::Value = + serde_json::from_str(&discovery_report::render_json(report)).expect("valid JSON"); + assert_eq!( + json["untriaged"].as_array().map(Vec::len), + Some(untriaged), + "the {label} machine-readable artifact must carry the bucket too" + ); + } + + // Every untriaged finding is individually named, so the bucket is actionable rather + // than only a number. + for (summary, path) in untouched_report.untriaged.iter().zip(paths) { + assert_eq!(summary.path, path); + assert!(summary.id.starts_with("fnd-")); + assert_eq!(summary.classification, None); + } + + // And an empty queue says what its emptiness does NOT mean. + let empty_md = discovery_report::render_md(&empty); + assert!( + empty_md.contains("it does not say the two implementations agree"), + "{empty_md}" + ); +} + +/// **T125 / FR-018**: no discovery source authors or extends an allowed-difference entry. +/// +/// The allowed-difference mechanism records **reviewed** tolerances. A discovery program +/// writing to it would let a difference disappear by being observed — the machinery would +/// grow quieter exactly as it found more, and the growth would look like progress. +/// +/// Note what is deliberately **not** forbidden: *reading* the mechanism. FR-017 requires a +/// difference already covered by a case, waiver, or allowed difference to be reported as +/// already-characterized rather than entering the queue as new, and that is exactly a read. +/// The rule is about the write, so the guard is about the write — and it asserts the read +/// still happens, because a scan that matched nothing would pass by checking nothing. +#[test] +fn no_discovery_source_writes_to_the_allowed_difference_mechanism() { + let sources = discovery_sources(); + for required in [ + "crates/conformance/src/discovery/queue.rs", + "crates/parity-harness/src/discovery/differential.rs", + "crates/parity-harness/src/discovery/campaign.rs", + "crates/conformance/src/bin/conformance.rs", + ] { + assert!( + sources.iter().any(|(p, _)| p == required), + "the scan must cover {required}; got {:?}", + sources.iter().map(|(p, _)| p).collect::>() + ); + } + + /// Tokens that would AUTHOR or EXTEND a tolerance rather than read one. + /// + /// Constructing the record, pushing one onto a case, or emitting the JSON key — each + /// is a way to make a difference disappear by having observed it. + const FORBIDDEN: &[&str] = &[ + "AllowedDifference {", + "AllowedDifference::", + "allowed_differences.push", + "allowed_differences.insert", + "allowed_differences.extend", + "allowed_differences.append", + "allowed_differences =", + "allowed_differences:", + "\"allowedDifferences\":", + "\"allowedDifferences\" :", + ]; + + let mut problems = Vec::new(); + let mut reads = 0usize; + for (path, text) in &sources { + for (line_no, line) in text.lines().enumerate() { + let code = line.trim_start(); + if code.starts_with("//") || code.starts_with("/*") || code.starts_with('*') { + continue; + } + for needle in FORBIDDEN { + if code.contains(needle) { + problems.push(format!( + "{path}:{}: `{needle}` authors or extends an allowed-difference \ + entry. That mechanism records REVIEWED tolerances; a discovery \ + program writing to it would let a difference disappear by being \ + observed (FR-018). Report the difference as a finding and let a \ + human decide whether it is tolerable.", + line_no + 1 + )); + } + } + if code.contains("allowed_differences") { + reads += 1; + } + } + } + + assert!( + problems.is_empty(), + "the discovery surface must never author a tolerance:\n{}", + problems.join("\n") + ); + assert!( + reads > 0, + "the scan found no reference to `allowed_differences` at all. FR-017 requires \ + discovery to READ the mechanism so an already-characterized difference does not \ + enter the queue as new — so zero references means either that read was lost or \ + this guard is matching nothing, and both are defects." + ); +} + +/// Every source file that makes up the discovery surface, hermetic **and** live. +/// +/// The live half is the half that could plausibly author a tolerance: it is the side +/// holding the registry it would have to write to. Scanning only the hermetic half would +/// guard the place the mistake cannot happen. +fn discovery_sources() -> Vec<(String, String)> { + /// This file. Excluded because it *is* the guard: its forbidden-token table contains + /// every pattern it searches for, so scanning itself would fail on its own definition. + /// Nothing is lost — a test file is not a discovery program, and the programs are all + /// scanned below. + const THE_GUARD_ITSELF: &str = "discovery_hermetic.rs"; + + let mut out = hermetic_discovery_sources(); + // Whole directories: every module of the live discovery half, scanned without a name + // filter. The live half is the one that could plausibly author a tolerance — it is the + // side holding the registry — and a name filter there would have quietly skipped + // `differential.rs`, which is exactly where the mistake would live. + out.extend(rust_sources_in( + "crates/parity-harness/src/discovery", + |_| true, + )); + // Directories that hold unrelated programs too: take the discovery ones by name, plus + // `conformance.rs` in full — that binary hosts the whole dev CLI of which `discovery` + // is one command group, so a write anywhere in it is reachable from `discovery` + // regardless of which function holds it. + for dir in [ + "crates/parity-harness/src/bin", + "crates/conformance/src/bin", + "crates/deacon/tests", + ] { + out.extend(rust_sources_in(dir, |name| { + name != THE_GUARD_ITSELF && (name.starts_with("discovery") || name == "conformance.rs") + })); + } + out.sort(); + out +} + +/// Every `.rs` file directly under `dir` (workspace-relative) whose file name `keep` +/// accepts, as `(relative path, contents)`. +fn rust_sources_in(dir: &str, keep: impl Fn(&str) -> bool) -> Vec<(String, String)> { + let root = workspace_root().join(dir); + let mut out = Vec::new(); + let Ok(rd) = std::fs::read_dir(&root) else { + return out; + }; + for entry in rd.filter_map(Result::ok) { + let path = entry.path(); + if path.extension().and_then(|s| s.to_str()) != Some("rs") { + continue; + } + let name = path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or_default() + .to_string(); + if !keep(&name) { + continue; + } + let text = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {path:?}: {e}")); + out.push((format!("{dir}/{name}"), text)); + } + out +} + +// --------------------------------------------------------------------------- +// T099/T100/T104 (US7) — the pinned real-world corpus, checked WITHOUT a network +// --------------------------------------------------------------------------- + +/// The committed corpus manifest. +fn load_corpus() -> Vec { + load_discovery().corpus +} + +/// One well-formed entry, id derived rather than chosen (the same discipline +/// [`synthetic_campaign`] follows: a hand-picked id would fail the D4 identity clause and +/// the fixture would stop being the clean manifest these tests need). +fn corpus_entry(repository: &str, commit: &str, path: &str) -> CorpusEntry { + CorpusEntry { + id: CorpusEntry::derive_id(repository, commit, path), + name: format!("synthetic-{}", repository.replace('/', "-")), + repository: repository.to_string(), + commit: commit.to_string(), + path: path.to_string(), + content_digest: None, + notes: String::new(), + } +} + +/// **T099 / FR-050 / SC-012**: a branch, a tag, `HEAD`, or `latest` is **D4**, rejected +/// with no network access whatsoever. +/// +/// This test is the entire argument for moving the manifest out of the Python fetcher and +/// into Rust-owned strict JSON (research D8). FR-050 is a property of the *manifest*, not +/// of a fetch — nothing needs to be retrieved to know that `main` names different content +/// tomorrow — so the check belongs somewhere it runs on every pull request. A validation +/// that only runs when the network is up is a validation that does not run. +/// +/// The no-network claim is structural rather than asserted here: `deacon-conformance` +/// declares no HTTP client, no async runtime, and no git transport, and no hermetic +/// discovery module may spawn a process or open a socket — +/// [`the_hermetic_discovery_surface_cannot_reach_the_network`] enforces both. This test +/// exercises the rule that guard makes reachable. +#[test] +fn a_mutable_corpus_reference_is_rejected_hermetically() { + // Every floating shape FR-050 names, plus the near-misses a naive length or hex check + // would wave through. + for mutable in [ + "main", + "master", + "HEAD", + "latest", + "v1.2.3", + "refs/heads/main", + "release/2024-01", + "0123456", // abbreviated + "0123456789ABCDEF0123456789ABCDEF01234567", // uppercase + "0123456789abcdef0123456789abcdef0123456", // 39 + "0123456789abcdef0123456789abcdef012345678", // 41 + "0123456789abcdef0123456789abcdef0123456g", // non-hex + "", + ] { + let entry = corpus_entry("microsoft/vscode-remote-try-node", mutable, ""); + let violations = corpus::check(std::slice::from_ref(&entry)); + assert!( + violations.iter().any(|v| v.class() == "D4"), + "`{mutable}` must be rejected as D4; got {violations:?}" + ); + } + + // And the same rule reaches the real validator over a real data root, so the class is + // wired in rather than merely implemented. + let registry = load_registry(); + let mut data = load_discovery(); + data.corpus + .push(corpus_entry("owner/repo", "main", "workspace")); + let violations = queue::check(&data, &queue::RegistryView::from_registry(®istry)); + assert!( + violations.iter().any(|v| v.class() == "D4"), + "`discovery check` must surface D4 over the data root: {violations:?}" + ); + + // The committed manifest itself is clean — otherwise the assertion above would be + // measuring a pre-existing violation rather than the one it injected. + assert!( + corpus::check(&load_corpus()).is_empty(), + "the committed corpus manifest must be D4-clean" + ); +} + +/// **T100 / FR-049**: every entry records a repository, an immutable commit, the path +/// within the repository, and a content digest. +/// +/// The digest field is `null` until first materialization and non-null (and verified) +/// forever after, so what FR-049 requires is that the *slot* is there and is either +/// absent-by-design or well formed — never a placeholder, never a truncated hash, never a +/// value nothing could compare against. +#[test] +fn every_corpus_entry_records_its_full_provenance() { + let corpus = load_corpus(); + assert_eq!( + corpus.len(), + 33, + "the manifest carries the 33 pinned entries the frozen `realworld::` \ + baseline units were derived from; a silently shrinking corpus explores less and \ + reports the same 'found nothing'" + ); + + for entry in &corpus { + assert!( + entry.id.starts_with("cor-"), + "{}: a corpus id is `cor-`", + entry.id + ); + assert_eq!( + entry.id, + entry.derived_id(), + "{}: the id must derive from `repository ‖ commit ‖ path`, or the record is \ + detached from the snapshot it claims to identify", + entry.name + ); + assert!( + !entry.name.is_empty(), + "{}: an entry needs a name", + entry.id + ); + assert!( + entry.repository.split('/').count() == 2 + && entry.repository.split('/').all(|p| !p.is_empty()), + "{}: `repository` must be `owner/repo`, got {:?}", + entry.id, + entry.repository + ); + assert!( + corpus::is_immutable_reference(&entry.commit), + "{}: `commit` must be a 40-hex object name, got {:?}", + entry.id, + entry.commit + ); + // `path` is a recorded field that may legitimately be empty (the repository root), + // so the assertion is about its SHAPE: a leading or trailing slash would make two + // spellings of one workspace root derive two different ids. + assert_eq!( + entry.path.trim_matches('/'), + entry.path, + "{}: `path` must be recorded without leading or trailing slashes, or two \ + spellings of one workspace root would derive two different ids", + entry.id + ); + match &entry.content_digest { + None => {} // Not yet materialized — the one legitimate absence. + Some(digest) => assert!( + corpus::is_well_formed_digest(digest), + "{}: a recorded digest must be `sha256:<64 lowercase hex>`, got {digest:?} \ + — a malformed digest is not a weaker check, it is one that can never \ + disagree", + entry.id + ), + } + assert!( + !entry.notes.trim().is_empty(), + "{}: an entry records WHY this workspace was selected; an unexplained pin \ + cannot be re-pinned by anyone but its author", + entry.id + ); + } + + // Provenance is only provenance if it is unique. Two entries sharing an id would make + // one snapshot two records; two sharing a name would make the frozen + // `realworld::` baseline reference ambiguous. + let mut ids: Vec<&str> = corpus.iter().map(|e| e.id.as_str()).collect(); + ids.sort_unstable(); + let before = ids.len(); + ids.dedup(); + assert_eq!(before, ids.len(), "corpus ids must be unique"); + + let mut names: Vec<&str> = corpus.iter().map(|e| e.name.as_str()).collect(); + names.sort_unstable(); + let before = names.len(); + names.dedup(); + assert_eq!(before, names.len(), "corpus names must be unique"); +} + +/// **T104 / FR-008a / FR-054a**: no corpus entry appears among the generator's mutation +/// seeds. +/// +/// The corpus is consumed **only** as a direct comparison input in the network-backed +/// lane. Letting it feed the generator would do two separate kinds of damage: the +/// generated space would stop being reproducible from a seed alone (it would depend on +/// what a third party's repository contained at fetch time), and third-party content would +/// enter the hermetic pull-request lane, which by FR-055 performs no network access at all +/// and therefore could never obtain it. +/// +/// The separation is **structural**, not a naming convention. The seed corpus is five +/// committed fixtures embedded with `include_str!`, so its membership is fixed when the +/// crate compiles; a fetched workspace does not exist then and cannot be added later. +/// This test asserts both halves — that the two sets are disjoint, and that the seeds come +/// from the committed fixture tree rather than the discovery data root. +#[test] +fn no_corpus_entry_is_a_mutation_seed() { + let seeds = generate::seed_fixture_names(); + assert!( + !seeds.is_empty(), + "a scan over an empty seed set would pass by checking nothing" + ); + + let corpus = load_corpus(); + for entry in &corpus { + assert!( + !seeds.contains(&entry.name.as_str()), + "corpus entry `{}` is also a mutation seed — the corpus is a direct comparison \ + input only (FR-054a), and seeding the generator from it would make the \ + generated space depend on third-party content the hermetic lane can never \ + fetch", + entry.name + ); + assert!( + !seeds.iter().any(|s| s.contains(&entry.name)), + "seed fixture names must not embed the corpus entry name `{}`", + entry.name + ); + } + + // The other direction, and the load-bearing one: the seeds are committed fixtures. + // A seed whose bytes came from `conformance/discovery/` would be corpus content in + // the generator no matter what it was called. + let fixtures = workspace_root().join("fixtures"); + for seed in &seeds { + assert!( + !corpus.iter().any(|e| e.name == *seed), + "seed `{seed}` names a corpus entry" + ); + } + assert!( + fixtures.is_dir(), + "the seed corpus is the committed fixture tree at {fixtures:?}" + ); + + // And the manifest is a *manifest*: it records provenance, never bytes (FR-053), so + // there is nothing in it a generator could seed from even if one tried. + for entry in &corpus { + assert!( + !entry.workspace_dir(&workspace_root()).exists(), + "corpus content must never be vendored into this repository (FR-053), but \ + {} exists", + entry.id + ); + } +} + +// =========================================================================== +// User Story 5 — promote a finding only through review (T074–T078, T083, T126) +// =========================================================================== + +/// The registry- and snapshot-owned write helpers **no** discovery program may reference. +/// +/// Named by the function each one is, spelled with its opening paren: a bare word that +/// happened to appear in prose would make this a scan that passes by matching nothing. +/// Behaviors, cases, waivers, and allowed differences have no writer at all — they are +/// hand-authored files the loader only reads — so what is scanned for is every writer that +/// *could* be pointed at the deterministic record. +const FORBIDDEN_WRITERS: [&str; 5] = [ + // Committed reference snapshots (022) — only the reviewed refresh bin writes one. + "write_snapshot(", + // The machine-owned constraint / clause inventories (020, 021). + "write_inventory(", + "write_clauses(", + // The machine-owned obligation set (024). + "write_obligations(", + // The frozen migration baseline (023). + "write_baseline(", +]; + +/// Every source tree a "discovery program" is made of. +/// +/// Both halves of the hermetic/live split (research D4) plus the two live bins. The +/// `discovery` command group lives in `crates/conformance/src/bin/conformance.rs` beside +/// commands that legitimately write the inventory, so a whole-file scan there would be a +/// false positive; its behavior is asserted from **outside the process** instead, by +/// `crates/conformance/tests/discovery_cli.rs`'s +/// `the_discovery_group_never_writes_into_the_registry`. +const DISCOVERY_SOURCE_ROOTS: [&str; 4] = [ + "crates/conformance/src/discovery", + "crates/parity-harness/src/discovery", + "crates/parity-harness/src/bin/discovery-campaign.rs", + "crates/parity-harness/src/bin/discovery-proof.rs", +]; + +/// Collect `.rs` files under `path` (which may itself be a file). +fn rust_sources(path: &Path, out: &mut Vec) { + if path.is_file() { + if path.extension().is_some_and(|e| e == "rs") { + out.push(path.to_path_buf()); + } + return; + } + let Ok(entries) = std::fs::read_dir(path) else { + return; + }; + for entry in entries.flatten() { + rust_sources(&entry.path(), out); + } +} + +/// **T083 / SC-008**: no discovery source file references a registry or snapshot write +/// helper. +/// +/// Modelled directly on `only_the_refresh_bin_writes_committed_snapshots` (022 T038), and +/// for the same reason: the property that matters is not "no discovery program writes the +/// record today" but "a write path introduced tomorrow fails a test". A behavioral check +/// can only observe the paths a run happens to take; a structural one observes the paths +/// that exist. +/// +/// Both the scan and the forbidden list carry positive controls, because the failure mode +/// of a guard like this is passing by looking at nothing. +#[test] +fn no_discovery_source_references_a_registry_or_snapshot_writer() { + let root = workspace_root(); + let mut sources: Vec = Vec::new(); + for rel in DISCOVERY_SOURCE_ROOTS { + let path = root.join(rel); + assert!( + path.exists(), + "the scan targets {rel}, which does not exist — a guard that scans nothing \ + passes by checking nothing" + ); + rust_sources(&path, &mut sources); + } + assert!( + sources.len() >= 10, + "expected the discovery source trees to hold both halves of the split; found only \ + {} file(s)", + sources.len() + ); + + let mut offenders: Vec = Vec::new(); + for source in &sources { + let text = std::fs::read_to_string(source) + .unwrap_or_else(|e| panic!("could not read {}: {e}", source.display())); + for writer in FORBIDDEN_WRITERS { + if text.contains(writer) { + offenders.push(format!( + "{} references `{writer}`", + source + .strip_prefix(&root) + .unwrap_or(source) + .to_string_lossy() + .replace('\\', "/") + )); + } + } + } + assert_eq!( + offenders, + Vec::::new(), + "a discovery program must not be able to write the deterministic record (FR-036): \ + promotion is a human editing the registry with a scaffold as a starting point, and \ + a stochastic process that could author the record it is tested against would make \ + every claim in that record unfalsifiable" + ); + + // Positive control: each forbidden writer is a real function with real callers + // SOMEWHERE, so an empty offender list means "not in discovery" rather than "these + // names no longer exist and the scan matches nothing". + let mut all: Vec = Vec::new(); + for crate_src in ["conformance/src", "parity-harness/src"] { + rust_sources(&root.join("crates").join(crate_src), &mut all); + } + for writer in FORBIDDEN_WRITERS { + assert!( + all.iter() + .any(|p| std::fs::read_to_string(p).is_ok_and(|t| t.contains(writer))), + "`{writer}` appears nowhere in the workspace, so scanning for it proves nothing \ + — the helper was renamed and this guard silently stopped guarding" + ); + } +} + +/// The four `conformance/` roots a discovery program must never touch — the homes of the +/// six record kinds FR-036 enumerates, plus the machine-owned artifacts that back them. +const DETERMINISTIC_RECORD_ROOTS: [&str; 4] = ["registry", "snapshots", "obligations", "inventory"]; + +/// Every file under `dir`, with its bytes, for a before/after comparison. +fn byte_census(dir: &Path) -> std::collections::BTreeMap> { + fn walk(dir: &Path, out: &mut std::collections::BTreeMap>) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + walk(&path, out); + } else if let Ok(bytes) = std::fs::read(&path) { + out.insert(path, bytes); + } + } + } + let mut out = std::collections::BTreeMap::new(); + walk(dir, &mut out); + out +} + +/// **T074 / SC-008**: exercising every writer the discovery half owns leaves the +/// deterministic record byte-identical. +/// +/// The behavioral companion to the structural scan above, and the two are complementary +/// rather than redundant: the scan proves no *path* exists, this proves the paths that DO +/// exist go somewhere else. Together they cover the two ways this could break — a new call +/// to a registry writer, and an existing discovery writer being pointed at a registry path. +#[test] +fn no_discovery_writer_can_reach_the_deterministic_record() { + use deacon_conformance::discovery::promote; + + let registry = load_registry(); + let conformance = deacon_conformance::default_registry_dir() + .parent() + .map(Path::to_path_buf) + .expect("the registry dir has a parent"); + + let before: Vec<_> = DETERMINISTIC_RECORD_ROOTS + .iter() + .map(|name| byte_census(&conformance.join(name))) + .collect(); + assert!( + before.iter().any(|c| !c.is_empty()), + "the census read nothing; a before/after comparison over an empty set passes by \ + comparing nothing" + ); + + let scratch = tempfile::tempdir().expect("tempdir"); + let data = synthetic_queue( + ®istry, + &["configuration.remoteUser", "configuration.image"], + ); + + // Every writer the discovery half exposes, driven for real against a scratch root. A + // regression that hardcoded a `conformance/registry/…` destination would be caught by + // the census below rather than by reading the code. + queue::write_findings(scratch.path(), &data.findings).expect("findings are writable"); + queue::write_campaigns(scratch.path(), &data.campaigns).expect("campaigns are writable"); + corpus::write(scratch.path(), &[]).expect("the corpus manifest is writable"); + let pins = discovery_report::CurrentPins::from_registry(®istry); + let report = discovery_report::build_queue_report(&data, &pins); + discovery_report::write_queue_report(scratch.path(), &report).expect("the report is writable"); + + // And the two promotion surfaces, which return documents and take no path at all — + // asserted here so "scaffold writes nothing" is checked rather than assumed. + let finding = data.findings.first().expect("a synthetic finding"); + promote::promotion_skeleton(finding).expect("a promotion skeleton"); + promote::tolerance_skeleton(finding).expect("a tolerance skeleton"); + + for (name, census) in DETERMINISTIC_RECORD_ROOTS.iter().zip(before) { + assert_eq!( + byte_census(&conformance.join(name)), + census, + "`conformance/{name}` changed while exercising the discovery writers; a \ + discovery program that can author a behavior, case, waiver, tolerated \ + difference, disposition, or snapshot makes the record it is tested against \ + unfalsifiable (FR-036)" + ); + } + + // The writers DID write — otherwise the assertion above is satisfied by a run in which + // nothing happened at all. + for name in [ + "findings.json", + "campaigns.json", + "corpus.json", + "queue.json", + ] { + assert!( + scratch.path().join(name).is_file(), + "{name} was not written to the scratch root, so the guard above compared a \ + record nothing tried to change" + ); + } +} + +/// **T075 / SC-009**: a promoted finding's case is an **ordinary** case — it satisfies +/// every validation rule a hand-authored one does, including a full scenario context and +/// the coverage-obligation records that accompany it. +/// +/// Asserted against a real committed case rather than a synthetic one, because the claim is +/// precisely that promotion produces nothing special: if a promoted case needed its own +/// validation path, "passes the full existing record validation" would be a claim about a +/// different validator. +#[test] +fn a_promoted_findings_case_satisfies_the_ordinary_record_validation() { + let registry = load_registry(); + + // A case with a FULL scenario context — the shape FR-040 requires a promoted case to + // have (V26: assign every dimension or none). + let dimensions: Vec<&str> = registry.scenario.iter().map(|d| d.id.as_str()).collect(); + assert!( + !dimensions.is_empty(), + "the registry declares no scenario dimensions, so `scenarioContext` completeness \ + would be satisfied by every case vacuously" + ); + let case = registry + .cases + .iter() + .find(|c| { + !c.scenario_context.is_empty() + && dimensions + .iter() + .all(|d| c.scenario_context.contains_key(*d)) + }) + .expect( + "the registry must carry at least one case with a complete scenarioContext — \ + that is the shape a promotion has to produce", + ); + + // The queue half: a finding promoted to it resolves (**D3** clean). + let mut data = synthetic_queue(®istry, &["configuration.remoteUser"]); + data.findings[0] + .triage( + queue::Classification::DeaconRegression, + Some("promoted by the SC-009 guard"), + ) + .expect("a promotable classification is accepted"); + data.findings[0] + .promote(&case.id) + .expect("promotion is permitted"); + assert_eq!( + queue::check(&data, &queue::RegistryView::from_registry(®istry)), + Vec::new(), + "a finding promoted to a real case must validate clean; the promotion is the only \ + reference that crosses out of the discovery root and it has to resolve" + ); + + // The registry half: the case passes the FULL existing validation, unchanged. Nothing + // here is discovery-specific — that is the point. + let registry_violations = deacon_conformance::validate::validate_path( + &deacon_conformance::default_registry_dir(), + "2026-07-27", + &workspace_root(), + ) + .expect("the committed registry loads"); + let about_the_case: Vec = registry_violations + .iter() + .filter(|v| v.record == case.id) + .map(|v| format!("{} {}: {}", v.code, v.record, v.message)) + .collect(); + assert_eq!( + about_the_case, + Vec::::new(), + "the case a promotion names must satisfy every rule a hand-authored case does" + ); + + // And the coverage-obligation half (FR-040): the case is cited by at least one + // obligation disposition. This is the trap 024 documents — adding a case and + // registering only the BEHAVIOR disposition leaves the combination records reading + // `gap` beside a case that covers them. + assert!( + registry + .obligation_dispositions + .iter() + .any(|d| d.cases.iter().any(|c| c == &case.id)), + "case `{}` is cited by no obligation disposition; a promoted case whose obligations \ + were never flipped off `gap` leaves the coverage report claiming a hole that is \ + filled (FR-040)", + case.id + ); +} + +/// **T076 / FR-038**: a promotion lacking a behavior identity or a disposition fails +/// validation **naming what is missing**. +/// +/// Both halves, because they fail in different places and a reviewer meets them at +/// different moments: the pre-flight refuses an incomplete record while the reviewer can +/// still act on it cheaply, and **D3** refuses an unresolvable promotion afterwards, over +/// committed data, where a deleted or renamed case would otherwise leave a finding reading +/// as covered while nothing executes it. +#[test] +fn a_promotion_missing_an_identity_or_a_disposition_fails_and_says_which() { + use deacon_conformance::discovery::promote::{self, PromotionError}; + + let registry = load_registry(); + let mut data = synthetic_queue(®istry, &["configuration.remoteUser"]); + data.findings[0] + .triage(queue::Classification::DeaconRegression, None) + .expect("classification accepted"); + let finding = &data.findings[0]; + + // One axis missing at a time, so a checker that reported "something is missing" without + // saying which would fail here rather than passing on a lump. + for missing in promote::BEHAVIOR_DISPOSITION_AXES { + let mut behavior = serde_json::json!({ + "id": "bhv-promoted-by-review", + "spec": "conformant", + "reference": "divergent", + "decision": "follow-spec", + }); + behavior + .as_object_mut() + .expect("an object") + .remove(missing) + .expect("the axis was present"); + + let errors = promote::validate_promotion(finding, &behavior, Some("case-x"), &["case-x"]); + assert_eq!( + errors.len(), + 1, + "exactly one objection is expected when exactly one axis is missing: {errors:?}" + ); + match &errors[0] { + PromotionError::MissingDisposition { axis, detail, .. } => { + assert_eq!(*axis, missing); + assert!( + detail.contains(missing), + "the diagnosis must NAME the axis: {detail}" + ); + } + other => panic!("expected a missing-disposition objection, got {other:?}"), + } + } + + // No behavior identity at all. + let anonymous = serde_json::json!({ + "spec": "conformant", + "reference": "divergent", + "decision": "follow-spec", + }); + let errors = promote::validate_promotion(finding, &anonymous, Some("case-x"), &["case-x"]); + assert!( + errors.iter().any(|e| matches!( + e, + PromotionError::MissingBehaviorIdentity { detail, .. } if detail.contains("`id` is absent") + )), + "a promotion with no behavior identity cannot be found again, which is the whole \ + point of promoting it: {errors:?}" + ); + + // The committed-data half: **D3** names the promotion that does not resolve. + let mut promoted = synthetic_queue(®istry, &["configuration.image"]); + promoted.findings[0] + .triage(queue::Classification::ReferenceQuirk, None) + .expect("classification accepted"); + promoted.findings[0] + .promote("case-that-nobody-committed") + .expect("the state machine permits the transition; D3 is what refuses it"); + let violations = queue::check(&promoted, &queue::RegistryView::from_registry(®istry)); + assert!( + violations + .iter() + .any(|v| v.class() == "D3" && v.to_string().contains("case-that-nobody-committed")), + "D3 must NAME the case that does not resolve: {violations:?}" + ); +} + +/// Recursively copy `from` into `to`. +fn copy_tree(from: &Path, to: &Path) { + std::fs::create_dir_all(to).unwrap_or_else(|e| panic!("create {}: {e}", to.display())); + let entries = + std::fs::read_dir(from).unwrap_or_else(|e| panic!("read {}: {e}", from.display())); + for entry in entries.flatten() { + let src = entry.path(); + let dst = to.join(entry.file_name()); + if src.is_dir() { + copy_tree(&src, &dst); + } else { + std::fs::copy(&src, &dst).unwrap_or_else(|e| panic!("copy {}: {e}", src.display())); + } + } +} + +/// **T077 / SC-018**: `certify`'s verdict with a queue holding unreviewed findings is +/// **identical** to its verdict with an empty queue. +/// +/// Verified by varying only the discovery root's content beside a registry copy, rather +/// than by reading the loader: the guarantee research D6 states is structural — the loader +/// enumerates *named* subdirectories under `conformance/registry/` and has no wildcard walk +/// at the root, so a sibling of `registry/` has no code path that could reach it — and the +/// way to check a structural claim is to try to break it. +/// +/// The queue is asserted non-empty, untriaged, and clean, because "certification is +/// unchanged" is trivially true of a queue that does not load. +#[test] +fn certification_is_unchanged_by_a_queue_full_of_unreviewed_findings() { + use deacon_conformance::certify::certify; + use deacon_conformance::validate::{ClauseInputs, InventoryInputs}; + + let real_registry_dir = deacon_conformance::default_registry_dir(); + let conformance = real_registry_dir + .parent() + .map(Path::to_path_buf) + .expect("the registry dir has a parent"); + + // A scratch `conformance/`-shaped root: a COPY of the real registry (so every record + // resolves) beside a discovery root this test owns. Copied rather than symlinked so the + // guard behaves identically on every platform. + let scratch = tempfile::tempdir().expect("tempdir"); + let scratch_registry = scratch.path().join("registry"); + copy_tree(&real_registry_dir, &scratch_registry); + let scratch_discovery = scratch.path().join("discovery"); + std::fs::create_dir_all(&scratch_discovery).expect("discovery root"); + + // The inventory / clause / snapshot inputs are read-only and unrelated to the queue, so + // they point at the real committed tree — varying them would be varying the thing this + // test holds constant. + let schemas_dir = conformance.join("schemas"); + let inventory_file = conformance.join("inventory").join("constraints.json"); + let spec_dir = conformance.join("spec"); + let clauses_file = conformance.join("inventory").join("clauses.json"); + let snapshots_dir = conformance.join("snapshots"); + + let run = || -> String { + let registry = Registry::load(&scratch_registry).expect("the registry copy loads"); + let result = certify( + ®istry, + "2026-07-27", + &InventoryInputs { + schemas_dir: &schemas_dir, + inventory_file: &inventory_file, + }, + &ClauseInputs { + spec_dir: &spec_dir, + clauses_file: &clauses_file, + }, + &snapshots_dir, + ); + serde_json::to_string_pretty(&result).expect("certification serializes") + }; + + // An EMPTY queue. + let empty: FindingsFile = serde_json::from_value(serde_json::json!({ + "schemaVersion": queue::SCHEMA_VERSION, + "records": [] + })) + .expect("an empty findings file"); + std::fs::write( + scratch_discovery.join("findings.json"), + queue::render_findings(&empty), + ) + .expect("write findings"); + let with_empty_queue = run(); + + // The SAME registry, with a queue full of unreviewed findings beside it. + let registry = load_registry(); + let populated = synthetic_queue( + ®istry, + &[ + "configuration.remoteUser", + "configuration.workspaceFolder", + "configuration.userEnvProbe", + "configuration.image", + ], + ); + queue::write_findings(&scratch_discovery, &populated.findings).expect("write findings"); + queue::write_campaigns(&scratch_discovery, &populated.campaigns).expect("write campaigns"); + corpus::write(&scratch_discovery, &[]).expect("write corpus"); + let with_full_queue = run(); + + // The queue is real: it loads, it is non-empty, and every finding is in the visible + // untriaged bucket. Without this the equality below would hold because nothing was + // there. + let loaded = DiscoveryData::load(&scratch_discovery).expect("the scratch queue loads"); + assert_eq!(loaded.findings.len(), 4); + assert!( + loaded + .findings + .iter() + .all(|f| f.state == queue::FindingState::Untriaged), + "the queue must hold UNREVIEWED findings — the state SC-018 is about" + ); + assert_eq!( + queue::check(&loaded, &queue::RegistryView::from_registry(®istry)), + Vec::new(), + "the synthetic queue must itself be clean, or this test would be asserting that a \ + BROKEN queue does not reach certify" + ); + + assert_eq!( + with_empty_queue, with_full_queue, + "certification must be byte-identical either way: a discovery finding is a \ + candidate for an assertion, not an assertion, and a stochastic process that could \ + move a release gate would make green non-reproducible" + ); +} + +/// **T126 / FR-041**: a tolerance scaffold is **scoped**, and a blanket or unscoped scope is +/// refused rather than emitted. +/// +/// The rule is asserted at its single definition and at the surface that uses it, because +/// the two failures are different: a rule that accepted a bare channel would let a blanket +/// tolerance be authored, and an emitter that bypassed the rule would produce one even +/// though the rule is correct. +#[test] +fn a_tolerance_scaffold_is_scoped_and_a_blanket_scope_is_refused() { + use deacon_conformance::discovery::promote::{self, PromotionError}; + + // The rule itself. A bare channel tolerates everything on that channel forever, which + // is the global ignore list the registry already refuses at load (V19). + for blanket in [ + "chan-structured-output", + "chan-exit-code", + "chan-structured-output.", + "", + ] { + assert!( + matches!( + promote::reject_blanket_observable_path("fnd-x", blanket), + Err(PromotionError::UnscopedTolerance { .. }) + ), + "{blanket:?} must be refused as a blanket tolerance" + ); + } + promote::reject_blanket_observable_path("fnd-x", "chan-structured-output.configuration") + .expect("a scoped path is accepted"); + + // The emitter. Nothing it produces may be a bare channel, and the waiver it emits must + // carry the two fields that make a tolerance self-invalidating rather than permanent. + let registry = load_registry(); + let data = synthetic_queue(®istry, &["configuration.remoteUser"]); + let finding = data.findings.first().expect("a synthetic finding"); + let document = promote::tolerance_skeleton(finding).expect("a tolerance scaffolds"); + + let path = document["allowedDifference"]["observablePath"] + .as_str() + .expect("the tolerance names an observable path"); + let (channel, rest) = path + .split_once('.') + .unwrap_or_else(|| panic!("`{path}` is a bare channel")); + assert!( + registry.channels.iter().any(|c| c.id == channel), + "the tolerance's channel `{channel}` must be one the registry declares" + ); + assert!( + !rest.trim().is_empty(), + "`{path}` scopes to nothing within its channel" + ); + assert_eq!(path, "chan-structured-output.configuration.remoteUser"); + + // Self-invalidating, not permanent: rationale + expiry are required, and both are + // sentinels the loader rejects until a human writes them. + for field in ["rationale", "expires"] { + assert_eq!( + document["waiver"][field], + serde_json::json!(promote::UNREVIEWED), + "a tolerance without a decided `{field}` is an unbacked silence" + ); + } + assert_eq!( + document["allowedDifference"]["waiverId"], document["waiver"]["id"], + "the allowed difference must reference the waiver that backs it; an unbacked \ + tolerance is exactly what V19 refuses" + ); + // The context, too: an empty context reads as "everywhere", which is the blanket + // tolerance moved into a different field. + let context = document["allowedDifference"]["context"] + .as_array() + .expect("a context array"); + assert!(!context.is_empty()); + assert!( + context + .iter() + .all(|c| c == &serde_json::json!(promote::UNREVIEWED)) + ); + + // And a signature with no observable path cannot be tolerated at all, rather than + // being tolerated channel-wide. + let mut pathless = data.findings[0].clone(); + pathless.signature = Signature::derive( + "chan-structured-output", + &Divergence { + kind: DivergenceKind::Value, + path: "", + deacon: None, + reference: None, + }, + ); + assert!(matches!( + promote::tolerance_skeleton(&pathless), + Err(PromotionError::UnscopedTolerance { .. }) + )); +} + +/// **T078 / SC-016**: an injected difference traverses the whole pipeline — generation, +/// comparison, minimization, candidate emission, classification, and review-only promotion +/// — and an injection that never lands **fails loudly** rather than reading as "found +/// nothing". +/// +/// This is the acceptance test for FR-042a, and it drives the REAL machinery end to end: +/// the real constrained generator (US1), the real comparison and signature derivation, the +/// real structural shrinker (US2), the real reviewable-candidate writer, and the real +/// finding state machine (US4). The only synthetic thing is the difference itself, planted +/// through the sealed `EvidenceSource` boundary (research D7) — where injecting into an +/// observer's *return* value does not compile, so the proof cannot assert on data it wrote +/// downstream of the part under test. +/// +/// Hermetic: no oracle, no Docker, no network. The counterpart is deacon's own unperturbed +/// run, which is also what makes the baseline provably empty and every surfaced difference +/// attributable to the injection. +#[tokio::test] +async fn an_injected_difference_traverses_the_whole_pipeline_and_a_dud_fails_loudly() { + use parity_harness::discovery::pipeline_proof::{self, ProofRequest, Stage, TraversalVerdict}; + use parity_harness::inject::RegressionHarness; + + let scratch = tempfile::tempdir().expect("tempdir"); + let request = ProofRequest { + deacon_binary: PathBuf::from(env!("CARGO_BIN_EXE_deacon")), + registry_dir: deacon_conformance::default_registry_dir(), + report_root: scratch.path().to_path_buf(), + seed_hex: "0x02542a1f".to_string(), + seed: 0x0254_2a1f, + profile: SYNTHETIC_PROFILE.to_string(), + bound: std::time::Duration::from_secs(60), + // Deliberately small: this guard asserts that reduction RUNS and that the signature + // survives it, not how far it gets. A large budget would spend fast-lane minutes + // re-establishing what the shrinker's own unit tests already cover. + shrink_budget: 6, + max_draws: 64, + }; + + // The FR-070 capability. Injection fails closed without it, so a proof that forgot to + // declare it would report every injection inapplicable rather than silently passing. + let capability = RegressionHarness::declare(); + let _ = &capability; + + let ctx = pipeline_proof::establish(&request) + .await + .expect("a clean baseline must be establishable; failing to find one proves nothing"); + + // --- the difference traverses --------------------------------------- + let injections = pipeline_proof::proof_injections().expect("the proof's injections load"); + assert!(!injections.is_empty()); + for record in &injections { + let traversal = pipeline_proof::traverse(&ctx, record) + .await + .unwrap_or_else(|e| panic!("traversing `{}` failed: {e}", record.id)); + assert_eq!( + traversal.verdict, + TraversalVerdict::Traversed, + "`{}` did not traverse: {traversal:?}", + record.id + ); + assert_eq!( + traversal + .stages + .iter() + .map(|s| s.stage) + .collect::>(), + Stage::all().to_vec(), + "every stage FR-042a names must be reached, in order: {traversal:?}" + ); + assert!(traversal.applied >= 1, "the perturbation must have LANDED"); + assert!( + traversal.signature.is_some() && traversal.finding.is_some(), + "a difference that traversed must carry the signature it derived and the finding \ + a campaign would have admitted" + ); + // The difference surfaced on the channel the record declared — not merely somewhere. + assert_eq!(traversal.channel, record.channel); + } + + // --- and an injection that never lands fails LOUDLY ------------------ + // + // The distinction FR-042a draws, and the one this whole machinery exists to keep: a + // perturbation that was never applied says NOTHING about the pipeline. Reporting it as + // "the pipeline found nothing" would make a mis-authored proof indistinguishable from a + // working one, which is the most comfortable possible way for this feature to be broken. + let dud: deacon_conformance::regression::RegressionFile = serde_json::from_str( + r#"{"records":[{ + "id": "reg-proof-dud", + "channel": "chan-structured-output", + "target": "structured-output-document", + "perturbation": { + "kind": "remove-json-pointer", + "pointer": "/configuration/aKeyNoConfigurationHasEverCarried" + }, + "expectedDetectingCases": ["discovery-proof"] + }]}"#, + ) + .expect("the dud record loads"); + let dud = dud.records.into_iter().next().expect("one dud record"); + + let traversal = pipeline_proof::traverse(&ctx, &dud) + .await + .expect("an inapplicable injection is a VERDICT, not an aborted run"); + match &traversal.verdict { + TraversalVerdict::InjectionInapplicable { cause } => { + assert!( + cause.contains("aKeyNoConfigurationHasEverCarried") || cause.contains("resolve"), + "the diagnosis must name why nothing was perturbed: {cause}" + ); + } + other => panic!( + "an injection that never landed must be reported as inapplicable, never as \ + traversed and never as a pipeline defect: {other:?}" + ), + } + assert_eq!( + traversal.applied, 0, + "nothing was perturbed, and the record says so" + ); + assert_eq!( + traversal.stages.len(), + 1, + "only generation was reached; claiming any later stage would claim the pipeline was \ + exercised by an injection that never entered it" + ); + + // The status rule, from the same function the bin calls: an inapplicable injection + // FAILS the run, and is counted apart from a pipeline defect. + let failing = + pipeline_proof::ProofReport::build("0x02542a1f", ctx.candidate_id(), vec![traversal]); + assert_eq!(failing.exit_status(), 1); + assert_eq!(failing.inapplicable_count, 1); + assert_eq!( + failing.failed_count, 0, + "an inapplicable injection is a PROOF defect, counted separately from a pipeline \ + defect — merging them would lose the distinction FR-042a exists to draw" + ); +} diff --git a/crates/deacon/tests/discovery_metamorphic.rs b/crates/deacon/tests/discovery_metamorphic.rs new file mode 100644 index 00000000..91ee37f9 --- /dev/null +++ b/crates/deacon/tests/discovery_metamorphic.rs @@ -0,0 +1,605 @@ +//! Live metamorphic-relation binary (025-exploratory-parity-discovery, US6). +//! +//! **Selected ONLY by `[profile.discovery]`**, like `discovery_campaign`. Its exclusion +//! from the pull-request lanes is about **stochasticity, not cost**: this tier needs +//! neither the pinned oracle, nor Docker, nor the network (research D12), so it is cheap +//! enough to run anywhere — and FR-055 is absolute regardless. A lane whose result varies +//! run to run cannot be a gate. +//! +//! That cheapness is load-bearing elsewhere, though: it makes this the only complete +//! vertical slice a contributor with no devcontainer CLI installed can develop and test +//! locally, which is why research D12 recommends building it first. +//! +//! ## What these tests are, and are not +//! +//! Each test drives the real `deacon` binary over a real workspace, applies one declared +//! transformation, and checks the relation the registry declares for it. They are **not** +//! unit tests of the transformations — those live beside the transformations in +//! `parity_harness::discovery::metamorphic_run` — and they are not a differential: no +//! oracle is involved, and the two sides being compared are both deacon. +//! +//! ## Lane wiring (T097) +//! +//! Two guards, deliberately at different levels, because they fail in different ways: +//! +//! - [`this_binary_runs_only_under_the_discovery_profile`] is a **runtime** check. It fires +//! when a lane actually selected this binary, which a config-shaped assertion cannot see. +//! - `discovery_hermetic::live_discovery_binaries_are_selected_by_exactly_one_lane` is the +//! **config** check, and it names `discovery_metamorphic` explicitly: selected by +//! `[profile.discovery]`, selected by none of the six pull-request profiles (`default`, +//! `dev-fast`, `full`, `ci`, `mvp-integration`, `parity`). It lives there rather than +//! here precisely so it runs in the fast lane — a guard that only runs in the lane it +//! guards guards nothing. + +use std::path::{Path, PathBuf}; + +use deacon_conformance::discovery::metamorphic::{ + MANDATED_RELATIONS, MetamorphicRelation, RelationEffect, +}; +use parity_harness::discovery::metamorphic_run::{ + MetamorphicCandidate, RelationOutcome, Sabotage, evaluate, evaluate_catalogue, +}; + +/// The environment variable nextest sets to the profile it selected the run under. +const NEXTEST_PROFILE: &str = "NEXTEST_PROFILE"; + +/// The one profile permitted to select this binary. +const DISCOVERY_PROFILE: &str = "discovery"; + +/// If this binary is running under a nextest profile at all, that profile must be +/// `discovery`. +/// +/// See `discovery_campaign`'s counterpart for why the invariant is asserted from inside +/// the binary as well as against the config: this catches a lane that actually selected +/// the binary, not merely a config that reads as though it would not. +#[test] +fn this_binary_runs_only_under_the_discovery_profile() { + let Some(profile) = std::env::var_os(NEXTEST_PROFILE) else { + // Run outside nextest: no profile selected it, so there is no selection to check. + return; + }; + let profile = profile.to_string_lossy().into_owned(); + assert_eq!( + profile, DISCOVERY_PROFILE, + "the live metamorphic binary was selected by [profile.{profile}]. Only \ + [profile.{DISCOVERY_PROFILE}] may select it — the exclusion is about \ + stochasticity, not cost, so \"this tier is cheap\" is never a reason to admit it \ + to a pull-request lane (FR-055/FR-057). Fix the profile's `default-filter` in \ + .config/nextest.toml." + ); +} + +// --------------------------------------------------------------------------- +// Harness +// --------------------------------------------------------------------------- + +/// The deacon binary under test. Only the test crate can expand this; the harness never +/// guesses a `target/…/deacon` path (the stale-artifact defect of 023 T115). +fn deacon() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_deacon")) +} + +/// The committed relation catalogue. +/// +/// Loaded from the registry rather than restated here, so these tests exercise the records +/// `validate` polices. A catalogue that stopped loading is a failure, never an empty run: +/// zero relations evaluated and zero relations violated look identical from the outside. +fn catalogue() -> Vec { + let path = deacon_conformance::default_registry_dir().join("metamorphic.json"); + let records = deacon_conformance::discovery::metamorphic::load_metamorphic(&path) + .unwrap_or_else(|e| panic!("the committed catalogue at {path:?} must load: {e}")); + assert!( + !records.is_empty(), + "the committed catalogue at {path:?} is empty — an empty catalogue makes every \ + relation test pass vacuously" + ); + records +} + +/// The catalogue record for `id`. +fn relation(id: &str) -> MetamorphicRelation { + catalogue() + .into_iter() + .find(|r| r.id == id) + .unwrap_or_else(|| panic!("`{id}` is not declared in the committed catalogue")) +} + +/// Evaluate one relation honestly and return its outcome. +async fn run(id: &str, root: &Path) -> RelationOutcome { + let record = relation(id); + evaluate(&deacon(), root, &record, Sabotage::None) + .await + .unwrap_or_else(|e| panic!("`{id}` must be evaluable: {e}")) +} + +/// Assert an invariance relation held, rendering every residual difference on failure. +fn assert_holds(outcome: &RelationOutcome) { + assert!( + outcome.holds, + "relation `{}` was violated.\n transformation: {}\n effect: {}\n residual \ + differences ({}):\n{}\n original workspace: {}\n transformed workspace: {}", + outcome.relation, + outcome.transformation, + outcome.effect.as_str(), + outcome.residual.len(), + render_residual(outcome), + outcome.original.workspace, + outcome.transformed.workspace, + ); +} + +fn render_residual(outcome: &RelationOutcome) -> String { + if outcome.residual.is_empty() { + return " (none)".to_string(); + } + outcome + .residual + .iter() + .map(|r| { + format!( + " {} [{}]\n original = {}\n transformed = {}", + r.path, + r.kind, + r.original + .as_ref() + .map(ToString::to_string) + .unwrap_or_else(|| "".to_string()), + r.transformed + .as_ref() + .map(ToString::to_string) + .unwrap_or_else(|| "".to_string()), + ) + }) + .collect::>() + .join("\n") +} + +fn tempdir() -> tempfile::TempDir { + tempfile::tempdir().expect("a temp workspace root") +} + +// --------------------------------------------------------------------------- +// T084 — invariance: formatting, JSONC comments/trailing commas, key order (FR-044) +// --------------------------------------------------------------------------- + +/// Reindenting the configuration and rewrapping its whitespace changes no token, so the +/// resolved configuration must be identical. A result that changes here is reading layout. +#[tokio::test] +async fn formatting_is_invariant() { + let dir = tempdir(); + let outcome = run("mrl-formatting-invariance", dir.path()).await; + assert_holds(&outcome); + // The transformation must actually have happened: an identity "transformation" would + // make the relation hold for a reason that says nothing about deacon. + assert_ne!( + outcome.original.input, outcome.transformed.input, + "the reindented document must differ from the original by more than nothing" + ); + assert_eq!( + outcome.original.normalized, outcome.transformed.normalized, + "an invariance relation that holds compares equal outright" + ); +} + +/// JSONC comments and trailing commas are well-formed in the parse dialect and denote no +/// member, so inserting them can change neither whether the document resolves nor what it +/// resolves to. This relation fails in both directions, and both matter. +#[tokio::test] +async fn jsonc_comments_and_trailing_commas_are_invariant() { + let dir = tempdir(); + let outcome = run("mrl-comment-invariance", dir.path()).await; + // The `exitCode` channel carries the first direction: a rejected document shows up + // here, not as a mysterious structural difference. + assert_eq!( + outcome.transformed.normalized.get("exitCode"), + outcome.original.normalized.get("exitCode"), + "a commented document must resolve exactly as the uncommented one does; residual: \n{}", + render_residual(&outcome) + ); + assert_holds(&outcome); + assert!( + outcome + .transformed + .input + .files() + .values() + .any(|f| f.contains("// injected")), + "the transformed fixture must genuinely carry comments" + ); +} + +/// Permuting the members of an unordered map cannot change the value the document denotes, +/// so the resolved configuration must be identical. +#[tokio::test] +async fn key_order_within_unordered_maps_is_invariant() { + let dir = tempdir(); + let outcome = run("mrl-key-order-invariance", dir.path()).await; + assert_holds(&outcome); + assert_ne!( + outcome.original.input, outcome.transformed.input, + "the member order must actually have been permuted" + ); +} + +// --------------------------------------------------------------------------- +// T085 — path relocation compares modulo the declared tokenization (FR-046) +// --------------------------------------------------------------------------- + +/// Relocating the identical workspace to a different absolute path must leave the result +/// equal **modulo the declared path tokenization**, and any residual the tokenization does +/// not account for is reported rather than tolerated. +/// +/// That residual is the interesting output, and it needs careful triage: a leaked absolute +/// path the tokenizer missed is as likely to be a `normalizer-defect` as a +/// `deacon-regression`, and misfiling it as the latter sends someone to fix code that is +/// correct. The failure message therefore names both possibilities rather than asserting +/// one. +#[tokio::test] +async fn path_relocation_is_invariant_modulo_the_declared_tokenization() { + let dir = tempdir(); + let outcome = run("mrl-path-relocation", dir.path()).await; + + // The two sides really were at different absolute paths, with the same basename — so a + // difference is relocation and not a renamed directory. + assert_ne!(outcome.original.workspace, outcome.transformed.workspace); + assert_eq!( + Path::new(&outcome.original.workspace).file_name(), + Path::new(&outcome.transformed.workspace).file_name(), + ); + + // The RAW evidence still carries the two distinct host paths; only the normalized + // evidence is tokenized. Raw and normalized are held separately for exactly this + // reason — conflating them would hide what the tokenization did. + assert_ne!( + outcome.original.raw, outcome.transformed.raw, + "the raw evidence must still carry the two distinct absolute paths, or the \ + relocation did not happen and the relation holds vacuously" + ); + + assert!( + outcome.holds, + "relation `{}` left a residual the declared path tokenization did not account \ + for.\n{}\nTriage carefully: a leaked absolute path is as likely to be a \ + normalizer-defect (the token map missed a form deacon emits) as a \ + deacon-regression (the result genuinely depends on where the workspace sits). \ + Filing it as the latter sends someone to fix code that is correct.", + outcome.relation, + render_residual(&outcome), + ); + + // And the tokenization actually did something: the tokenized host path is gone from + // both normalized documents. + for side in [&outcome.original, &outcome.transformed] { + let rendered = side.normalized.to_string(); + assert!( + !rendered.contains(&side.workspace), + "the normalized evidence still contains the raw workspace path {}; the \ + tokenization did not run, so this relation would compare two documents that \ + cannot be equal", + side.workspace + ); + assert!( + rendered.contains(""), + "the normalized evidence carries no `` token, so the fixture puts \ + no host path into the result and the relation is vacuous" + ); + } +} + +// --------------------------------------------------------------------------- +// T086 — lifecycle equivalence across the permitted forms +// --------------------------------------------------------------------------- + +/// The bare form and the single-entry named-object form denote the same command, so both +/// must resolve and switching between them must change nothing outside the lifecycle +/// property itself. +#[tokio::test] +async fn equivalent_lifecycle_command_forms_resolve_equivalently() { + let dir = tempdir(); + let outcome = run("mrl-lifecycle-equivalence", dir.path()).await; + + // Both permitted forms are accepted — the `chan-exit-code` half of the assertion. + for side in [&outcome.original, &outcome.transformed] { + assert_eq!( + side.normalized.get("exitCode"), + Some(&serde_json::json!(0)), + "every permitted lifecycle command form must resolve; workspace {}", + side.workspace + ); + } + + assert_holds(&outcome); + + // The difference the relation accounts for is the transformed site and nothing else. + // Asserting it is present matters as much as asserting the residual is empty: an + // accounting that absorbed nothing would mean the rewrite never reached the output, + // and the relation would hold because nothing happened. + assert!( + outcome.accounted.iter().any(|r| r + .path + .starts_with("structuredOutput.configuration.postCreateCommand")), + "the rewritten lifecycle property must show up in the output; accounted: {:?}", + outcome.accounted + ); + assert_eq!( + outcome.transformed.raw["structuredOutput"]["configuration"]["postCreateCommand"], + serde_json::json!({ "solo": "echo hi" }), + "the object form must be echoed as authored" + ); +} + +// --------------------------------------------------------------------------- +// T087 — sensitivity: permuting a declaration-ordered collection MUST change the result +// --------------------------------------------------------------------------- + +/// Reversing a declaration-ordered collection describes a different configuration, so the +/// result **must** change. A failure to change is a finding, not a pass. +/// +/// This is the assertion the differential structurally cannot make. If deacon and the +/// reference both canonicalized this list, the two sides would agree and the differential +/// would be clean — the defect invisible precisely because both implementations share it. +#[tokio::test] +async fn permuting_a_declaration_ordered_collection_must_change_the_result() { + let dir = tempdir(); + let outcome = run("mrl-declaration-order-sensitivity", dir.path()).await; + assert_eq!(outcome.effect, RelationEffect::Sensitivity); + + assert!( + outcome.holds, + "relation `{}` was violated: reversing a declaration-ordered collection left the \ + resolved configuration unchanged. The cited clause says the order of this array \ + matters, so an unchanged result means the order was discarded — a defect the \ + differential cannot see, because a reference that discards it too would agree.\n\ + original = {}\n transformed = {}", + outcome.relation, outcome.original.normalized, outcome.transformed.normalized, + ); + + // The change is where the clause says it is, and it is a reordering rather than a loss. + let changed = outcome + .residual + .iter() + .chain(outcome.accounted.iter()) + .find(|r| r.path.contains("dockerComposeFile")) + .unwrap_or_else(|| { + panic!( + "the result changed, but not at `dockerComposeFile` — the relation would be \ + satisfied by an unrelated difference. residual: {:?} accounted: {:?}", + outcome.residual, outcome.accounted + ) + }); + assert!( + changed.original.is_some() && changed.transformed.is_some(), + "the overlay list must be present on both sides, reordered rather than dropped: {changed:?}" + ); +} + +// --------------------------------------------------------------------------- +// T090 — zero relations are inert (SC-011) +// --------------------------------------------------------------------------- + +/// Deliberately breaking each declared relation causes **exactly that relation** to fail +/// and be named. +/// +/// The break is applied to the **input**, never to an observation. Perturbing what the +/// comparison returns would make a comparison that ignores its arguments look alive — the +/// same defect 024's regression harness forbids by sealing its injection boundary. The two +/// breaks used here are the two ways a relation can genuinely be violated: an invariance +/// relation is given a genuinely different input, and a sensitivity relation has its +/// transformation taken away. +/// +/// A relation that still holds under its own break is **inert**: it reports a pass that no +/// defect could turn into a failure, which is worse than having no relation at all, because +/// the pass is trusted. +#[tokio::test] +async fn breaking_each_relation_fails_exactly_that_relation() { + let dir = tempdir(); + let records = catalogue(); + + // Every mandated family is present, so "each relation" is the declared floor and not + // whatever happens to be in the file. + for family in MANDATED_RELATIONS { + assert!( + records.iter().any(|r| &r.id.as_str() == family), + "mandated relation `{family}` is absent from the catalogue, so this test would \ + silently not cover it" + ); + } + + for record in &records { + let root = dir.path().join(format!("sabotage-{}", record.id)); + + // Sanity: the relation holds when it is not broken. Without this the next + // assertion would also pass for a relation that fails unconditionally. + let clean = evaluate(&deacon(), &root.join("clean"), record, Sabotage::None) + .await + .unwrap_or_else(|e| panic!("`{}` must be evaluable: {e}", record.id)); + assert!( + clean.holds, + "`{}` does not hold even unbroken, so breaking it proves nothing: {}", + record.id, + render_residual(&clean) + ); + + let broken = evaluate(&deacon(), &root.join("broken"), record, Sabotage::Break) + .await + .unwrap_or_else(|e| panic!("`{}` must be evaluable when broken: {e}", record.id)); + assert!( + !broken.holds, + "`{}` still holds after being deliberately broken — it is INERT (SC-011). An \ + inert relation reports a pass no defect could turn into a failure, which is \ + worse than no relation at all, because the pass is trusted. Effect: {}.", + record.id, + record.effect.as_str(), + ); + + // The failure names the relation, and its candidate names it too. + let candidate = broken + .candidate() + .unwrap_or_else(|| panic!("`{}` failed but produced no candidate", record.id)); + assert_eq!(candidate.relation, record.id); + } +} + +/// Breaking one relation must not make the others fail: a break that turned the whole tier +/// red would prove the machinery reacts to *something*, not that each relation observes its +/// own transformation. +#[tokio::test] +async fn a_break_is_confined_to_the_relation_it_targets() { + let dir = tempdir(); + let records = catalogue(); + let target = &records[0]; + + let broken = evaluate( + &deacon(), + &dir.path().join("target"), + target, + Sabotage::Break, + ) + .await + .expect("evaluable"); + assert!(!broken.holds, "the targeted relation must fail"); + + for other in records.iter().skip(1) { + let outcome = evaluate( + &deacon(), + &dir.path().join(format!("other-{}", other.id)), + other, + Sabotage::None, + ) + .await + .unwrap_or_else(|e| panic!("`{}` must be evaluable: {e}", other.id)); + assert!( + outcome.holds, + "`{}` failed while `{}` was the one being broken: {}", + other.id, + target.id, + render_residual(&outcome) + ); + } +} + +// --------------------------------------------------------------------------- +// T127 — the failure candidate (FR-047) +// --------------------------------------------------------------------------- + +/// A metamorphic failure produces a candidate naming the relation, the transformation +/// applied, **both** inputs, and **both** normalized outputs. +/// +/// Every one of the four is required, and the reason is the reviewer: a candidate that +/// names the relation but not the transformation cannot be reproduced, and one that carries +/// a verdict but not both sides' evidence is a bug report with the evidence left behind. +#[tokio::test] +async fn a_metamorphic_failure_emits_a_reviewable_candidate() { + let dir = tempdir(); + let record = relation("mrl-formatting-invariance"); + let broken = evaluate(&deacon(), dir.path(), &record, Sabotage::Break) + .await + .expect("evaluable"); + assert!(!broken.holds, "the broken relation must fail"); + + let candidate: MetamorphicCandidate = broken.candidate().expect("a failure yields a candidate"); + + // 1. The relation. + assert_eq!(candidate.relation, "mrl-formatting-invariance"); + // 2. The transformation, worded as the catalogue words it — what a reviewer reproduces. + assert_eq!(candidate.transformation, record.transformation); + assert!(!candidate.transformation.trim().is_empty()); + // 3. Both inputs, whole and distinct. + assert!( + candidate + .original_input + .files() + .contains_key(".devcontainer/devcontainer.json"), + "the original input must be the whole workspace tree" + ); + assert!( + candidate + .transformed_input + .files() + .contains_key(".devcontainer/devcontainer.json") + ); + assert_ne!(candidate.original_input, candidate.transformed_input); + // 4. Both normalized outputs, and they differ — which is what the candidate claims. + assert_ne!( + candidate.original_normalized, + candidate.transformed_normalized + ); + + // The residual is the reviewable difference, and the derived signature is computed by + // the same function the differential uses — so a metamorphic finding and a differential + // finding at the same path deduplicate against each other. + assert!( + !candidate.residual.is_empty(), + "an invariance failure must name at least one residual difference" + ); + let signatures = candidate.signatures("chan-structured-output"); + assert_eq!(signatures.len(), candidate.residual.len()); + for signature in &signatures { + assert_eq!( + signature.derived_id(), + signature.id, + "the signature id must recompute from its own substance" + ); + assert!(signature.finding_id().starts_with("fnd-")); + } + + // A candidate round-trips through strict JSON, so it can be written to the queue and + // read back without losing the evidence. + let raw = serde_json::to_string(&candidate).expect("serializes"); + let back: MetamorphicCandidate = serde_json::from_str(&raw).expect("round-trips"); + assert_eq!(back, candidate); +} + +// --------------------------------------------------------------------------- +// The tier entry point (the seam T096 will call from the campaign driver) +// --------------------------------------------------------------------------- + +/// The whole committed catalogue evaluates in one call, in catalogue order, with no +/// oracle, no Docker, and no network (FR-048, research D12). +/// +/// This is the entry point the campaign driver will invoke, and it is tested here rather +/// than left to that integration: an untested entry point is where an integration breaks, +/// and its two failure modes — a relation with no transformation, and a relation silently +/// skipped — are precisely the ones that would surface as a quiet green tier. +#[tokio::test] +async fn the_whole_catalogue_evaluates_with_no_external_prerequisite() { + let dir = tempdir(); + let records = catalogue(); + let outcomes = evaluate_catalogue(&deacon(), dir.path(), &records, Sabotage::None) + .await + .expect("the committed catalogue must be evaluable end to end"); + + assert_eq!( + outcomes.len(), + records.len(), + "every declared relation must be evaluated — a skipped relation reports nothing, \ + and reporting nothing is indistinguishable from holding" + ); + for (outcome, record) in outcomes.iter().zip(records.iter()) { + assert_eq!(outcome.relation, record.id, "catalogue order is preserved"); + assert_holds(outcome); + } + + // Both effects are exercised, so a tier that is green is green on both kinds of + // assertion rather than on six invariances and a relation nobody ran. + assert!( + outcomes + .iter() + .any(|o| o.effect == RelationEffect::Invariance) + ); + assert!( + outcomes + .iter() + .any(|o| o.effect == RelationEffect::Sensitivity) + ); +} + +/// A relation that holds emits no candidate. A tier that produced a candidate per +/// evaluation would drown the queue in non-findings and make the count meaningless. +#[tokio::test] +async fn a_relation_that_holds_emits_no_candidate() { + let dir = tempdir(); + let outcome = run("mrl-key-order-invariance", dir.path()).await; + assert!(outcome.holds); + assert!(outcome.candidate().is_none()); +} diff --git a/crates/deacon/tests/parity_registry_check.rs b/crates/deacon/tests/parity_registry_check.rs index 7df2980e..d9981e50 100644 --- a/crates/deacon/tests/parity_registry_check.rs +++ b/crates/deacon/tests/parity_registry_check.rs @@ -18,10 +18,20 @@ //! plumbing) (FR-023). use parity_harness::registry::{ - self, META_TEST_BINARIES, ParityRegistry, filter_selects, parse_nextest_profiles, + self, DISCOVERY_PROFILE, DiscoveryRole, GUARD_REQUIRED_PROFILES, META_TEST_BINARIES, + PULL_REQUEST_PROFILES, ParityRegistry, filter_selects, parse_nextest_profiles, }; use parity_harness::workspace_root; +/// Parse `.config/nextest.toml`'s `[profile.*]` `default-filter` expressions from the +/// real file. Shared by the parity and discovery cross-checks so both read one source. +fn real_nextest_profiles() -> registry::NextestProfiles { + let toml_path = workspace_root().join(".config/nextest.toml"); + let toml_text = + std::fs::read_to_string(&toml_path).unwrap_or_else(|e| panic!("read {toml_path:?}: {e}")); + parse_nextest_profiles(&toml_text).unwrap_or_else(|e| panic!("parse nextest.toml: {e}")) +} + /// 1. Registry ↔ test-file bidirectional match. #[test] fn registry_matches_test_files_both_directions() { @@ -56,11 +66,7 @@ fn registry_matches_test_files_both_directions() { #[test] fn nextest_parity_profile_selects_exactly_live_binaries() { let reg = ParityRegistry::load().unwrap_or_else(|e| panic!("registry.json: {e}")); - let toml_path = workspace_root().join(".config/nextest.toml"); - let toml_text = - std::fs::read_to_string(&toml_path).unwrap_or_else(|e| panic!("read {toml_path:?}: {e}")); - let profiles = - parse_nextest_profiles(&toml_text).unwrap_or_else(|e| panic!("parse nextest.toml: {e}")); + let profiles = real_nextest_profiles(); // [profile.parity] must be declared. assert!( @@ -498,6 +504,271 @@ fn no_coverage_or_regression_command_reaches_the_shipped_cli() { ); } +/// **T054** (025 US3, FR-057): the discovery lane's profile selection. +/// +/// Every **live** discovery binary is selected by `[profile.discovery]` and by no +/// pull-request profile; every hermetic **guard** is the exact opposite — selected by the +/// fast lanes and *not* captured by the discovery allow-list. +/// +/// The role split is the substance of this test, not a technicality. FR-057 says "every +/// discovery program is selected by a discovery lane and by no pull-request lane", and +/// read without roles that sentence would exile `discovery_hermetic` and `discovery_cli` +/// from the fast lane — which is precisely the `discovery_*` glob failure research D9 +/// exists to prevent. The guards are the lane's *machinery*, not campaigns: they are what +/// makes the campaigns' exclusion checkable, so a lane that stopped running them would +/// remove the only thing watching it. +/// +/// Every verdict is reached by **evaluating** the filter expression. The profiles name +/// the discovery binaries in order to *exclude* them, and a token match would read +/// `… & not (binary(=discovery_campaign))` as a selection and fail a correct file. +#[test] +fn the_discovery_lane_selects_the_campaigns_and_no_pull_request_lane_does() { + let reg = ParityRegistry::load().unwrap_or_else(|e| panic!("registry.json: {e}")); + let profiles = real_nextest_profiles(); + + // Positive control: a registry that lost its discovery entries would make every + // assertion below vacuous, and the "nothing to check" state is indistinguishable + // from "everything checks out" in a passing run. + let live: Vec = reg + .discovery_of_role(DiscoveryRole::Live) + .into_iter() + .map(|b| b.name.clone()) + .collect(); + let guards: Vec = reg + .discovery_of_role(DiscoveryRole::Guard) + .into_iter() + .map(|b| b.name.clone()) + .collect(); + assert!( + !live.is_empty() && !guards.is_empty(), + "registry.json must enumerate both discovery roles; found live={live:?} guards={guards:?}" + ); + + let discovery_filter = profiles + .default_filters + .get(DISCOVERY_PROFILE) + .unwrap_or_else(|| { + panic!(".config/nextest.toml must declare [profile.{DISCOVERY_PROFILE}]") + }) + .as_deref() + .unwrap_or_else(|| { + panic!( + "[profile.{DISCOVERY_PROFILE}] must declare a default-filter; without one it \ + selects every binary in the workspace" + ) + }); + + for name in &live { + assert!( + filter_selects(discovery_filter, name) + .unwrap_or_else(|e| panic!("[profile.{DISCOVERY_PROFILE}] filter: {e}")), + "[profile.{DISCOVERY_PROFILE}] must select live discovery binary `{name}` — it \ + is the only sanctioned entry point, so nothing else would ever run it" + ); + } + for name in &guards { + assert!( + !filter_selects(discovery_filter, name) + .unwrap_or_else(|e| panic!("[profile.{DISCOVERY_PROFILE}] filter: {e}")), + "[profile.{DISCOVERY_PROFILE}] captures the hermetic guard `{name}`. Its \ + default-filter must stay an explicit `binary(=…)` allow-list — a \ + `discovery_*` glob would swallow the guards and silently remove them from \ + the fast lane (research D9)" + ); + } + + for profile in PULL_REQUEST_PROFILES { + let expr = profiles + .default_filters + .get(*profile) + .unwrap_or_else(|| panic!("[profile.{profile}] must exist")) + .as_deref() + .unwrap_or_else(|| { + panic!( + "[profile.{profile}] has no default-filter, so it selects every binary \ + including the live discovery campaigns" + ) + }); + for name in &live { + assert!( + !filter_selects(expr, name) + .unwrap_or_else(|e| panic!("[profile.{profile}] filter: {e}")), + "[profile.{profile}] selects live discovery binary `{name}` — discovery \ + gates nothing, so a green pull-request run must never imply a campaign \ + ran (FR-055/FR-057)" + ); + } + if GUARD_REQUIRED_PROFILES.contains(profile) { + for name in &guards { + assert!( + filter_selects(expr, name) + .unwrap_or_else(|e| panic!("[profile.{profile}] filter: {e}")), + "[profile.{profile}] does not select the hermetic guard `{name}` — a \ + guard that does not run in the fast lane is a guard nobody notices \ + going stale" + ); + } + } + } +} + +/// **T056** (025 US3): registry ↔ `tests/*.rs` ↔ `.config/nextest.toml` agreement for the +/// discovery lane, the same three-place rule the parity lane already lives under. +/// +/// A half-landed discovery binary is the failure this catches: a source file with no +/// registry entry has no declared lane, so nothing checks which profiles select it; a +/// registry entry with no source file makes `binary(=…)` name a binary that does not +/// exist, which nextest treats as a hard config error and which therefore breaks *every* +/// lane in the workspace rather than only this one. +/// +/// The two source directories are load-bearing: `discovery_cli` drives the +/// `deacon-conformance` binary and so must live in that crate's test tree, while the +/// campaigns and the repository-wiring guard live in `crates/deacon/tests`. A check that +/// assumed one directory would silently stop covering whichever binary moved, so the +/// registry records `tests_dir` per binary and the file→registry sweep scans both. +#[test] +fn the_discovery_lane_is_wired_in_registry_tests_and_nextest() { + let reg = ParityRegistry::load().unwrap_or_else(|e| panic!("registry.json: {e}")); + let root = workspace_root(); + + // 1 + 2. Registry ↔ source files, both directions. + let problems = registry::check_discovery_files(®, &root); + assert!( + problems.is_empty(), + "registry ↔ discovery tests/*.rs mismatch:\n{}", + problems.join("\n") + ); + + // Each entry's declared directory must be the one the file is actually in — an entry + // pointing at the wrong crate passes a naive existence check only when a file of the + // same name happens to exist in both. + for binary in ®.discovery_binaries { + let declared = root + .join(&binary.tests_dir) + .join(format!("{}.rs", binary.name)); + assert!( + declared.is_file(), + "discovery binary `{}` declares tests_dir `{}`, but {} does not exist", + binary.name, + binary.tests_dir, + declared.display() + ); + } + + // 3. Registry ↔ `.config/nextest.toml`. + let problems = registry::check_discovery_profiles(®, &real_nextest_profiles()); + assert!( + problems.is_empty(), + "discovery lane nextest wiring problems:\n{}", + problems.join("\n") + ); + + // The two lanes must stay disjoint in BOTH directions. `check_nextest_profiles` + // already proves no non-parity profile selects a live parity binary (so + // `[profile.discovery]` cannot pick one up); this proves the converse, which nothing + // else states: the parity lane must not acquire a campaign. The exclusion is about + // the lanes answering different questions on different budgets, so "the metamorphic + // tier is cheap" is never a reason to relax it. + let profiles = real_nextest_profiles(); + let parity_filter = profiles + .default_filters + .get("parity") + .and_then(|f| f.as_deref()) + .expect("[profile.parity] must declare a default-filter"); + for binary in reg.discovery_of_role(DiscoveryRole::Live) { + assert!( + !filter_selects(parity_filter, &binary.name).expect("evaluate the parity filter"), + "[profile.parity] selects the discovery campaign `{}`; a campaign would exceed \ + the parity lane's budget and would make a certification lane stochastic", + binary.name + ); + } +} + +/// **T057** (025 US3, FR-059): this feature adds NO subcommand to the shipped `deacon` +/// CLI, at any depth of the command tree. +/// +/// The `discovery` command group is a `deacon-conformance` bin surface and the campaign / +/// proof programs are `parity-harness` bins: contributor tooling about how deacon is +/// *tested*, not consumer functionality the containers.dev spec describes. A +/// conformance-tracking command that ships is a scope violation users can then depend on, +/// which makes it expensive to withdraw. +/// +/// Walks one level deeper than the top-level column for the same reason +/// `no_coverage_or_regression_command_reaches_the_shipped_cli` does: `discovery` has +/// subcommands (`check`/`report`/`triage`/`split`/`scaffold`), and the cheapest way to +/// leak one is to hang it off an existing consumer command rather than add a new +/// top-level entry. +#[test] +fn no_discovery_command_reaches_the_shipped_cli() { + /// The 025 command surfaces plus the nouns they would most plausibly leak as. Every + /// entry is a word that has no business being a consumer subcommand — deliberately + /// NOT `corpus` or `campaign`-adjacent English that a future consumer feature might + /// legitimately want, because a guard that forbids plausible names gets deleted. + const DEV_ONLY: &[&str] = &[ + "discovery", + "discovery-campaign", + "discovery-proof", + "findings", + "triage", + "metamorphic", + "mutate", + "shrink", + ]; + + fn subcommands_of(args: &[&str]) -> Vec { + let output = std::process::Command::new(env!("CARGO_BIN_EXE_deacon")) + .args(args) + .arg("--help") + .output() + .unwrap_or_else(|e| panic!("`deacon {} --help` runs: {e}", args.join(" "))); + assert!( + output.status.success(), + "`deacon {} --help` must succeed", + args.join(" ") + ); + String::from_utf8_lossy(&output.stdout) + .lines() + .skip_while(|l| !l.trim_start().starts_with("Commands:")) + .skip(1) + .take_while(|l| !l.trim().is_empty()) + .filter_map(|l| l.split_whitespace().next()) + .map(str::to_string) + .collect() + } + + let top = subcommands_of(&[]); + // Positive control: a parse that silently found nothing would pass vacuously. + assert!( + top.iter().any(|c| c == "up"), + "expected to parse the real subcommand list from `deacon --help`; got {top:?}" + ); + + let mut leaked: Vec = Vec::new(); + for dev_only in DEV_ONLY { + if top.iter().any(|c| c == dev_only) { + leaked.push((*dev_only).to_string()); + } + } + for parent in &top { + if parent == "help" { + continue; + } + for nested in subcommands_of(&[parent.as_str()]) { + if DEV_ONLY.contains(&nested.as_str()) { + leaked.push(format!("{parent} {nested}")); + } + } + } + + assert!( + leaked.is_empty(), + "dev-only discovery command(s) reached the shipped consumer CLI: {leaked:?}. The \ + discovery group belongs to `deacon-conformance` and the campaign/proof bins to \ + `parity-harness` (constitution II, FR-059)." + ); +} + /// Guard: the tests dir this file audits is the real one (fail loud if the anchor /// ever drifts, rather than silently auditing nothing). #[test] @@ -539,6 +810,13 @@ fn no_surface_references_a_removed_binary() { // hermetic retirements belong here alongside the live carriers. "baseline_drift", "baseline_enumeration", + // Retired by 025 US7 (T109). The 33 pinned entries moved into the Rust-owned + // `conformance/discovery/corpus.json`, where the immutable-reference rule (D4) can + // run hermetically on every pull request instead of nowhere. Registered here for + // the same reason the hermetic retirements above are: a doc that still tells a + // reader to run a deleted script is a lie with a long half-life, and this file's + // own baseline enumeration reads the manifest path. + "fetch_realworld_corpus", ]; /// Words that mark a mention as historical rather than current-state. const RETIREMENT_MARKERS: &[&str] = &[ @@ -571,6 +849,9 @@ fn no_surface_references_a_removed_binary() { "fixtures/parity-corpus/registry.json", "Makefile", ".github/workflows/parity.yml", + // The discovery lane's workflow is machine-consumed on the same terms (025 T058): + // a stale binary name in it is executed, not read. + ".github/workflows/discovery.yml", ] { let path = root.join(rel); let text = std::fs::read_to_string(&path) @@ -642,6 +923,10 @@ fn no_surface_globs_a_removed_path() { "fixtures/parity-corpus/errors/*", "fixtures/config/basic/devcontainer", "fixtures/config/with-variables/devcontainer", + // 025 US7 (T109): the Python fetcher is gone; the manifest is + // `conformance/discovery/corpus.json` and the fetch lives in + // `parity_harness::discovery::corpus_fetch`. + "fixtures/parity-corpus/fetch_realworld_corpus.py", ]; let mut problems: Vec = Vec::new(); diff --git a/crates/parity-harness/src/bin/discovery-campaign.rs b/crates/parity-harness/src/bin/discovery-campaign.rs new file mode 100644 index 00000000..fe423d13 --- /dev/null +++ b/crates/parity-harness/src/bin/discovery-campaign.rs @@ -0,0 +1,313 @@ +//! `discovery-campaign` — run one exploratory parity discovery campaign +//! (025-exploratory-parity-discovery, T038; contracts/discovery-cli.md § Live commands). +//! +//! ```text +//! cargo run -p parity-harness --bin discovery-campaign -- \ +//! --seed --tier [--budget-seconds ] [--lane ] [--candidates ] \ +//! [--profile ] [--dry-run] +//! ``` +//! +//! # `--seed` is required, never defaulted +//! +//! A defaulted seed would let a campaign run without its reproducibility input being a +//! conscious choice, and FR-001 depends on the seed being *recorded* rather than inferred. +//! A default would also be the worst kind of silent: every unattended run would share one +//! stream, so the campaign would explore the same neighbourhood forever while reporting +//! volume as if it were coverage. +//! +//! # Exit status reflects whether it ran, never what it found (FR-058) +//! +//! | Status | Meaning | +//! |---|---| +//! | `0` | the campaign ran to completion or to budget exhaustion — **regardless of findings** | +//! | `1` | a prerequisite failed, normalization failed, or the data root was unwritable | +//! | `2` | the invocation itself was malformed (a usage error) | +//! +//! A campaign that finds forty differences exits `0`. Any command whose status depends on +//! its findings becomes a gate the moment someone wires it into CI, and a stochastic gate +//! makes green non-reproducible — which is exactly what would make the discovery lane +//! unsafe to schedule. +//! +//! # What it writes, and what it must never write +//! +//! Writes `conformance/discovery/{findings,campaigns}.json` (atomically) and raw evidence +//! under `target/discovery/`. It **never** writes anything under `conformance/registry/`, +//! `conformance/snapshots/`, or `conformance/obligations/` (FR-036): a stochastic process +//! must not be able to author the record it is tested against. + +use std::process::ExitCode; + +use deacon_conformance::discovery::queue::{ + Budget, CampaignLane, CampaignTier, DEFAULT_ADMISSION_CAP, + DEFAULT_PER_CANDIDATE_SECONDS_CONTAINER, DEFAULT_PER_CANDIDATE_SECONDS_HERMETIC, + DEFAULT_WALL_CLOCK_SECONDS, +}; +use deacon_conformance::discovery::report::render_campaign_json; +use deacon_conformance::{default_discovery_dir, default_registry_dir, workspace_root}; + +use parity_harness::discovery::campaign::{self, CampaignRequest}; +use parity_harness::prereq::deacon_binary; + +/// The default certification profile a campaign records itself under when none is given. +/// +/// Named rather than derived: the profile is a claim about the environment the run +/// happened in, and guessing it from the host would make two machines silently disagree +/// about what a finding is a claim about. +const DEFAULT_PROFILE: &str = "prof-linux-amd64-docker-0870"; + +/// How many candidates a campaign plans to reach when the caller does not say. +/// +/// The denominator of `spaceCoveredFraction`: on exhaustion the run reports the portion of +/// *this plan* it covered rather than presenting a truncated run as complete (FR-005). +const DEFAULT_PLANNED_CANDIDATES: u64 = 200; + +fn main() -> ExitCode { + let args: Vec = std::env::args().skip(1).collect(); + let parsed = match parse(&args) { + Ok(p) => p, + Err(message) => return usage(&message), + }; + + let runtime = match tokio::runtime::Runtime::new() { + Ok(r) => r, + Err(e) => { + eprintln!("error: could not start async runtime: {e}"); + return ExitCode::from(1); + } + }; + + let deacon = match runtime.block_on(deacon_binary()) { + Ok(path) => path, + Err(e) => { + eprintln!("error: {e}"); + return ExitCode::from(1); + } + }; + + let request = CampaignRequest { + seed_hex: parsed.seed_hex.clone(), + seed: parsed.seed, + tier: parsed.tier, + lane: parsed.lane, + profile: parsed.profile.clone(), + budget: Budget { + wall_clock_seconds: parsed.budget_seconds, + per_candidate_seconds: match parsed.tier { + CampaignTier::ContainerDifferential => DEFAULT_PER_CANDIDATE_SECONDS_CONTAINER, + _ => DEFAULT_PER_CANDIDATE_SECONDS_HERMETIC, + }, + shrink_steps_per_finding: 64, + admission_cap: DEFAULT_ADMISSION_CAP, + }, + planned_candidates: parsed.planned_candidates, + registry_dir: default_registry_dir(), + discovery_dir: default_discovery_dir(), + report_root: workspace_root().join("target").join("discovery"), + deacon_binary: deacon, + oracle_override: None, + persist: !parsed.dry_run, + }; + + match runtime.block_on(campaign::run(&request)) { + Ok(run) => { + // stdout: a single JSON campaign outcome. stderr: everything else — so a + // caller can pipe the outcome into `jq` without the diagnostics landing in it. + print!("{}", render_campaign_json(&run.report)); + eprintln!( + "campaign {} ran: {} generated, {} executed, {} discarded, {} finding(s) \ + admitted, {} suppressed, {} already characterized", + run.campaign.id, + run.report.candidates_generated, + run.report.candidates_executed, + run.report.candidates_discarded_unsafe, + run.report.signatures_admitted, + run.report.signatures_suppressed, + run.characterized_observations, + ); + // The corpus tier's per-entry outcomes, named one by one (FR-051/FR-052). The + // aggregate counters cannot carry this: "33 generated, 31 executed" says two + // entries did not run, and an ecological canary whose whole job is to notice + // the ecosystem moving must say WHICH two and whether the reason was an + // unreachable snapshot or content that stopped matching its recorded digest. + if !run.corpus_statuses.is_empty() { + eprintln!("corpus entries ({}):", run.corpus_statuses.len()); + for status in &run.corpus_statuses { + eprintln!(" {}", status.summary()); + } + } + // Only the *generating* tiers can have a generation deficiency. The metamorphic + // and corpus tiers draw nothing and mutate nothing — their inputs are + // hand-authored relations and pinned third-party snapshots — so all eleven + // categories are legitimately zero, and reporting that as a deficiency would + // teach a reader to ignore the one line that matters when a real generator + // regression makes a category stop firing. + let generates = matches!( + parsed.tier, + CampaignTier::ConfigDifferential | CampaignTier::ContainerDifferential + ); + if generates && !run.report.unapplied_categories.is_empty() { + eprintln!( + "generation deficiency: {} mutation categor(y|ies) never applied — {}", + run.report.unapplied_categories.len(), + run.report.unapplied_categories.join(", ") + ); + } + // Exit ZERO whatever was found. This is the whole exit-status contract. + ExitCode::SUCCESS + } + Err(e) => { + eprintln!("error: {e}"); + ExitCode::from(1) + } + } +} + +/// The parsed invocation. +struct Parsed { + seed_hex: String, + seed: u64, + tier: CampaignTier, + lane: CampaignLane, + profile: String, + budget_seconds: u64, + planned_candidates: u64, + dry_run: bool, +} + +fn parse(args: &[String]) -> Result { + let mut seed_hex: Option = None; + let mut tier: Option = None; + let mut lane = CampaignLane::Invoked; + let mut profile = DEFAULT_PROFILE.to_string(); + let mut budget_seconds = DEFAULT_WALL_CLOCK_SECONDS; + let mut planned_candidates = DEFAULT_PLANNED_CANDIDATES; + let mut dry_run = false; + + let mut i = 0; + while i < args.len() { + let flag = args[i].as_str(); + let mut value = || -> Result { + i += 1; + args.get(i) + .cloned() + .ok_or_else(|| format!("{flag} requires a value")) + }; + match flag { + "--seed" => seed_hex = Some(value()?), + "--tier" => { + let raw = value()?; + tier = Some( + CampaignTier::all() + .iter() + .copied() + .find(|t| t.as_str() == raw) + .ok_or_else(|| { + format!( + "unknown tier {raw:?}; expected one of {}", + CampaignTier::all() + .iter() + .map(|t| t.as_str()) + .collect::>() + .join(", ") + ) + })?, + ); + } + "--lane" => { + let raw = value()?; + lane = match raw.as_str() { + "scheduled" => CampaignLane::Scheduled, + "invoked" => CampaignLane::Invoked, + other => { + return Err(format!( + "unknown lane {other:?}; expected `scheduled` or `invoked`" + )); + } + }; + } + "--profile" => profile = value()?, + "--budget-seconds" => { + let raw = value()?; + budget_seconds = raw + .parse() + .map_err(|e| format!("--budget-seconds {raw:?} is not a number: {e}"))?; + } + "--candidates" => { + let raw = value()?; + planned_candidates = raw + .parse() + .map_err(|e| format!("--candidates {raw:?} is not a number: {e}"))?; + } + "--dry-run" => dry_run = true, + other => return Err(format!("unknown argument {other:?}")), + } + i += 1; + } + + // Required, never defaulted (FR-001). + let seed_hex = seed_hex.ok_or_else(|| { + "--seed is required and is never defaulted: a campaign must not run without its \ + reproducibility input being a conscious, recorded choice (FR-001)" + .to_string() + })?; + let seed = parse_seed(&seed_hex)?; + let tier = tier.ok_or_else(|| { + format!( + "--tier is required; expected one of {}", + CampaignTier::all() + .iter() + .map(|t| t.as_str()) + .collect::>() + .join(", ") + ) + })?; + if planned_candidates == 0 { + return Err( + "--candidates must be at least 1: a campaign that plans nothing covers \ + nothing, and reporting a fraction of zero would divide by it" + .to_string(), + ); + } + + Ok(Parsed { + seed_hex, + seed, + tier, + lane, + profile, + budget_seconds, + planned_candidates, + dry_run, + }) +} + +/// Parse a seed, accepting `0x`-prefixed hex or plain hex. +/// +/// Rejected rather than hashed into a `u64` on failure: a seed that silently became some +/// other number would record a value that does not reproduce the run it names, which is +/// worse than refusing the invocation. +fn parse_seed(raw: &str) -> Result { + let trimmed = raw.trim(); + let digits = trimmed.strip_prefix("0x").unwrap_or(trimmed); + u64::from_str_radix(digits, 16).map_err(|e| { + format!( + "--seed {raw:?} is not a 64-bit hex value: {e}. The seed is recorded \ + verbatim and must reproduce the run it names." + ) + }) +} + +fn usage(message: &str) -> ExitCode { + eprintln!("error: {message}"); + eprintln!( + "usage: discovery-campaign --seed --tier <{}> \ + [--lane scheduled|invoked] [--profile ] [--budget-seconds ] \ + [--candidates ] [--dry-run]", + CampaignTier::all() + .iter() + .map(|t| t.as_str()) + .collect::>() + .join("|") + ); + ExitCode::from(2) +} diff --git a/crates/parity-harness/src/bin/discovery-proof.rs b/crates/parity-harness/src/bin/discovery-proof.rs new file mode 100644 index 00000000..96a980ce --- /dev/null +++ b/crates/parity-harness/src/bin/discovery-proof.rs @@ -0,0 +1,298 @@ +//! `discovery-proof` — prove the discovery pipeline can carry an injected difference end +//! to end (025-exploratory-parity-discovery, T082; contracts/discovery-cli.md § Live +//! commands). +//! +//! ```text +//! cargo run -p parity-harness --bin discovery-proof -- \ +//! [--seed ] [--profile ] [--candidates ] [--shrink-budget ] \ +//! [--out ] +//! ``` +//! +//! # The one discovery command whose status depends on an outcome +//! +//! Every other discovery command's exit status reflects **whether it ran**, never **what it +//! found** (FR-058) — a campaign that surfaces forty differences exits `0`, because a +//! stochastic gate makes green non-reproducible. This command is the exception, and it is +//! not really an exception: its status is not finding-dependent. It asserts a property of +//! the **machinery**, so non-zero means the pipeline is broken, which is exactly the thing +//! that should fail a lane. +//! +//! | Status | Meaning | +//! |---|---| +//! | `0` | every injected difference traversed all six stages | +//! | `1` | an injection landed and failed to surface (**pipeline defect**), an injection never landed (**proof defect**), or a prerequisite failed | +//! | `2` | the invocation itself was malformed (a usage error) | +//! +//! An injection that never landed exits `1` as `InjectionInapplicable` **rather than being +//! counted as "found nothing"**. Those are opposite conclusions, and a proof that merged +//! them would be the most comfortable possible way for this feature to be broken. +//! +//! # Prerequisites: deacon, and nothing else +//! +//! No oracle, no Docker, no network. The proof compares deacon against its own unperturbed +//! run with a known difference injected into one side at the sealed evidence-source +//! boundary — see the module docs of +//! [`pipeline_proof`](parity_harness::discovery::pipeline_proof) for why a reference takes +//! no part in whether the machinery propagates a difference, and why the self-comparison is +//! what makes the baseline provably clean. +//! +//! # What it writes +//! +//! `target/discovery/proof.json` plus reviewable candidates under +//! `target/discovery/proof/candidates/`. It writes **nothing** under +//! `conformance/registry/`, `conformance/discovery/`, `conformance/snapshots/`, or +//! `conformance/obligations/` (FR-036): a stochastic process must not be able to author the +//! record it is tested against, and the promotion stage passes by showing that gate holding +//! rather than by walking through it. + +use std::path::PathBuf; +use std::process::ExitCode; +use std::time::Duration; + +use deacon_conformance::{default_registry_dir, workspace_root}; + +use parity_harness::discovery::pipeline_proof::{self, ProofRequest, TraversalVerdict}; +use parity_harness::inject::RegressionHarness; +use parity_harness::prereq::deacon_binary; + +/// The default certification profile the proof records itself under. +/// +/// Named rather than derived from the host, for the same reason `discovery-campaign` names +/// one: the profile is a claim about the environment a run happened in, and guessing it +/// would let two machines silently disagree about what a run is a claim about. +const DEFAULT_PROFILE: &str = "prof-linux-amd64-docker-0870"; + +/// The default seed. +/// +/// Unlike a campaign's, this one **is** defaulted, and the difference is principled. A +/// campaign's seed is its reproducibility input: defaulting it would let every unattended +/// run share one stream and explore the same neighbourhood forever while reporting volume +/// as coverage. The proof does not explore — it needs *any* candidate deacon accepts, and +/// it plants the difference itself. A fixed default therefore makes the proof reproducible +/// by default, which is what a machinery assertion wants; `--seed` still overrides it, and +/// the value used is recorded in the report either way. +const DEFAULT_SEED: &str = "0x02542a1f"; + +/// How many candidates may be drawn while looking for one deacon accepts and that compares +/// clean against itself. +const DEFAULT_MAX_DRAWS: u64 = 64; + +/// Probes minimization may spend per injection. +const DEFAULT_SHRINK_BUDGET: u64 = 24; + +/// The per-invocation bound. `read-configuration` returns in about a second; a minute is +/// generous enough that a bound is never the reason a proof fails, and short enough that a +/// hung deacon does not hang a lane. +const INVOCATION_BOUND: Duration = Duration::from_secs(60); + +fn main() -> ExitCode { + let args: Vec = std::env::args().skip(1).collect(); + let parsed = match parse(&args) { + Ok(p) => p, + Err(message) => return usage(&message), + }; + + let runtime = match tokio::runtime::Runtime::new() { + Ok(r) => r, + Err(e) => { + eprintln!("error: could not start async runtime: {e}"); + return ExitCode::from(1); + } + }; + + let deacon = match runtime.block_on(deacon_binary()) { + Ok(path) => path, + Err(e) => { + eprintln!("error: {e}"); + return ExitCode::from(1); + } + }; + + let report_root = workspace_root().join("target").join("discovery"); + let out = parsed + .out + .clone() + .unwrap_or_else(|| report_root.join("proof.json")); + + let request = ProofRequest { + deacon_binary: deacon, + registry_dir: default_registry_dir(), + report_root, + seed_hex: parsed.seed_hex.clone(), + seed: parsed.seed, + profile: parsed.profile.clone(), + bound: INVOCATION_BOUND, + shrink_budget: parsed.shrink_budget, + max_draws: parsed.max_draws, + }; + + // The FR-070 capability, taken out HERE and nowhere else in this program. Held as a + // value rather than a bare call so the one place a process becomes able to inject is + // visible in its `main`. + let capability = RegressionHarness::declare(); + + let report = match runtime.block_on(pipeline_proof::run(&request, &capability)) { + Ok(report) => report, + Err(e) => { + // A prerequisite failure, a normalization failure, or a baseline that could not + // be established. Distinct from a failed traversal and reported as such: the + // pipeline was not exercised at all, which is not the same as it being broken. + eprintln!("error: {e}"); + return ExitCode::from(1); + } + }; + + let rendered = match report.render() { + Ok(rendered) => rendered, + Err(e) => { + eprintln!("error: {e}"); + return ExitCode::from(1); + } + }; + if let Some(parent) = out.parent() + && let Err(e) = std::fs::create_dir_all(parent) + { + eprintln!("error: could not create {}: {e}", parent.display()); + return ExitCode::from(1); + } + if let Err(e) = std::fs::write(&out, &rendered) { + eprintln!("error: could not write {}: {e}", out.display()); + return ExitCode::from(1); + } + + // stdout: the single JSON document. stderr: everything else, so a caller can pipe the + // report into `jq` without the diagnostics landing in it. + print!("{rendered}"); + + for traversal in &report.injections { + match &traversal.verdict { + TraversalVerdict::Traversed => eprintln!( + "{} [{}] traversed all {} stage(s) as {}", + traversal.injection, + traversal.channel, + traversal.stages.len(), + traversal.signature.as_deref().unwrap_or("") + ), + TraversalVerdict::FailedToSurface { stage, cause } => eprintln!( + "PIPELINE DEFECT: {} [{}] landed on {} artifact(s) and stopped at `{}`: {cause}", + traversal.injection, + traversal.channel, + traversal.applied, + stage.as_str() + ), + TraversalVerdict::InjectionInapplicable { cause } => eprintln!( + "PROOF DEFECT: {} [{}] never landed: {cause}. This says nothing about the \ + pipeline — a perturbation that was not applied is a mis-authored record, \ + not a campaign that found nothing.", + traversal.injection, traversal.channel + ), + } + } + eprintln!( + "proof: {} injection(s) over baseline candidate {} (seed {}); {} failed to surface, \ + {} inapplicable; report at {}", + report.injections.len(), + report.baseline_candidate, + report.seed, + report.failed_count, + report.inapplicable_count, + out.display() + ); + + ExitCode::from(report.exit_status()) +} + +/// The parsed invocation. +struct Parsed { + seed_hex: String, + seed: u64, + profile: String, + max_draws: u64, + shrink_budget: u64, + out: Option, +} + +fn parse(args: &[String]) -> Result { + let mut seed_hex = DEFAULT_SEED.to_string(); + let mut profile = DEFAULT_PROFILE.to_string(); + let mut max_draws = DEFAULT_MAX_DRAWS; + let mut shrink_budget = DEFAULT_SHRINK_BUDGET; + let mut out: Option = None; + + let mut i = 0; + while i < args.len() { + let flag = args[i].as_str(); + let mut value = || -> Result { + i += 1; + args.get(i) + .cloned() + .ok_or_else(|| format!("{flag} requires a value")) + }; + match flag { + "--seed" => seed_hex = value()?, + "--profile" => profile = value()?, + "--candidates" => { + let raw = value()?; + max_draws = raw + .parse() + .map_err(|e| format!("--candidates {raw:?} is not a number: {e}"))?; + } + "--shrink-budget" => { + let raw = value()?; + shrink_budget = raw + .parse() + .map_err(|e| format!("--shrink-budget {raw:?} is not a number: {e}"))?; + } + "--out" => out = Some(PathBuf::from(value()?)), + other => return Err(format!("unknown argument {other:?}")), + } + i += 1; + } + + let seed = parse_seed(&seed_hex)?; + if max_draws == 0 { + return Err( + "--candidates must be at least 1: a proof that may draw nothing can \ + never establish the baseline it needs" + .to_string(), + ); + } + if shrink_budget == 0 { + return Err( + "--shrink-budget must be at least 1: a reduction that may spend no \ + probe never runs, and the minimization stage would report a defect for \ + an invocation that forbade it" + .to_string(), + ); + } + + Ok(Parsed { + seed_hex, + seed, + profile, + max_draws, + shrink_budget, + out, + }) +} + +/// Parse a seed, accepting `0x`-prefixed hex or plain hex. +/// +/// Rejected rather than hashed into a `u64` on failure, for the same reason +/// `discovery-campaign` rejects one: a seed that silently became some other number would +/// record a value that does not reproduce the run it names. +fn parse_seed(raw: &str) -> Result { + let trimmed = raw.trim(); + let digits = trimmed.strip_prefix("0x").unwrap_or(trimmed); + u64::from_str_radix(digits, 16) + .map_err(|e| format!("--seed {raw:?} is not a 64-bit hex value: {e}")) +} + +fn usage(message: &str) -> ExitCode { + eprintln!("error: {message}"); + eprintln!( + "usage: discovery-proof [--seed ] [--profile ] [--candidates ] \ + [--shrink-budget ] [--out ]" + ); + ExitCode::from(2) +} diff --git a/crates/parity-harness/src/discovery/campaign.rs b/crates/parity-harness/src/discovery/campaign.rs new file mode 100644 index 00000000..45782ea4 --- /dev/null +++ b/crates/parity-harness/src/discovery/campaign.rs @@ -0,0 +1,1576 @@ +//! Campaign driver: seed, tier, budget, per-candidate timeout, admission cap, and +//! outcome accumulation (025-exploratory-parity-discovery, T033/T037, +//! FR-001 – FR-005, FR-011/FR-012, FR-034b, FR-062). +//! +//! The driver owns the four tiers and their prerequisites (research D10): `metamorphic` +//! needs nothing external, `config-differential` is the nightly scheduled tier, +//! `container-differential` is invoked-only, and `corpus` is the weekly network-backed +//! tier. Budgets are per-tier rather than shared, because sharing lets the slow tier +//! starve the fast one — and the fast tier is where nearly all the exploration happens. +//! +//! This module drives all four tiers: the two differential tiers, the `metamorphic` tier +//! (T096), and the network-backed `corpus` tier (T108, US7) — and, since US2 (T052), +//! minimizes each new finding and emits its reviewable candidate. +//! +//! ## What minimization costs, and the two gates on paying it +//! +//! A shrink probe is two CLI invocations, so it is the most expensive thing a campaign +//! does per unit of information. The driver therefore reduces a finding only when both +//! hold: +//! +//! 1. **The signature is new to this campaign.** A finding the queue already carries has a +//! reduced input recorded against it from the campaign that admitted it; re-deriving it +//! would spend a full reduction to arrive at the same document. +//! 2. **The wall clock has not run out.** Minimization is not the place to overrun a budget +//! the candidate loop above already respects. +//! +//! Neither gate may quietly present an unreduced input as minimal: both route through +//! `Reduction::not_attempted`, which carries `isMinimal: false` **and** the reason (FR-022). +//! +//! ## Three shapes of campaign, one record +//! +//! [`run`] dispatches the metamorphic tier **before** acquiring any prerequisite, because +//! it has none to acquire (research D12): it compares deacon against deacon over a declared +//! transformation, so there is no oracle to verify, no Docker to probe, and no network to +//! reach. Routing it through the differential's prerequisite step would make the one tier a +//! contributor can run with nothing installed depend on everything. +//! +//! The `corpus` tier dispatches **after** prerequisites, because it needs both the verified +//! oracle and the network — it is the same deacon-vs-reference comparison the differential +//! runs, over inputs nobody in this repository wrote. Its inputs are pinned third-party +//! snapshots rather than generated documents, so it draws no candidates and applies no +//! mutations; everything downstream of the comparison is shared. +//! +//! All three shapes produce the **same** [`Campaign`] / [`CampaignOutcomeReport`] record and +//! admit through the same [`AdmissionQueue`], so a reader of `campaigns.json` does not need +//! to know which driver wrote a row, and the admission cap means the same thing on all of +//! them. +//! +//! ## What a campaign's exit status means +//! +//! It reflects **whether it ran**, never **what it found** (contracts/discovery-cli.md, +//! FR-058): forty differences exit `0`; an unverifiable oracle exits non-zero. Any command +//! whose status depends on its findings becomes a gate the moment someone wires it into +//! CI, and a stochastic gate makes green non-reproducible. +//! +//! ## Prerequisites fail loudly, never silently (FR-003) +//! +//! A missing or wrong-version oracle is [`HarnessError::OracleUnverified`] naming the +//! cause, and the campaign reports **no findings at all** — not an empty set, which would +//! be indistinguishable from agreement. The same for Docker on the container-backed tier. +//! There is no skip path. +//! +//! ## Two guards on what may be executed +//! +//! - **FR-011** — a candidate that cannot be executed within the declared safety +//! constraints is **discarded and counted**, never executed. The predicates are hermetic +//! (`deacon_conformance::discovery::generate::unsafe_reasons`) so they are testable +//! without a campaign; the driver's only job is to discard and count. +//! - **FR-012** — a container-bound candidate referencing an unpinned image is discarded +//! the same way. An unpinned input makes the comparison non-reproducible in the one way +//! the pinned input set cannot record: `alpine:latest` is a different image tomorrow, so +//! a finding recorded against it is a claim about content nobody can retrieve. +//! +//! Both are counted into `candidatesDiscardedUnsafe`, so a *rising* discard rate is +//! visible rather than silently shrinking the explored space. + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use deacon_conformance::discovery::generate::{ + Candidate, Generator, required_keys, unpinned_image_inputs, unsafe_reasons, +}; +use deacon_conformance::discovery::grammar::Grammar; +use deacon_conformance::discovery::mutate::{self, ApplicationCounts, MUTATION_CATALOG_VERSION}; +use deacon_conformance::discovery::queue::{ + Budget, Campaign, CampaignLane, CampaignOutcome, CampaignTier, DiscoveryData, Finding, + ObservedValues, PinnedInputSet, Witness, upsert_finding, write_campaigns, write_findings, +}; +use deacon_conformance::discovery::report::{CampaignOutcomeReport, build_campaign_outcome_report}; +use deacon_conformance::discovery::shrink::{self, ReductionInput}; +use deacon_conformance::discovery::signature::Signature; +use deacon_conformance::load::Registry; +use serde_json::Value; + +use crate::HarnessError; +use crate::normalize::NORMALIZER_VERSION; +use crate::oracle::{Oracle, OraclePin, VerifiedOracle}; +use crate::prereq; + +use super::corpus_fetch::{self, EntryStatus}; +use super::differential::{self, Characterization, DifferentialInput}; +use super::metamorphic_run::{self, Sabotage}; +use super::{candidate, minimize}; + +/// The channel a metamorphic residual is keyed under. +/// +/// The evidence document a relation compares spans both declared channel families — +/// `chan-exit-code` at `exitCode`, `chan-structured-output` under `structuredOutput` — but a +/// [`Signature`] carries exactly one channel, and every residual a relation reports is a +/// difference in the *resolved configuration document*. Keying them all here matches +/// `discovery_metamorphic`'s own use of the same channel, so a metamorphic finding and a +/// differential finding at the same path deduplicate against each other — which is correct: +/// they are the same defect observed two ways. +const METAMORPHIC_CHANNEL: &str = "chan-structured-output"; + +/// Everything one campaign needs, resolved by the caller (the bin, or a test). +/// +/// Every knob is explicit rather than read from the environment, so a test can drive a +/// short bounded campaign without mutating process-global state — which is `unsafe` under +/// this workspace's edition and hostile to a parallel runner besides. It is the same +/// explicit-seam discipline [`crate::prereq::probe_docker`] and +/// [`crate::oracle::verify_binary`] already establish. +#[derive(Debug, Clone)] +pub struct CampaignRequest { + /// The seed, exactly as recorded (FR-001). **Never defaulted** at the CLI. + pub seed_hex: String, + /// The seed's numeric value, feeding the generator's stream. + pub seed: u64, + /// Which tier to run. + pub tier: CampaignTier, + /// Which lane invoked it. + pub lane: CampaignLane, + /// The certification profile the run happens under (a `prof-` id). + pub profile: String, + /// The declared budget. + pub budget: Budget, + /// How many candidates the run **plans** to reach. The denominator of + /// `spaceCoveredFraction`: on budget exhaustion the campaign reports the portion of + /// its plan it covered rather than presenting a truncated run as complete (FR-005). + pub planned_candidates: u64, + /// The registry root, for resolving pins, channels, and the tolerance index. + pub registry_dir: PathBuf, + /// The discovery data root the queue is read from and written to. + pub discovery_dir: PathBuf, + /// Where raw artifacts are written (`target/discovery` by default). + pub report_root: PathBuf, + /// The deacon binary under test. + pub deacon_binary: PathBuf, + /// An explicit oracle binary, bypassing `PATH` resolution. The verification seam: a + /// test can point this at a stub reporting the wrong version and observe the fail-loud + /// path without touching process env. + pub oracle_override: Option, + /// Whether to persist the campaign and its findings. `false` leaves the committed data + /// root untouched, which is what an acceptance test wants. + pub persist: bool, +} + +impl CampaignRequest { + /// Whether this tier brings containers up, which decides which safety predicates apply + /// (FR-011) and whether the pinned-image guard is in force (FR-012). + pub fn container_backed(&self) -> bool { + self.tier == CampaignTier::ContainerDifferential + } +} + +/// What a campaign produced. +#[derive(Debug, Clone)] +pub struct CampaignRun { + /// The provenance record, ready to append to `campaigns.json`. + pub campaign: Campaign, + /// The findings queue after this campaign's upserts. + pub findings: Vec, + /// The ids this campaign admitted or re-witnessed, in admission order. + pub admitted: Vec, + /// The campaign's own report (FR-061). + pub report: CampaignOutcomeReport, + /// How many observations were already characterized by a case or waiver (FR-017). + /// + /// Not part of the record — the record's counters are the declared ones — but reported + /// so a reader can tell "we saw nothing" from "we saw only what we already knew". + pub characterized_observations: u64, + /// Probes minimization spent across every finding (T052). + /// + /// Also not part of the record: the *cost* of reduction is a property of the run, not + /// of the findings it produced, and a witness already carries whether its own input + /// reached minimality. Reported so a campaign can say what reduction cost it rather + /// than leaving it folded invisibly into the wall clock. + pub shrink_probes: u64, + /// Where the reviewable candidates were written (`/candidates`). + pub candidates_root: PathBuf, + /// The findings a reviewable candidate was emitted for, in admission order. + /// + /// Deliberately **not** every finding in the queue. A finding admitted from a + /// *comparison* has a full `DifferentialResult` behind it — both sides' raw evidence, + /// both normalized documents, the diff — which is what FR-024's six parts are made of. + /// A finding admitted from a minimization *drift* (FR-023) does not: it was seen in + /// passing by a probe while reducing something else, and the campaign kept its + /// signature and the input that reproduces it, not a full evidence set. + /// + /// Reported separately rather than papered over. Assembling a six-part candidate for a + /// drift would mean either fabricating the parts we did not gather or spending another + /// pair of CLI invocations per drift; leaving the queue record without a candidate and + /// *saying so* is the honest third option — the drift is a lead, and the campaign that + /// generates a candidate reproducing it will evidence it properly. + pub candidates: Vec, + /// Per-entry outcomes of the `corpus` tier, empty for every other tier (FR-051/FR-052). + /// + /// Reported alongside the counters rather than folded into them because the two + /// non-comparing outcomes are *different facts*: an unreachable entry says nothing was + /// compared, a digest mismatch says the upstream snapshot is not what was recorded. + /// A single "did not run" tally would let content drift at a pinned commit read as a + /// flaky network, which is the one confusion an ecological canary cannot afford. + pub corpus_statuses: Vec, +} + +/// Run one campaign. +/// +/// Fails loudly on any prerequisite problem and reports **no findings** in that case +/// (FR-003): an empty finding set from an unverified reference would be indistinguishable +/// from agreement, which is the single most comfortable way for this machinery to be +/// broken. +pub async fn run(request: &CampaignRequest) -> Result { + let registry = Registry::load(&request.registry_dir).map_err(|e| HarnessError::Report { + cause: format!( + "could not load the conformance registry at {}: {e}", + request.registry_dir.display() + ), + })?; + let grammar = Grammar::load_default().map_err(|e| HarnessError::Report { + cause: format!("could not load the generation grammar: {e}"), + })?; + + // The metamorphic tier branches BEFORE any prerequisite is acquired: it has none + // (research D12). Verifying an oracle it never invokes would make the one tier a + // contributor can run with nothing installed fail for the absence of a reference that + // takes no part in its comparison. + if request.tier == CampaignTier::Metamorphic { + return run_metamorphic(request, ®istry, &grammar).await; + } + + // Prerequisites first, so a campaign never produces a partial record against an + // unverified reference. + let oracle = if request.tier.requires_oracle() { + Some(verified_oracle(request).await?) + } else { + None + }; + if request.container_backed() { + prereq::require_docker().await?; + } + if request.tier == CampaignTier::Corpus { + // The network is this tier's prerequisite, and it fails as loudly as a missing + // oracle: a corpus campaign with no network reports zero entries, which is + // byte-identical to one in which the whole ecosystem agreed with deacon. + corpus_fetch::require_git().await?; + let Some(oracle) = oracle.as_ref() else { + return Err(HarnessError::OracleUnverified { + cause: "the corpus tier reached its driver with no verified oracle".to_string(), + }); + }; + return run_corpus(request, ®istry, &grammar, oracle).await; + } + + let pinned_input_set = pinned_input_set(&grammar, oracle.as_ref())?; + let campaign_id = Campaign::derive_id( + &request.seed_hex, + &pinned_input_set, + request.lane, + &request.profile, + request.tier, + ); + + let characterization = Characterization::from_registry(®istry); + if characterization.is_empty() { + tracing::warn!( + campaign = %campaign_id, + "the tolerance index is empty: every already-characterized divergence will be \ + reported as new" + ); + } + + // The standing queue, so deduplication spans campaigns (FR-030/FR-034). + let existing = + DiscoveryData::load(&request.discovery_dir).map_err(|e| HarnessError::Report { + cause: format!( + "could not load the discovery data root at {}: {e}", + request.discovery_dir.display() + ), + })?; + let mut queue = AdmissionQueue::new( + &existing.findings, + &campaign_id, + request.budget.admission_cap, + ); + + let mut generator = Generator::new(&grammar, request.seed); + let mut counters = Counters::new(); + let mut observed: BTreeSet = BTreeSet::new(); + + let wall_clock = Duration::from_secs(request.budget.wall_clock_seconds); + let per_candidate = Duration::from_secs(request.budget.per_candidate_seconds); + let started = Instant::now(); + let candidates_root = request.report_root.join("candidates"); + let mut candidates: Vec = Vec::new(); + + while counters.generated < request.planned_candidates { + if started.elapsed() >= wall_clock { + break; + } + let candidate = generator.next_candidate(); + counters.generated += 1; + for mutation in &candidate.mutations { + if let Some(slot) = counters.mutations.get_mut(mutation.category.name()) { + *slot += 1; + } + } + + // FR-011 / FR-012: discard and count, never execute. + let mut refusals = unsafe_reasons(&candidate.document, request.container_backed()); + if request.container_backed() { + for image in unpinned_image_inputs(&candidate.document) { + refusals.push(format!( + "references the unpinned image input `{image}`, which is a different \ + image tomorrow" + )); + } + } + if !refusals.is_empty() { + counters.discarded_unsafe += 1; + tracing::debug!( + candidate = %candidate.id, + reasons = ?refusals, + "discarded an unsafe candidate before execution" + ); + continue; + } + + let Some(oracle) = oracle.as_ref() else { + // Only the metamorphic tier has no oracle, and [`run`] now routes it to + // [`run_metamorphic`] before this loop is reached (T096). Arriving here would + // mean a tier was routed into the differential without its prerequisite — which + // must fail rather than compare against nothing. + return Err(HarnessError::OracleUnverified { + cause: format!( + "tier `{}` reached the differential with no verified oracle", + request.tier.as_str() + ), + }); + }; + + let workspace = match materialize(&candidate) { + Ok(w) => w, + Err(e) => { + counters.discarded_unsafe += 1; + tracing::warn!( + candidate = %candidate.id, + error = %e, + "could not materialize a candidate workspace" + ); + continue; + } + }; + + let result = differential::compare( + DifferentialInput { + candidate_id: &candidate.id, + workspace: workspace.path(), + deacon: &request.deacon_binary, + oracle, + bound: per_candidate, + report_root: &request.report_root, + // A near-valid draw violates a `required` key on purpose, and a mutated + // document was made adjacent-to-valid on purpose. Either way the candidate + // is deliberately malformed, which is the scope the strictness waivers + // characterize — a plain grammar-valid draw is NOT, so deacon refusing one + // stays news (FR-017 read narrowly, on purpose). + deliberately_invalid: candidate.kind + == deacon_conformance::discovery::generate::CandidateKind::NearValid + || !candidate.mutations.is_empty(), + }, + &characterization, + ) + .await; + + let result = match result { + Ok(r) => r, + Err(HarnessError::OracleTimeout { .. }) => { + // One pathological input must not consume the tier's whole budget. It is + // discarded and COUNTED, so a rising rate is visible rather than silent. + counters.timed_out += 1; + tracing::debug!( + candidate = %candidate.id, + "candidate exceeded its per-candidate bound" + ); + continue; + } + Err(other) => return Err(other), + }; + + counters.executed += 1; + if result.parse_stage_failure { + counters.parse_stage_failures += 1; + } + counters.characterized += result.characterized_count() as u64; + + for observation in &result.observations { + observed.insert(observation.signature.id.clone()); + } + + // The reduction's starting point, shared by every finding this candidate produced: + // the same document, the same recorded operators, and the branch's `required` keys + // read from the grammar rather than restated (research D1). + let reduction_input = ReductionInput { + document: candidate.document.clone(), + mutations: candidate.mutations.clone(), + required_keys: candidate + .branch + .map(|branch| required_keys(&grammar, branch)) + .unwrap_or_default(), + }; + let deliberately_invalid = candidate.kind + == deacon_conformance::discovery::generate::CandidateKind::NearValid + || !candidate.mutations.is_empty(); + + for observation in result.new_observations() { + let finding_id = observation.signature.finding_id(); + + // T052 — minimize, under a per-finding shrink budget. + // + // Two gates, both about not spending the expensive step on something already + // known. A finding the queue already carries has a reduced input recorded + // against it from the campaign that admitted it, and re-deriving it would cost + // a full reduction to reach the same document. A campaign past its wall clock + // has stopped exploring, and minimization is not the place to overrun a budget + // the loop above already respects. + // + // Neither gate may present the result as minimal: both route through + // `Reduction::not_attempted`, which carries the reason (FR-022). + let reduction = if !queue.is_new(&finding_id) { + shrink::Reduction::not_attempted( + &reduction_input, + "the findings queue already carries this signature, and the reduced \ + input recorded when it was admitted still stands", + ) + } else if started.elapsed() >= wall_clock { + shrink::Reduction::not_attempted( + &reduction_input, + "the campaign's wall-clock budget was exhausted before minimization \ + could run", + ) + } else { + let mut probe = minimize::DifferentialProbe::new( + observation.signature.clone(), + &candidate.id, + &request.deacon_binary, + oracle, + per_candidate, + &request.report_root, + &characterization, + deliberately_invalid, + ); + shrink::reduce( + &reduction_input, + request.budget.shrink_steps_per_finding, + &mut probe, + ) + .await? + }; + counters.shrink_probes += reduction.probes; + + let witness = Witness { + id: Witness::derived_id(&campaign_id, &candidate.id), + campaign_id: campaign_id.clone(), + candidate_id: candidate.id.clone(), + minimal_input: reduction.document.clone(), + // Never asserted, always reported: `true` only when a complete pass over + // the seven-step catalogue preserved nothing (FR-021), and `false` carries + // its reason (FR-022). + is_minimal: reduction.is_minimal, + reduction_steps: reduction.steps.clone(), + observed_values: ObservedValues { + deacon: observation.observed.deacon.clone(), + reference: observation.observed.reference.clone(), + }, + mutation_operators: candidate.operator_ids(), + }; + queue.offer(observation.signature.clone(), witness); + + // FR-023 — a step that changed the signature was rejected for THIS finding, and + // what it produced instead is a candidate finding in its own right. Admitting + // it here rather than waiting for a later campaign to rediscover it is the + // difference between observing a difference and remembering it. + for drifted in &reduction.drifted { + observed.insert(drifted.signature.id.clone()); + let drift_witness = Witness { + id: Witness::derived_id(&campaign_id, &drifted.signature.id), + campaign_id: campaign_id.clone(), + candidate_id: candidate.id.clone(), + // The rejected proposal the drift was SEEN on — not the candidate's + // document and not the reduction's result, neither of which produces + // this signature. A witness naming an input that does not reproduce + // its own signature is a record nobody can re-examine. + minimal_input: drifted.document.clone(), + // It was never itself reduced; it is a by-product of reducing + // something else, and saying otherwise would claim work nobody did. + is_minimal: false, + reduction_steps: Vec::new(), + observed_values: ObservedValues::default(), + mutation_operators: candidate.operator_ids(), + }; + queue.offer(drifted.signature.clone(), drift_witness); + } + + // The reviewable candidate (FR-024 – FR-027). Emitted only for a finding the + // queue actually holds: a candidate directory for a signature the admission cap + // turned away would be a reviewable artifact for a finding nobody can find. + if queue.holds(&finding_id) { + let dir = candidate::write(candidate::CandidateInputs { + finding_id: &finding_id, + signature: &observation.signature, + observation, + campaign_id: &campaign_id, + seed_hex: &request.seed_hex, + lane: request.lane.as_str(), + tier: request.tier.as_str(), + profile: &request.profile, + pinned_input_set: &pinned_input_set, + candidate_id: &candidate.id, + operations: &candidate.operations, + mutation_operators: &candidate.operator_ids(), + reduction: &reduction, + result: &result, + reference: candidate::ReferenceProvenance::Oracle(oracle), + registry: ®istry, + root: &candidates_root, + })?; + if !candidates.contains(&finding_id) { + candidates.push(finding_id.clone()); + } + tracing::debug!(finding = %finding_id, path = %dir.display(), "wrote a reviewable candidate"); + } + } + + drop(workspace); + } + + complete( + request, + &existing.campaigns, + Completion { + campaign_id, + pinned_input_set, + counters, + observed, + queue, + planned: request.planned_candidates, + candidates, + }, + ) +} + +/// The `metamorphic` tier: deacon against deacon over the declared relation catalogue +/// (T096, FR-044 – FR-048, research D12). +/// +/// # No prerequisite, by construction +/// +/// No oracle, no Docker, no network. That is not an optimization — it is what makes this +/// the one complete vertical slice a contributor with nothing installed can run, and it is +/// why [`run`] dispatches here *before* the prerequisite step rather than inside it. +/// +/// # The plan IS the catalogue +/// +/// There is nothing to generate for this tier: the relations are hand-authored registry +/// records, so [`CampaignRequest::planned_candidates`] — a *generator* denominator — has +/// nothing to denominate. The plan is therefore the catalogue's own size, which makes +/// `spaceCoveredFraction` read `1.0` when every relation was reached and less **only** when +/// the wall clock cut the run short. Using the request's figure would report a seven-relation +/// tier as having covered 3.5% of a two-hundred-candidate plan it never had. +/// +/// `mutationApplications` is all-zero for the same reason (no mutation occurs here, and +/// FR-010 requires every category to be present as an explicit zero rather than absent), and +/// `parseStageFailures` is zero because there is no document-syntax stage to fail: the +/// fixtures are authored, not drawn. +/// +/// # What is deliberately NOT wired in +/// +/// [`Characterization`] — the FR-017 already-characterized suppression the differential +/// applies — is **not** consulted here. That index is keyed to differential-style observable +/// paths drawn from `cases/.json` and the waivers, both of which describe a +/// deacon-vs-reference difference. A metamorphic residual is a deacon-vs-deacon difference +/// at a path the index says nothing about, so consulting it would either suppress nothing +/// (harmless but dishonest bookkeeping) or suppress by path collision (silently dropping a +/// real finding). Wiring it correctly needs a tolerance model this tier does not yet have. +/// Left out visibly rather than approximated. +async fn run_metamorphic( + request: &CampaignRequest, + registry: &Registry, + grammar: &Grammar, +) -> Result { + let catalogue = ®istry.metamorphic; + if catalogue.is_empty() { + // Not a zero-relation campaign: V32 already forbids a mandated family with no + // record, so an empty catalogue here means the registry did not load what it should + // have. Reporting it as a clean run of nothing would be byte-identical to a run in + // which every relation held — the exact indistinguishability SC-011 exists to + // prevent. + return Err(HarnessError::Report { + cause: format!( + "the metamorphic relation catalogue at {} is empty. A campaign over zero \ + relations reports the same thing as one in which every relation held, so \ + it is refused rather than run.", + request.registry_dir.join("metamorphic.json").display() + ), + }); + } + + let pinned_input_set = pinned_input_set(grammar, None)?; + let campaign_id = Campaign::derive_id( + &request.seed_hex, + &pinned_input_set, + request.lane, + &request.profile, + request.tier, + ); + + // The standing queue, so deduplication spans campaigns AND tiers (FR-030/FR-034): a + // metamorphic residual and a differential divergence at the same path derive the same + // signature, and they are the same defect observed two ways. + let existing = + DiscoveryData::load(&request.discovery_dir).map_err(|e| HarnessError::Report { + cause: format!( + "could not load the discovery data root at {}: {e}", + request.discovery_dir.display() + ), + })?; + let mut queue = AdmissionQueue::new( + &existing.findings, + &campaign_id, + request.budget.admission_cap, + ); + + let mut counters = Counters::new(); + let mut observed: BTreeSet = BTreeSet::new(); + + let root = request.report_root.join("metamorphic").join(&campaign_id); + let wall_clock = Duration::from_secs(request.budget.wall_clock_seconds); + let per_candidate = Duration::from_secs(request.budget.per_candidate_seconds); + let started = Instant::now(); + + for (index, relation) in catalogue.iter().enumerate() { + if started.elapsed() >= wall_clock { + break; + } + counters.generated += 1; + + // An index prefix so the layout is stable and readable after a failure — the same + // scheme `evaluate_catalogue` uses, kept identical so a reviewer following a + // campaign's artifacts finds the tree they expect. + let relation_root = root.join(format!("{index:02}-{}", relation.id)); + let evaluation = metamorphic_run::evaluate( + &request.deacon_binary, + &relation_root, + relation, + // The honest evaluation. `Sabotage::Break` is the SC-011 anti-inert probe, and + // a live campaign that ran it would manufacture findings out of its own + // deliberate breakage. + Sabotage::None, + ); + let outcome = match tokio::time::timeout(per_candidate, evaluation).await { + Ok(Ok(outcome)) => outcome, + // A relation the harness cannot apply is fail-loud, never a skip: reporting + // nothing for it is byte-identical to it holding. + Ok(Err(e)) => return Err(e), + Err(_elapsed) => { + // One pathological relation must not consume the tier's whole budget. It is + // discarded and COUNTED, so a rising rate is visible rather than silent. + counters.timed_out += 1; + tracing::debug!( + relation = %relation.id, + "relation exceeded its per-candidate bound" + ); + continue; + } + }; + counters.executed += 1; + + let Some(candidate) = outcome.candidate() else { + continue; + }; + let signatures = candidate.signatures(METAMORPHIC_CHANNEL); + if signatures.is_empty() { + // A SENSITIVITY failure is the absence of a difference, so there is nothing to + // key a signature on, and the catalogue deliberately refuses to invent one (it + // would collide with a genuine value difference at the touched site and merge + // two unrelated defects). The violation is still REPORTED here rather than + // dropped in silence. + tracing::warn!( + relation = %relation.id, + effect = %outcome.effect.as_str(), + transformation = %outcome.transformation, + "relation was VIOLATED but carries no deduplication key, so it cannot enter \ + the findings queue; it is identified by its relation id" + ); + continue; + } + + // The reviewable candidate — both inputs and both normalized outputs — is the + // witness's input. Recording only one side would name an input that does not + // reproduce the observation: a metamorphic input is a PAIR. + let evidence = serde_json::to_value(&candidate).map_err(|e| HarnessError::Report { + cause: format!( + "could not record the metamorphic candidate for `{}`: {e}", + relation.id + ), + })?; + + for signature in signatures { + observed.insert(signature.id.clone()); + // Paired by observable path rather than by position: `signatures()` filters, + // so an index-aligned zip would silently mis-attribute values the moment one + // residual failed to classify. + let residual = candidate.residual.iter().find(|r| r.path == signature.path); + let witness = Witness { + id: Witness::derived_id(&campaign_id, &relation.id), + campaign_id: campaign_id.clone(), + // The relation id IS the candidate id for this tier: the input is not drawn + // from a stream, it is the relation's declared base fixture plus its + // declared transformation, and naming it anything else would lose the one + // thing that reproduces the observation. + candidate_id: relation.id.clone(), + minimal_input: evidence.clone(), + // No reduction is performed, so the input is NOT minimal and says so + // (FR-022). A relation's fixture is already small by construction; that is + // not the same claim as having been reduced. + is_minimal: false, + reduction_steps: Vec::new(), + observed_values: ObservedValues { + // `reference` is the original run and `deacon` the transformed one — + // the mapping `MetamorphicCandidate::signatures` fixes, restated here + // so the two views cannot drift apart. + deacon: residual.and_then(|r| r.transformed.clone()), + reference: residual.and_then(|r| r.original.clone()), + }, + // No mutation operator produced this input; the transformation did, and it + // is named in the candidate the witness carries. + mutation_operators: Vec::new(), + }; + queue.offer(signature, witness); + } + } + + complete( + request, + &existing.campaigns, + Completion { + campaign_id, + pinned_input_set, + counters, + observed, + queue, + // The plan is the catalogue — see this function's docs. + planned: catalogue.len() as u64, + // No reviewable candidate is packaged here: FR-024's six parts are built from a + // deacon-vs-reference evidence set, and this tier compares deacon against + // deacon. The relation's own artifacts under `target/discovery/metamorphic/` + // are its review surface. + candidates: Vec::new(), + }, + ) +} + +/// The `corpus` tier: deacon against the verified pinned oracle over the pinned real-world +/// workspace corpus (T108, FR-049 – FR-054a, research D8/D10). +/// +/// # An ecological canary, not a generator +/// +/// The inputs are 33 third-party workspace snapshots nobody in this repository wrote, so +/// nothing is drawn and nothing is mutated: `mutationApplications` is all-zero (present as +/// explicit zeroes, FR-010) and the plan is the manifest's own size, exactly as the +/// metamorphic tier's plan is its catalogue's. Using the request's `planned_candidates` +/// would report a 33-entry tier as having covered 16% of a plan it never had. +/// +/// # The corpus is never a mutation seed (FR-008a / FR-054a) +/// +/// It is consumed **here**, as a direct comparison input, and nowhere else. The generator's +/// seed corpus is five committed fixtures embedded with `include_str!` +/// (`deacon_conformance::discovery::generate`), so a corpus entry cannot reach it even by +/// accident: there is no code path from a fetched workspace into the generator, and the +/// seeds are fixed at compile time rather than discovered at run time. +/// +/// # Two ways an entry does not get compared, and they are never merged +/// +/// - **Unreachable** (FR-052) — the snapshot could not be retrieved, or the pinned path +/// carries no devcontainer configuration at that commit. Nothing was compared. +/// - **Digest mismatch** (FR-051) — the snapshot was retrieved and is not what the +/// manifest recorded. It is deliberately *not* compared: attributing a change in the +/// upstream workspace to a difference between the implementations is exactly the wrong +/// conclusion, and it is the one a tolerant fetch would invite. +/// +/// Both are counted into `candidatesDiscardedUnsafe` — the record's "generated but +/// deliberately not executed" tally, which already carries the differential's timeouts — +/// and both are named individually on [`CampaignRun::corpus_statuses`], because the +/// aggregate number cannot tell a reviewer *which* fact occurred. +/// +/// # Everything after the comparison is shared +/// +/// FR-054 requires a corpus finding to enter the same pipeline as a generated one, and it +/// does so by construction rather than by parallel implementation: the same +/// [`differential::compare`], the same [`Characterization`] tolerance index, the same +/// [`AdmissionQueue`] deduplication and cap, and the same [`complete`]. The only +/// corpus-specific thing about a corpus finding is that its witness's `minimalInput` names +/// the upstream provenance — repository, commit, path, and the verified digest — because a +/// witness whose input nobody can retrieve names nothing. +async fn run_corpus( + request: &CampaignRequest, + registry: &Registry, + grammar: &Grammar, + oracle: &VerifiedOracle, +) -> Result { + let pinned_input_set = pinned_input_set(grammar, Some(oracle))?; + let campaign_id = Campaign::derive_id( + &request.seed_hex, + &pinned_input_set, + request.lane, + &request.profile, + request.tier, + ); + + let existing = + DiscoveryData::load(&request.discovery_dir).map_err(|e| HarnessError::Report { + cause: format!( + "could not load the discovery data root at {}: {e}", + request.discovery_dir.display() + ), + })?; + + if existing.corpus.is_empty() { + // Not a zero-entry campaign. An empty manifest here means the data root did not + // load what it should have, and reporting it as a clean run of nothing would be + // byte-identical to a run in which every pinned workspace agreed — the exact + // indistinguishability this tier exists to prevent. + return Err(HarnessError::Report { + cause: format!( + "the corpus manifest at {} is empty. A campaign over zero entries reports \ + the same thing as one in which every entry agreed, so it is refused \ + rather than run.", + deacon_conformance::discovery::queue::corpus_path(&request.discovery_dir).display() + ), + }); + } + + let characterization = Characterization::from_registry(registry); + if characterization.is_empty() { + tracing::warn!( + campaign = %campaign_id, + "the tolerance index is empty: every already-characterized divergence will be \ + reported as new" + ); + } + + let mut queue = AdmissionQueue::new( + &existing.findings, + &campaign_id, + request.budget.admission_cap, + ); + let mut counters = Counters::new(); + let mut observed: BTreeSet = BTreeSet::new(); + let mut statuses: Vec = Vec::new(); + + // An external temp root, reclaimed on both success and unwind. Corpus content is never + // vendored (FR-053) and never lands in the repository: materializing under the + // workspace would make `git status` the review surface for third-party bytes nobody + // reviewed. + let fetch_root = tempfile::Builder::new() + .prefix("deacon-discovery-corpus-") + .tempdir() + .map_err(|e| HarnessError::Report { + cause: format!("could not create the corpus fetch root: {e}"), + })?; + + let wall_clock = Duration::from_secs(request.budget.wall_clock_seconds); + let per_candidate = Duration::from_secs(request.budget.per_candidate_seconds); + let fetch_bound = per_candidate.max(corpus_fetch::DEFAULT_ENTRY_BOUND); + let started = Instant::now(); + + for entry in &existing.corpus { + if started.elapsed() >= wall_clock { + break; + } + counters.generated += 1; + + let status = corpus_fetch::materialize(entry, fetch_root.path(), fetch_bound).await?; + let materialized = match &status { + EntryStatus::Materialized(m) => m.clone(), + EntryStatus::Unreachable { cause, .. } => { + counters.discarded_unsafe += 1; + tracing::warn!( + entry = %entry.id, + name = %entry.name, + %cause, + "corpus entry is UNREACHABLE — nothing was compared for it" + ); + statuses.push(status); + continue; + } + EntryStatus::DigestMismatch { + expected, actual, .. + } => { + counters.discarded_unsafe += 1; + tracing::warn!( + entry = %entry.id, + name = %entry.name, + %expected, + %actual, + "corpus entry DIGEST MISMATCH — the pinned snapshot is not what was \ + recorded, so it is not compared" + ); + statuses.push(status); + continue; + } + }; + + let result = differential::compare( + DifferentialInput { + candidate_id: &entry.id, + workspace: &materialized.workspace, + deacon: &request.deacon_binary, + oracle, + bound: per_candidate, + report_root: &request.report_root, + // A real-world workspace is not deliberately malformed. deacon refusing + // one is precisely the news this tier exists to surface, so it must never + // be swallowed by the strictness characterization that covers deliberately + // invalid candidates (FR-017 read narrowly, on purpose). + deliberately_invalid: false, + }, + &characterization, + ) + .await; + + let result = match result { + Ok(r) => r, + Err(HarnessError::OracleTimeout { .. }) => { + counters.timed_out += 1; + tracing::debug!( + entry = %entry.id, + "corpus entry exceeded its per-candidate bound" + ); + statuses.push(status); + continue; + } + Err(other) => return Err(other), + }; + + counters.executed += 1; + if result.parse_stage_failure { + counters.parse_stage_failures += 1; + } + counters.characterized += result.characterized_count() as u64; + for observation in &result.observations { + observed.insert(observation.signature.id.clone()); + } + + // The witness's input NAMES the upstream provenance rather than embedding the + // workspace (FR-054, FR-053): the content is not vendored, so the reproducible + // thing is the pin plus the digest that says which bytes were compared. + let provenance = serde_json::json!({ + "corpusEntry": entry.id, + "name": entry.name, + "repository": entry.repository, + "commit": entry.commit, + "path": entry.path, + "contentDigest": materialized.digest, + }); + for observation in result.new_observations() { + let witness = Witness { + id: Witness::derived_id(&campaign_id, &entry.id), + campaign_id: campaign_id.clone(), + // The corpus entry id IS the candidate id for this tier: the input was not + // drawn from a stream, it is a pinned third-party snapshot, and naming it + // anything else would lose the one thing that reproduces the observation. + candidate_id: entry.id.clone(), + minimal_input: provenance.clone(), + // No reduction is performed, so the input is NOT minimal and says so + // (FR-022). A real-world workspace is the least minimal input this feature + // has; claiming otherwise would be the exact overstatement FR-022 forbids. + is_minimal: false, + reduction_steps: Vec::new(), + observed_values: ObservedValues { + deacon: observation.observed.deacon.clone(), + reference: observation.observed.reference.clone(), + }, + // No mutation operator produced this input; a third party did. + mutation_operators: Vec::new(), + }; + queue.offer(observation.signature.clone(), witness); + } + + statuses.push(status); + } + + // Record the digests this run settled, but only when the campaign is persisting at all + // — an acceptance test running against an isolated root must not rewrite the committed + // manifest, for the same reason it must not append to the committed queue. + if request.persist { + let written = + corpus_fetch::record_digests(&request.discovery_dir, &existing.corpus, &statuses)?; + if !written.is_empty() { + tracing::info!( + campaign = %campaign_id, + entries = ?written, + "recorded a content digest at first materialization; every later fetch \ + verifies it" + ); + } + } + + let planned = existing.corpus.len() as u64; + let mut run = complete( + request, + &existing.campaigns, + Completion { + campaign_id, + pinned_input_set, + counters, + observed, + queue, + // The plan is the manifest — see this function's docs. + planned, + // No reviewable candidate is packaged here (T103's own note: this needs a + // deliberate follow-up, not an assumption). The differential/metamorphic tiers' + // `candidate::write` and `minimize::DifferentialProbe` were both built around a + // candidate materialized from a single generated JSON document + // (`campaign::materialize`), which can freely reduce and re-synthesize because + // it owns the whole workspace tree it wrote. A corpus entry's workspace is a + // real third-party directory `corpus_fetch::materialize` fetched, which may + // contain a Dockerfile, a Compose file, local features, or other files the + // devcontainer.json references by relative path — content this tier does not + // own and must not vendor (FR-053). Reducing just the JSON portion through that + // machinery could silently break those references and misreport "no longer + // parses" as a reduction step rather than a broken reference. The witness + // already names the upstream provenance (repository/commit/path/digest), which + // is what FR-054 requires; packaging a six-part reviewable candidate for a + // real-world workspace is a separate design question this tier defers rather + // than answers by reusing machinery built for a different input shape. + candidates: Vec::new(), + }, + )?; + run.corpus_statuses = statuses; + Ok(run) +} + +/// The seven pinned inputs (FR-002), built identically for every tier. +/// +/// `oracle` is `None` for the metamorphic tier (and for the FR-042a pipeline proof), which +/// never invokes the reference — but the pin it *would* have been compared against is still +/// part of what makes its findings checkable, and the pinned input set has no optional +/// elements. One function rather than one per driver, so a tier cannot quietly record a +/// different set of pins than another. +pub(crate) fn pinned_input_set( + grammar: &Grammar, + oracle: Option<&VerifiedOracle>, +) -> Result { + Ok(PinnedInputSet { + schema_pin: deacon_conformance::CURRENT_SCHEMA_PIN.to_string(), + prose_pin: deacon_conformance::CURRENT_SPEC_PIN.to_string(), + oracle_version: match oracle { + Some(o) => o.version.clone(), + None => OraclePin::load()?.version, + }, + normalizer_version: NORMALIZER_VERSION.to_string(), + grammar_version: grammar.revision().to_string(), + mutation_catalog_version: MUTATION_CATALOG_VERSION.to_string(), + generator_version: deacon_conformance::discovery::generate::generator_identity(), + }) +} + +/// What a finished driver hands to [`complete`]. +struct Completion { + campaign_id: String, + pinned_input_set: PinnedInputSet, + counters: Counters, + observed: BTreeSet, + queue: AdmissionQueue, + /// The denominator of `spaceCoveredFraction` — what the run *planned* to reach. + planned: u64, + /// The findings a reviewable candidate was emitted for. Empty for the metamorphic + /// tier, which compares deacon against deacon and so has no two-sided evidence set to + /// package. + candidates: Vec, +} + +/// Turn a finished run into its record, persist it if asked, and build its report. +/// +/// Shared by both drivers so the exhaustion semantics, the append-only campaign history, +/// and the report shape are one implementation. A second copy would be the one that starts +/// presenting a truncated run as complete. +fn complete( + request: &CampaignRequest, + existing_campaigns: &[Campaign], + completion: Completion, +) -> Result { + let Completion { + campaign_id, + pinned_input_set, + mut counters, + observed, + queue, + planned, + candidates, + } = completion; + + // Exhaustion is "we stopped short of the plan", whatever stopped us. Reporting it only + // for the clock would let a run that ended early for any other reason present itself + // as complete — the presentation FR-005 forbids. + counters.budget_exhausted = counters.generated < planned; + + let space_covered_fraction = if planned == 0 { + 0.0 + } else { + (counters.generated as f64 / planned as f64).clamp(0.0, 1.0) + }; + + let campaign = Campaign { + id: campaign_id, + seed: request.seed_hex.clone(), + lane: request.lane, + tier: request.tier, + profile: request.profile.clone(), + pinned_input_set, + budget: request.budget, + outcome: CampaignOutcome { + candidates_generated: counters.generated, + candidates_executed: counters.executed, + candidates_discarded_unsafe: counters.discarded_unsafe + counters.timed_out, + parse_stage_failures: counters.parse_stage_failures, + budget_exhausted: counters.budget_exhausted, + space_covered_fraction, + mutation_applications: counters.mutations.clone(), + signatures_observed: observed.len() as u64, + signatures_admitted: queue.admitted.len() as u64, + signatures_suppressed: queue.suppressed.len() as u64, + }, + }; + + if request.persist { + let mut campaigns = existing_campaigns.to_vec(); + // Append-only: a campaign record is never rewritten, because a finding names the + // campaign that observed it and a rewritten campaign would retroactively change + // what that finding claims. + if !campaigns.iter().any(|c| c.id == campaign.id) { + campaigns.push(campaign.clone()); + } + write_campaigns(&request.discovery_dir, &campaigns).map_err(|e| HarnessError::Report { + cause: format!("could not write the campaign history: {e}"), + })?; + write_findings(&request.discovery_dir, &queue.findings).map_err(|e| { + HarnessError::Report { + cause: format!("could not write the findings queue: {e}"), + } + })?; + } + + let report = build_campaign_outcome_report(&campaign, &queue.admitted); + Ok(CampaignRun { + campaign, + findings: queue.findings, + admitted: queue.admitted, + report, + characterized_observations: counters.characterized, + shrink_probes: counters.shrink_probes, + candidates_root: request.report_root.join("candidates"), + candidates, + // Filled in by the corpus driver after this returns; every other tier has none. + corpus_statuses: Vec::new(), + }) +} + +/// The findings queue plus one campaign's admission bookkeeping (FR-030/FR-034/FR-034b). +/// +/// Shared by both drivers rather than reimplemented per tier. The deduplication rule (a +/// signature already in the standing queue is a re-witness, not an admission), the cap, and +/// the "suppression is counted, never silent" rule are one behavior; the copy that drifts is +/// the one that starts truncating quietly. +struct AdmissionQueue { + /// The queue after this campaign's upserts. + findings: Vec, + /// The finding ids present before the campaign started, so deduplication spans + /// campaigns (FR-030/FR-034). + known_before: BTreeSet, + /// The ids this campaign admitted or re-witnessed, in admission order. + admitted: Vec, + /// The ids the cap turned away (FR-034b). + suppressed: BTreeSet, + /// Signatures THIS campaign admitted that the standing queue did NOT already carry — + /// what the cap is measured against (FR-034b). Re-witnessing a finding the queue + /// already knows about must not consume this budget: `DEFAULT_ADMISSION_CAP`'s own + /// docs describe it as a per-campaign limit on *newly distinct* signatures, and a + /// queue that has grown past the cap would otherwise reach it on re-witnesses alone + /// and permanently suppress every genuinely new signature from then on. + newly_admitted: BTreeSet, + admission_cap: u64, + campaign_id: String, +} + +impl AdmissionQueue { + fn new(existing: &[Finding], campaign_id: &str, admission_cap: u64) -> AdmissionQueue { + AdmissionQueue { + findings: existing.to_vec(), + known_before: existing.iter().map(|f| f.id.clone()).collect(), + admitted: Vec::new(), + suppressed: BTreeSet::new(), + newly_admitted: BTreeSet::new(), + admission_cap, + campaign_id: campaign_id.to_string(), + } + } + + /// Offer one observation to the queue. + /// + /// Two rules, both preserved verbatim by the extraction: + /// + /// - **A signature the standing queue already knows is never suppressed**, whatever the + /// cap. Refusing to re-witness something a reviewer has already seen would let the cap + /// quietly stop `lastObserved` from advancing, and a finding that stopped being + /// re-witnessed for that reason is indistinguishable from one that stopped + /// reproducing. + /// - **Only a genuinely new signature can be suppressed**, and every suppression is + /// recorded (FR-034b) — never a silent truncation, so a campaign that keeps hitting + /// the cap is itself a visible signal that something systemic is diverging. + /// + /// The cap is measured against `newly_admitted`, not `admitted`: `admitted` holds every + /// id this campaign *touched*, re-witnesses included, and measuring the cap against + /// that would mean a campaign against a standing queue larger than the cap could never + /// admit anything new — every campaign would exhaust its budget on re-witnesses before + /// reaching its first genuinely new signature. + /// Whether this campaign has never seen `finding_id` — neither in the standing queue + /// nor earlier in its own run. + /// + /// A read-only query, added for T052 rather than a change to how admission works: it is + /// what lets the driver decide whether minimization is worth an oracle invocation. A + /// signature the queue already carries has a reduced input recorded against it, and + /// re-deriving that costs a full reduction to reach the same document. + fn is_new(&self, finding_id: &str) -> bool { + !self.known_before.contains(finding_id) && !self.admitted.iter().any(|id| id == finding_id) + } + + /// Whether the queue actually holds a record for `finding_id` after everything offered + /// so far — false for a signature the cap turned away. + fn holds(&self, finding_id: &str) -> bool { + self.findings.iter().any(|f| f.id == finding_id) + } + + fn offer(&mut self, signature: Signature, witness: Witness) { + let finding_id = signature.finding_id(); + let already_known = + self.known_before.contains(&finding_id) || self.admitted.contains(&finding_id); + if !already_known && self.newly_admitted.len() as u64 >= self.admission_cap { + self.suppressed.insert(finding_id); + return; + } + upsert_finding(&mut self.findings, signature, witness, &self.campaign_id); + if !self.admitted.contains(&finding_id) { + self.admitted.push(finding_id.clone()); + } + if !already_known { + self.newly_admitted.insert(finding_id); + } + } +} + +/// Resolve and verify the oracle, mapping every failure onto +/// [`HarnessError::OracleUnverified`] with the underlying cause preserved. +/// +/// One variant rather than five so a caller can say "the reference could not be verified" +/// in a single match, while the message still names *which* prerequisite failed — the +/// distinction FR-003 draws between failing loudly and failing usefully. +async fn verified_oracle(request: &CampaignRequest) -> Result { + let result = match &request.oracle_override { + Some(path) => { + let pin = OraclePin::load()?; + crate::oracle::verify_binary(path, &pin, crate::oracle::VERSION_QUERY_BOUND).await + } + None => Oracle::acquire().await, + }; + result.map_err(|e| HarnessError::OracleUnverified { + cause: e.to_string(), + }) +} + +/// A materialized candidate workspace, reclaimed when dropped. +/// +/// The `.devcontainer/devcontainer.json` carries the candidate. The Compose file and +/// Dockerfile beside it are **fixture scaffolding**, not candidate content: a candidate +/// that names `docker-compose.yml` must find one, or the comparison would measure whether +/// each implementation reports a missing file the same way rather than how each resolves +/// the configuration. They are written at both the workspace root and inside +/// `.devcontainer/` because a Compose path is resolved against different anchors by +/// different code paths, and a generated candidate is not the place to adjudicate that. +pub(crate) struct CandidateWorkspace { + dir: tempfile::TempDir, +} + +impl CandidateWorkspace { + pub(crate) fn path(&self) -> &Path { + self.dir.path() + } +} + +/// The minimal Compose project a Compose-shaped candidate needs to exist. +pub(crate) const COMPOSE_SCAFFOLD: &str = "services:\n app:\n image: alpine:3.19\n db:\n image: alpine:3.19\n cache:\n image: alpine:3.19\n"; + +/// The minimal Dockerfile a Dockerfile-shaped candidate needs to exist. +pub(crate) const DOCKERFILE_SCAFFOLD: &str = "FROM alpine:3.19\n"; + +fn materialize(candidate: &Candidate) -> std::io::Result { + materialize_document(&candidate.document) +} + +/// Materialize an arbitrary configuration document into a throwaway workspace. +/// +/// Shared with [`super::minimize`] and [`super::candidate`] rather than reimplemented per +/// caller. That is load-bearing rather than tidy: a minimization probe runs a *reduced* +/// document and asks whether the same signature still appears, so a probe workspace whose +/// scaffolding differed from the candidate workspace's would be measuring the scaffold. The +/// emitted candidate's `fixture/` tree is the same shape for the same reason — FR-027's +/// self-containment claim is that the fixture reproduces the observation, and it can only +/// do that if it is the tree the observation was made in. +pub(crate) fn materialize_document(document: &Value) -> std::io::Result { + let dir = tempfile::Builder::new() + .prefix("deacon-discovery-") + .tempdir()?; + write_workspace_tree(dir.path(), document)?; + Ok(CandidateWorkspace { dir }) +} + +/// Write the candidate workspace tree (config + scaffolding) into `root`. +pub(crate) fn write_workspace_tree(root: &Path, document: &Value) -> std::io::Result<()> { + let config_dir = root.join(".devcontainer"); + std::fs::create_dir_all(&config_dir)?; + std::fs::write( + config_dir.join("devcontainer.json"), + serde_json::to_string_pretty(document) + .unwrap_or_else(|e| unreachable!("a candidate document always serializes: {e}")), + )?; + for base in [root, config_dir.as_path()] { + std::fs::write(base.join("docker-compose.yml"), COMPOSE_SCAFFOLD)?; + std::fs::write(base.join("docker-compose.override.yml"), COMPOSE_SCAFFOLD)?; + std::fs::write(base.join("Dockerfile"), DOCKERFILE_SCAFFOLD)?; + } + Ok(()) +} + +/// The running tallies a campaign accumulates. +struct Counters { + generated: u64, + executed: u64, + discarded_unsafe: u64, + timed_out: u64, + parse_stage_failures: u64, + characterized: u64, + /// Probes minimization spent — reported so the cost of reduction is visible rather + /// than folded invisibly into the wall clock. + shrink_probes: u64, + budget_exhausted: bool, + mutations: ApplicationCounts, +} + +impl Counters { + fn new() -> Counters { + Counters { + generated: 0, + executed: 0, + discarded_unsafe: 0, + timed_out: 0, + parse_stage_failures: 0, + characterized: 0, + shrink_probes: 0, + budget_exhausted: false, + // All eleven keys from the start (FR-010): a category absent from the map is + // indistinguishable from one that was never applied, so the map is never built + // by inserting only the categories that fired. + mutations: mutate::empty_application_counts(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use deacon_conformance::discovery::generate::{CandidateKind, Operation}; + use serde_json::json; + + fn candidate(id: &str, document: serde_json::Value) -> Candidate { + Candidate { + id: id.to_string(), + index: 0, + kind: CandidateKind::Valid, + branch: None, + fixture: None, + document, + mutations: Vec::new(), + violated_required: Vec::new(), + operations: vec![Operation::read_configuration()], + } + } + + #[test] + fn a_materialized_workspace_carries_the_candidate_and_its_scaffolding() { + let candidate = candidate( + "cnd-11111111", + json!({ "image": "alpine:3.19", "name": "m" }), + ); + let workspace = materialize(&candidate).expect("materializes"); + let config = workspace + .path() + .join(".devcontainer") + .join("devcontainer.json"); + let raw = std::fs::read_to_string(&config).expect("read back"); + assert_eq!( + serde_json::from_str::(&raw).expect("valid JSON"), + candidate.document, + "the document written must be the document compared" + ); + // Scaffolding at both anchors: a Compose-shaped candidate must find its file + // whichever anchor an implementation resolves against, or the comparison would + // measure missing-file reporting rather than configuration resolution. + for base in [ + workspace.path().to_path_buf(), + workspace.path().join(".devcontainer"), + ] { + assert!(base.join("docker-compose.yml").is_file()); + assert!(base.join("docker-compose.override.yml").is_file()); + assert!(base.join("Dockerfile").is_file()); + } + } + + #[test] + fn the_workspace_is_reclaimed_on_drop() { + let candidate = candidate("cnd-22222222", json!({ "image": "alpine:3.19" })); + let path = { + let workspace = materialize(&candidate).expect("materializes"); + workspace.path().to_path_buf() + }; + assert!( + !path.exists(), + "a campaign generates thousands of workspaces; one that outlives its candidate \ + fills the disk" + ); + } + + // ----------------------------------------------------------------------- + // The shared admission queue (T096 extraction) + // ----------------------------------------------------------------------- + + fn signature(path: &str) -> Signature { + Signature::derive( + METAMORPHIC_CHANNEL, + &deacon_conformance::discovery::signature::Divergence { + kind: deacon_conformance::discovery::signature::DivergenceKind::Value, + path, + deacon: None, + reference: None, + }, + ) + } + + fn witness(campaign_id: &str, candidate_id: &str) -> Witness { + Witness { + id: Witness::derived_id(campaign_id, candidate_id), + campaign_id: campaign_id.to_string(), + candidate_id: candidate_id.to_string(), + minimal_input: json!({}), + is_minimal: false, + reduction_steps: Vec::new(), + observed_values: ObservedValues::default(), + mutation_operators: Vec::new(), + } + } + + #[test] + fn a_new_signature_is_admitted_once_however_often_it_is_offered() { + let mut queue = AdmissionQueue::new(&[], "cmp-11111111", 25); + let sig = signature("structuredOutput.configuration.name"); + + queue.offer(sig.clone(), witness("cmp-11111111", "cnd-a")); + queue.offer(sig.clone(), witness("cmp-11111111", "cnd-b")); + + assert_eq!( + queue.admitted, + vec![sig.finding_id()], + "one signature is one finding however many witnesses it collects" + ); + assert_eq!(queue.findings.len(), 1); + assert_eq!(queue.findings[0].witnesses.len(), 2); + assert!(queue.suppressed.is_empty()); + } + + #[test] + fn the_cap_turns_away_new_signatures_and_reports_every_one_it_did() { + let mut queue = AdmissionQueue::new(&[], "cmp-22222222", 2); + let first = signature("structuredOutput.configuration.name"); + let second = signature("structuredOutput.configuration.image"); + let third = signature("structuredOutput.configuration.remoteUser"); + queue.offer(first.clone(), witness("cmp-22222222", "cnd-1")); + queue.offer(second.clone(), witness("cmp-22222222", "cnd-2")); + queue.offer(third.clone(), witness("cmp-22222222", "cnd-3")); + + assert_eq!( + queue.admitted, + vec![first.finding_id(), second.finding_id()], + "the cap bounds admissions in offer order" + ); + assert_eq!( + queue.suppressed.iter().cloned().collect::>(), + vec![third.finding_id()], + "the excess is REPORTED (FR-034b), never a silent truncation: a campaign that \ + keeps hitting the cap is itself a visible signal that something systemic is \ + diverging" + ); + assert!( + !queue.findings.iter().any(|f| f.id == third.finding_id()), + "a suppressed signature must not reach the queue" + ); + } + + #[test] + fn a_signature_the_standing_queue_already_knows_is_never_suppressed() { + let known = signature("structuredOutput.configuration.image"); + let mut seed = Vec::new(); + upsert_finding( + &mut seed, + known.clone(), + witness("cmp-00000000", "cnd-old"), + "cmp-00000000", + ); + + // A cap of ZERO, so nothing new can enter at all — and the standing finding still + // collects its witness. Refusing to re-witness what a reviewer has already seen + // would let the cap quietly stop `lastObserved` from advancing, and a finding that + // stopped being re-witnessed for THAT reason is indistinguishable from one that + // stopped reproducing. + let mut queue = AdmissionQueue::new(&seed, "cmp-33333333", 0); + queue.offer(known.clone(), witness("cmp-33333333", "cnd-new")); + let fresh = signature("structuredOutput.configuration.name"); + queue.offer(fresh.clone(), witness("cmp-33333333", "cnd-other")); + + assert_eq!(queue.admitted, vec![known.finding_id()]); + assert_eq!( + queue.suppressed.iter().cloned().collect::>(), + vec![fresh.finding_id()], + "a zero cap admits nothing NEW, which is exactly what it should mean" + ); + let record = queue + .findings + .iter() + .find(|f| f.id == known.finding_id()) + .expect("the standing finding survives"); + assert_eq!(record.witnesses.len(), 2, "it collected the new witness"); + assert_eq!(record.last_observed, "cmp-33333333"); + } + + #[test] + fn the_counters_start_with_every_mutation_category_present() { + let counters = Counters::new(); + assert_eq!(counters.mutations.len(), 11); + assert!(counters.mutations.values().all(|&n| n == 0)); + assert_eq!( + mutate::unapplied_categories(&counters.mutations).len(), + 11, + "before anything runs, every category is unapplied — and says so" + ); + assert!(!counters.budget_exhausted); + } +} diff --git a/crates/parity-harness/src/discovery/candidate.rs b/crates/parity-harness/src/discovery/candidate.rs new file mode 100644 index 00000000..1607121c --- /dev/null +++ b/crates/parity-harness/src/discovery/candidate.rs @@ -0,0 +1,650 @@ +//! Reviewable-candidate assembly under `target/discovery/candidates//` +//! (025-exploratory-parity-discovery, T049 – T051, data-model.md § 9, +//! FR-024 – FR-027). +//! +//! Six parts, all required for the candidate to be self-contained (FR-024/FR-027): +//! +//! | Part | File | What it answers | +//! |---|---|---| +//! | minimal fixture | `fixture/` | *what input?* — a materializable workspace tree | +//! | operations + argv | `context.json` | *run how?* — plus the campaign, seed, and pinned input set (FR-026) | +//! | raw evidence | `raw.json` | *what did each side actually emit?* | +//! | normalized difference | `normalized.json` | *what did the comparison conclude?* | +//! | reference provenance | `provenance.json` | *compared against what, and how was the input produced?* | +//! | suggested behavior mapping | `mapping.json` | *does the project already have a name for this?* | +//! +//! ## Raw and normalized are separate files (T050, FR-014) +//! +//! Mirroring the committed-snapshot layout (the FR-016 precedent from 022). Conflating them +//! would make it impossible to tell a difference the *implementations* produced from one the +//! *normalizer* produced — which is precisely the `normalizer-defect` classification the +//! triage vocabulary reserves, and a reviewer who cannot separate the two cannot reach it. +//! +//! ## `mapping.json` never invents an identity (T051, FR-025) +//! +//! It carries either a `bhv-` id **that resolves in the loaded registry** or an explicit +//! `{"match": "none"}`. The guard is structural rather than careful: whatever the suggestion +//! rule proposes is filtered against `registry.behaviors` before it is written, and a +//! proposal that does not survive the filter becomes a no-match. A suggestion that fabricated +//! a plausible-looking id would turn the reviewer's job into *verifying* a mapping rather +//! than *deciding* one, which is worse than offering none. +//! +//! The rule itself is deliberately narrow: a behavior is suggested only when a committed case +//! already declares an assertion **on the same channel, at the same observable path**. That is +//! a real, reviewed statement that the project reasons about that field; anything looser — +//! matching by area, by subcommand, by word overlap — would be the report asserting a +//! relationship nobody established. + +use std::path::{Path, PathBuf}; + +use deacon_conformance::discovery::queue::PinnedInputSet; +use deacon_conformance::discovery::shrink::Reduction; +use deacon_conformance::discovery::signature::Signature; +use deacon_conformance::load::Registry; +use serde_json::{Map, Value, json}; + +use crate::HarnessError; +use crate::normalize::NORMALIZER_VERSION; +use crate::oracle::{OracleSource, VerifiedOracle}; + +use super::campaign::write_workspace_tree; +use super::differential::{DifferentialResult, Observation, SideEvidence}; + +/// The six file names, in the order data-model.md § 9 lists them. +/// +/// Named once so the writer and the completeness check cannot disagree about what "all six +/// parts" means — SC-005 is a claim about this list, and a second copy of it would be a +/// second definition of the claim. +pub const CANDIDATE_PARTS: [&str; 6] = [ + "fixture", + "context.json", + "raw.json", + "normalized.json", + "provenance.json", + "mapping.json", +]; + +/// What the deacon side of a comparison was measured against — the honest answer to +/// `provenance.json`'s "compared against what?". +/// +/// A two-variant enum rather than an unconditional [`VerifiedOracle`] because there are +/// genuinely two answers and conflating them would be a lie in the one file whose job is to +/// say what the evidence is. Every *campaign* tier compares against the verified pinned +/// oracle; the FR-042a pipeline proof compares deacon against **its own unperturbed run**, +/// with a known difference injected into one side at the sealed evidence-source boundary. +/// +/// The proof's counterpart is not a reference implementation and must never be recorded as +/// one: a `provenance.json` that claimed a verified oracle for a run that never invoked one +/// would make the candidate's central claim — *this is what the two implementations did* — +/// false, and a reviewer has no way to detect that from inside the file. +#[derive(Debug, Clone, Copy)] +pub enum ReferenceProvenance<'a> { + /// The verified pinned oracle. Taking the verified type rather than a path is what + /// makes "never compare against an unverified reference" (FR-003) a fact about the + /// value rather than a rule the caller has to remember. + Oracle(&'a VerifiedOracle), + /// deacon's own unperturbed run, with a difference injected into the other side at the + /// sealed evidence-source boundary (FR-042a, research D7). + /// + /// Carries the injection's record id so the candidate names *what was planted*: this + /// candidate documents the machinery working, not a divergence between two + /// implementations, and it says so in its own provenance. + InjectedSelfComparison { + /// The `reg-`-shaped perturbation record that was applied. + injection: &'a str, + }, +} + +/// Everything one reviewable candidate is assembled from. +pub struct CandidateInputs<'a> { + /// The finding this candidate belongs to (`fnd-…`) — also its directory name. + pub finding_id: &'a str, + /// The signature under review. + pub signature: &'a Signature, + /// The observation that produced it — the concrete values behind the signature. + pub observation: &'a Observation, + /// The campaign record: seed, lane, tier, profile, and the pinned input set (FR-026). + pub campaign_id: &'a str, + pub seed_hex: &'a str, + pub lane: &'a str, + pub tier: &'a str, + pub profile: &'a str, + pub pinned_input_set: &'a PinnedInputSet, + /// The generated candidate that surfaced it. + pub candidate_id: &'a str, + /// The operations, `${WORKSPACE}`-tokenized. + pub operations: &'a [deacon_conformance::discovery::generate::Operation], + /// The `mop-` operators applied to produce the candidate (FR-009 attribution). + pub mutation_operators: &'a [String], + /// The reduction: the minimal fixture and how it was reached. + pub reduction: &'a Reduction, + /// The comparison the observation came from — the raw and normalized evidence. + pub result: &'a DifferentialResult, + /// What the deacon side was compared **against** (FR-026's provenance). + pub reference: ReferenceProvenance<'a>, + /// The loaded registry, for resolving the suggested mapping. **Read only** — FR-018 + /// forbids discovery writing anything the registry owns. + pub registry: &'a Registry, + /// The candidates root, normally `target/discovery/candidates`. + pub root: &'a Path, +} + +/// Write one reviewable candidate, returning its directory. +/// +/// The directory is **replaced** rather than merged into: a stale file from an earlier +/// campaign sitting beside a fresh one would be evidence about a comparison nobody ran, and +/// the reviewer has no way to tell which is which. +pub fn write(inputs: CandidateInputs<'_>) -> Result { + let dir = inputs.root.join(inputs.finding_id); + if dir.exists() { + std::fs::remove_dir_all(&dir).map_err(|e| io_error(&dir, e))?; + } + std::fs::create_dir_all(&dir).map_err(|e| io_error(&dir, e))?; + + // 1. The minimal fixture, as a materializable workspace tree — the SAME shape the + // campaign compared, so FR-027's "reproducing requires only the candidate" holds. + let fixture = dir.join("fixture"); + std::fs::create_dir_all(&fixture).map_err(|e| io_error(&fixture, e))?; + write_workspace_tree(&fixture, &inputs.reduction.document) + .map_err(|e| io_error(&fixture, e))?; + + // 2. Operations + argv, the campaign and seed, and the pinned input set (FR-026). + write_json(&dir.join("context.json"), &context(&inputs))?; + + // 3./4. Raw and normalized, SEPARATELY (T050, FR-014). + write_json(&dir.join("raw.json"), &raw(&inputs))?; + write_json(&dir.join("normalized.json"), &normalized(&inputs))?; + + // 5. Reference provenance + how the input was produced and reduced. + write_json(&dir.join("provenance.json"), &provenance(&inputs))?; + + // 6. The suggested behavior mapping, or an explicit no-match (T051, FR-025). + write_json(&dir.join("mapping.json"), &mapping(&inputs))?; + + Ok(dir) +} + +/// The candidate directory for a finding under `root`. +pub fn candidate_dir(root: &Path, finding_id: &str) -> PathBuf { + root.join(finding_id) +} + +// --------------------------------------------------------------------------- +// The six parts +// --------------------------------------------------------------------------- + +fn context(inputs: &CandidateInputs<'_>) -> Value { + json!({ + "findingId": inputs.finding_id, + "signature": inputs.signature, + "campaignId": inputs.campaign_id, + "seed": inputs.seed_hex, + "lane": inputs.lane, + "tier": inputs.tier, + "profile": inputs.profile, + "candidateId": inputs.candidate_id, + // `${WORKSPACE}` is the same token the declarative conformance runner uses, so a + // candidate's operations read exactly the way a case's do — which is the shape a + // reviewer will need if they promote this finding. + "operations": inputs.operations, + "fixture": "fixture", + "pinnedInputSet": pinned_input_set(inputs.pinned_input_set), + "reproduce": format!( + "materialize `fixture/` into a workspace W, then run each operation with \ + ${{WORKSPACE}} = W against deacon and against @devcontainers/cli@{}", + inputs.pinned_input_set.oracle_version + ), + }) +} + +/// Both sides' evidence, **unnormalized** — the bytes each CLI actually produced. +fn raw(inputs: &CandidateInputs<'_>) -> Value { + json!({ + "deacon": raw_side(&inputs.result.deacon), + "reference": raw_side(&inputs.result.reference), + }) +} + +fn raw_side(side: &SideEvidence) -> Value { + json!({ + "outcome": side.outcome.as_str(), + // Preserved but never compared (FR-016): two implementations spell "I refused + // this" with different non-zero codes, and comparing the numbers would report the + // wording of a status rather than its meaning. + "exitCode": side.exit_code, + "stdout": read_capture(&side.stdout_path), + "stderr": read_capture(&side.stderr_path), + "stdoutPath": side.stdout_path.to_string_lossy(), + "stderrPath": side.stderr_path.to_string_lossy(), + }) +} + +/// Read a capture back, or record why it could not be read. +/// +/// A missing capture is recorded as an explicit note rather than as an empty string: an +/// empty stdout and an unreadable one are different facts, and the reviewer reading +/// `raw.json` is precisely the person who needs to tell them apart. +fn read_capture(path: &Path) -> Value { + match std::fs::read(path) { + Ok(bytes) => Value::String(String::from_utf8_lossy(&bytes).into_owned()), + Err(e) => json!({ "unavailable": format!("{}: {e}", path.display()) }), + } +} + +/// Both sides' normalized evidence and the difference the comparison concluded. +fn normalized(inputs: &CandidateInputs<'_>) -> Value { + json!({ + "normalizerVersion": NORMALIZER_VERSION, + "deacon": inputs.result.deacon.normalized, + "reference": inputs.result.reference.normalized, + "difference": { + "signature": inputs.signature, + "channel": inputs.signature.channel, + "path": inputs.signature.path, + "deacon": inputs.observation.observed.deacon, + "reference": inputs.observation.observed.reference, + }, + // Every difference the same comparison saw, so a reviewer can tell a lone + // divergence from one member of a cluster without re-running anything. + "allObservations": inputs + .result + .observations + .iter() + .map(|o| json!({ + "signature": o.signature, + "deacon": o.observed.deacon, + "reference": o.observed.reference, + "new": o.is_new(), + })) + .collect::>(), + "parseStageFailure": inputs.result.parse_stage_failure, + }) +} + +fn provenance(inputs: &CandidateInputs<'_>) -> Value { + json!({ + "reference": reference_provenance(inputs), + "mutationOperators": inputs.mutation_operators, + "reduction": { + "steps": inputs.reduction.steps, + "isMinimal": inputs.reduction.is_minimal, + "notMinimalReason": inputs.reduction.not_minimal_reason, + "probes": inputs.reduction.probes, + "originalSize": inputs.reduction.original_size, + "reducedSize": inputs.reduction.reduced_size, + "sizeReductionFraction": inputs.reduction.size_reduction_fraction(), + "remainingMutations": inputs.reduction.remaining_mutations, + "driftedSignatures": inputs + .reduction + .drifted + .iter() + .map(|d| d.signature.id.clone()) + .collect::>(), + "catalogue": deacon_conformance::discovery::shrink::reduction_catalogue_identity(), + }, + "pinnedInputSet": pinned_input_set(inputs.pinned_input_set), + }) +} + +/// What the deacon side was compared against, said plainly. +/// +/// `kind` is the first key on purpose: a reviewer reading `provenance.json` must be able to +/// tell "this documents a divergence from the reference" from "this documents the pipeline +/// proving itself" without inferring it from which other keys happen to be present. +fn reference_provenance(inputs: &CandidateInputs<'_>) -> Value { + match inputs.reference { + ReferenceProvenance::Oracle(oracle) => json!({ + "kind": "verified-oracle", + "version": oracle.version, + "path": oracle.path.to_string_lossy(), + "source": match oracle.source { + OracleSource::Override => "override", + OracleSource::PathLookup => "path-lookup", + }, + // The oracle is a `VerifiedOracle`, and only the verification path hands one + // out — so this is a statement about how the value was obtained, not a hopeful + // assertion (FR-003). + "verified": true, + "verification": format!( + "reported version equals the pinned {}", + inputs.pinned_input_set.oracle_version + ), + }), + ReferenceProvenance::InjectedSelfComparison { injection } => json!({ + "kind": "injected-self-comparison", + "injection": injection, + "verified": false, + "verification": "NOT a reference comparison: deacon was compared against its \ + own unperturbed run, with the named difference injected into \ + one side at the sealed evidence-source boundary. This \ + candidate is evidence that the PIPELINE works (FR-042a), not \ + evidence that the two implementations disagree.", + }), + } +} + +fn pinned_input_set(pins: &PinnedInputSet) -> Value { + json!({ + "schemaPin": pins.schema_pin, + "prosePin": pins.prose_pin, + "oracleVersion": pins.oracle_version, + "normalizerVersion": pins.normalizer_version, + "grammarVersion": pins.grammar_version, + "mutationCatalogVersion": pins.mutation_catalog_version, + "generatorVersion": pins.generator_version, + }) +} + +// --------------------------------------------------------------------------- +// T051 — the suggested behavior mapping +// --------------------------------------------------------------------------- + +/// The suggested mapping: a resolvable `bhv-` id, or an explicit no-match (FR-025). +fn mapping(inputs: &CandidateInputs<'_>) -> Value { + let suggestions = suggest(inputs.registry, inputs.signature); + + match suggestions.as_slice() { + [only] => json!({ + "match": "behavior", + "behavior": only.behavior, + "basis": format!( + "case `{}` already declares an assertion on `{}` at `{}`", + only.case, inputs.signature.channel, only.path + ), + "confidence": "suggestion", + "note": "A suggestion, not a decision. The reviewer assigns the behavior; this \ + names the one the registry already reasons about at this exact \ + observable path.", + }), + // Zero is the ordinary case for a genuinely new difference. Several is ambiguity, + // and picking one of them would be a coin flip wearing a suggestion's clothes — so + // the candidates are LISTED (every one resolvable) and the decision is left where + // it belongs. + _ => json!({ + "match": "none", + "reason": if suggestions.is_empty() { + format!( + "no committed case declares an assertion on `{}` at `{}`, so the \ + registry has no existing identity for this difference", + inputs.signature.channel, inputs.signature.path + ) + } else { + format!( + "{} committed behaviors are declared at `{}` on `{}`; choosing one \ + would be a guess presented as a suggestion", + suggestions.len(), + inputs.signature.path, + inputs.signature.channel + ) + }, + "considered": suggestions + .iter() + .map(|s| s.behavior.clone()) + .collect::>(), + }), + } +} + +/// One suggestion and the reviewed record it rests on. +#[derive(Debug, Clone, PartialEq, Eq)] +struct Suggestion { + behavior: String, + case: String, + path: String, +} + +/// Behaviors the registry already declares at this signature's exact observable location. +/// +/// Every returned id is filtered against `registry.behaviors`, so an id that does not +/// resolve cannot escape this function — FR-025 held structurally rather than by care. +fn suggest(registry: &Registry, signature: &Signature) -> Vec { + let mut out: Vec = Vec::new(); + for case in ®istry.cases { + for expected in &case.expected { + if expected.channel != signature.channel { + continue; + } + let Some(assertion) = &expected.assertion else { + continue; + }; + for path in assertion_paths(assertion) { + if path != signature.path { + continue; + } + for behavior in &case.behaviors { + // The structural guard: only an id that RESOLVES may be suggested. + if !registry.behaviors.iter().any(|b| &b.id == behavior) { + continue; + } + let suggestion = Suggestion { + behavior: behavior.clone(), + case: case.id.clone(), + path: path.clone(), + }; + if !out.iter().any(|s| s.behavior == suggestion.behavior) { + out.push(suggestion); + } + } + } + } + } + out.sort_by(|a, b| a.behavior.cmp(&b.behavior)); + out +} + +/// The dotted observable paths a declarative assertion names. +/// +/// Only the document-shaped assertion kinds contribute: `jsonSubset` and `jsonEquals` carry +/// a path space that is the same one a signature's `path` lives in. `equals` / `contains` / +/// `matches` / `nonZero` do not — they assert about a whole stream or a whole status — so +/// they name no path and contribute nothing rather than contributing the empty path, which +/// would match every signature on their channel. +fn assertion_paths(assertion: &Value) -> Vec { + let Some(object) = assertion.as_object() else { + return Vec::new(); + }; + let mut out = Vec::new(); + for kind in ["jsonSubset", "jsonEquals"] { + if let Some(payload) = object.get(kind) { + collect_paths(payload, &mut String::new(), &mut out); + } + } + out +} + +fn collect_paths(value: &Value, prefix: &mut String, out: &mut Vec) { + let Value::Object(map) = value else { + return; + }; + for (key, child) in map { + let restore = prefix.len(); + if !prefix.is_empty() { + prefix.push('.'); + } + prefix.push_str(key); + out.push(prefix.clone()); + collect_paths(child, prefix, out); + prefix.truncate(restore); + } +} + +// --------------------------------------------------------------------------- +// Writing +// --------------------------------------------------------------------------- + +/// Write one part atomically: a temp file beside the target, then a rename. +/// +/// A candidate directory is read by a human while campaigns may still be running, and a +/// plain write truncates-then-streams — a reader arriving mid-write sees a truncated +/// document and concludes the evidence is malformed. +fn write_json(path: &Path, value: &Value) -> Result<(), HarnessError> { + let mut rendered = serde_json::to_string_pretty(value).map_err(|e| HarnessError::Report { + cause: format!("could not render {}: {e}", path.display()), + })?; + rendered.push('\n'); + let temp = path.with_extension("json.tmp"); + std::fs::write(&temp, rendered).map_err(|e| io_error(&temp, e))?; + std::fs::rename(&temp, path).map_err(|e| io_error(path, e)) +} + +fn io_error(path: &Path, error: std::io::Error) -> HarnessError { + HarnessError::Report { + cause: format!( + "could not write the reviewable candidate at {}: {error}", + path.display() + ), + } +} + +/// Whether every one of the six parts is present under `dir` (SC-005's claim). +/// +/// Exposed so a test asserts the claim against the same list the writer honors, rather than +/// against a second list that could drift from it. +pub fn missing_parts(dir: &Path) -> Vec<&'static str> { + CANDIDATE_PARTS + .into_iter() + .filter(|part| !dir.join(part).exists()) + .collect() +} + +/// Every JSON part, parsed — so a caller can assert on content without re-deriving the +/// file list. A part that is absent or unparseable is simply missing from the map, which +/// [`missing_parts`] reports separately. +pub fn read_parts(dir: &Path) -> Map { + let mut out = Map::new(); + for part in CANDIDATE_PARTS { + if !part.ends_with(".json") { + continue; + } + if let Ok(raw) = std::fs::read_to_string(dir.join(part)) + && let Ok(value) = serde_json::from_str::(&raw) + { + out.insert(part.to_string(), value); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use deacon_conformance::model::{CHAN_EXIT_CODE, CHAN_STRUCTURED_OUTPUT}; + + fn signature(channel: &str, path: &str) -> Signature { + use deacon_conformance::discovery::signature::{Divergence, DivergenceKind}; + Signature::derive( + channel, + &Divergence { + kind: DivergenceKind::Value, + path, + deacon: None, + reference: None, + }, + ) + } + + #[test] + fn the_six_parts_are_declared_once() { + assert_eq!( + CANDIDATE_PARTS, + [ + "fixture", + "context.json", + "raw.json", + "normalized.json", + "provenance.json", + "mapping.json", + ], + "SC-005 is a claim about THIS list; a second copy of it would be a second \ + definition of the claim" + ); + } + + #[test] + fn assertion_paths_are_read_only_from_document_shaped_assertions() { + let subset = json!({ "jsonSubset": { "configuration": { "name": "x" } } }); + assert_eq!( + assertion_paths(&subset), + vec!["configuration", "configuration.name"], + "a nested subset names every path it constrains, parent included" + ); + + // A whole-stream or whole-status assertion names NO path. Contributing the empty + // path instead would match every signature on the channel and turn the suggestion + // into "the first case that ever asserted on this channel". + for whole in [ + json!({ "equals": 0 }), + json!({ "nonZero": true }), + json!({ "contains": "something" }), + json!({ "matches": "^.*$" }), + ] { + assert!(assertion_paths(&whole).is_empty(), "{whole} named a path"); + } + assert!(assertion_paths(&json!("not an object")).is_empty()); + } + + #[test] + fn a_suggestion_names_only_a_behavior_the_committed_registry_resolves() { + // The FR-025 guard, against the REAL registry: whatever the rule proposes, every id + // it emits must resolve. An invented id would make the reviewer's job verifying a + // plausible-looking identity rather than deciding one. + let registry = Registry::load(&crate::conformance_registry_root()) + .expect("the committed registry loads"); + assert!( + !registry.behaviors.is_empty(), + "an empty behavior set would make every assertion below vacuous" + ); + + // A path the registry demonstrably reasons about (`case-readconfig-*` assert on it). + let known = signature(CHAN_STRUCTURED_OUTPUT, "configuration.name"); + let suggestions = suggest(®istry, &known); + assert!( + !suggestions.is_empty(), + "the committed registry declares assertions at `configuration.name`; finding \ + none means the reader stopped seeing them and every candidate would report \ + `match: none` regardless of what the registry knows" + ); + for suggestion in &suggestions { + assert!( + registry + .behaviors + .iter() + .any(|b| b.id == suggestion.behavior), + "`{}` does not resolve in the registry", + suggestion.behavior + ); + assert!(registry.cases.iter().any(|c| c.id == suggestion.case)); + } + + // A path nothing declares yields nothing — and the mapping then says so explicitly + // rather than reaching for the nearest plausible id. + let novel = signature( + CHAN_STRUCTURED_OUTPUT, + "configuration.thisFieldDoesNotExist", + ); + assert!(suggest(®istry, &novel).is_empty()); + + // Scoped to its channel: the same path on a different channel is a different + // observable location. + assert!( + suggest(®istry, &signature(CHAN_EXIT_CODE, "configuration.name")).is_empty(), + "a suggestion must be scoped to the channel the assertion was declared on" + ); + } + + #[test] + fn suggestions_are_deduplicated_and_ordered() { + let registry = Registry::load(&crate::conformance_registry_root()).expect("loads"); + let suggestions = suggest( + ®istry, + &signature(CHAN_STRUCTURED_OUTPUT, "configuration"), + ); + let ids: Vec = suggestions.iter().map(|s| s.behavior.clone()).collect(); + let mut sorted = ids.clone(); + sorted.sort(); + assert_eq!(ids, sorted, "suggestions are emitted in a stable order"); + let mut deduped = sorted.clone(); + deduped.dedup(); + assert_eq!(deduped, sorted, "a behavior is suggested at most once"); + } +} diff --git a/crates/parity-harness/src/discovery/corpus_fetch.rs b/crates/parity-harness/src/discovery/corpus_fetch.rs new file mode 100644 index 00000000..cf8b5fc3 --- /dev/null +++ b/crates/parity-harness/src/discovery/corpus_fetch.rs @@ -0,0 +1,649 @@ +//! Network-lane corpus fetch with content-digest verification +//! (025-exploratory-parity-discovery, US7, T107). +//! +//! The only network-touching code in the feature. A digest is recorded on **first** +//! materialization and verified on every later fetch (FR-051); a mismatch fails that +//! entry loudly rather than comparing against unexpected content, and an unreachable +//! entry is reported as unreachable rather than as "ran and found nothing" (FR-052). +//! +//! ## Why `git`, and why a partial clone +//! +//! The retired Python fetcher walked GitHub's contents API one file at a time through +//! `gh`, which needs an authenticated CLI and burns a rate-limit budget that 33 entries +//! exhaust immediately. This uses `git` directly with a **blob-filtered partial clone plus +//! a sparse checkout**, so one network round trip per entry materializes only the +//! devcontainer subtree — `microsoft/vscode` costs the same as `vscode-remote-try-node`. +//! No API token, no rate limit, and `git` is already a prerequisite of working in this +//! repository at all. +//! +//! Only `/.devcontainer/**` and `/.devcontainer.json` are materialized. That +//! is not a shortcut: the corpus tier is a **configuration-resolution** differential +//! (research D10), so an entry's application sources take part in nothing being compared, +//! and fetching them would trade the tier's budget for bytes neither implementation reads. +//! +//! ## Three outcomes, never two +//! +//! [`EntryStatus`] deliberately has no "failed" catch-all. `Unreachable` and +//! `DigestMismatch` are different facts — the first says the snapshot could not be +//! retrieved, the second says it was retrieved and is not what was recorded — and +//! collapsing them would let content drift at a pinned commit read as a flaky network. +//! +//! A genuine *machinery* failure (an unwritable destination) is an `Err`, because it is a +//! statement about the run rather than about an entry. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use deacon_conformance::discovery::corpus::{self, CorpusEntry, digest_of}; + +use crate::HarnessError; + +/// Path override for the `git` binary — the fault-injection seam. +/// +/// Public and explicit for the same reason [`crate::prereq::probe_docker`] is: the +/// alternative is a test calling `std::env::set_var`, which is `unsafe` under this +/// workspace's edition (and `unsafe_code = "deny"` forbids), besides being process-global +/// and hostile to a parallel runner. +pub const GIT_OVERRIDE_ENV: &str = "DEACON_DISCOVERY_GIT"; + +/// Bound on the `git version` prerequisite probe. +pub const GIT_PROBE_BOUND: Duration = Duration::from_secs(60); + +/// Bound on one entry's network fetch. Generous — a cold partial clone of a large +/// repository is still a single round trip, and a bound tight enough to trip on a slow +/// runner would report a healthy entry as unreachable. +pub const DEFAULT_ENTRY_BOUND: Duration = Duration::from_secs(300); + +/// The `git` binary this run uses. +pub fn git_binary() -> PathBuf { + std::env::var_os(GIT_OVERRIDE_ENV) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("git")) +} + +/// Require a working `git`. Any failure is [`HarnessError::NetworkUnavailable`]. +pub async fn require_git() -> Result<(), HarnessError> { + probe_git(&git_binary(), GIT_PROBE_BOUND).await +} + +/// Probe a specific `git` binary. Pure over its inputs so fault injection can point it at +/// a failing stub without touching process environment. +pub async fn probe_git(bin: &Path, bound: Duration) -> Result<(), HarnessError> { + let mut cmd = tokio::process::Command::new(bin); + cmd.arg("version") + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .kill_on_drop(true); + + match tokio::time::timeout(bound, cmd.status()).await { + Ok(Ok(status)) if status.success() => Ok(()), + Ok(Ok(status)) => Err(HarnessError::NetworkUnavailable { + cause: format!("`{} version` exited with {status}", bin.display()), + }), + Ok(Err(e)) => Err(HarnessError::NetworkUnavailable { + cause: format!("could not run `{} version`: {e}", bin.display()), + }), + Err(_elapsed) => Err(HarnessError::NetworkUnavailable { + cause: format!( + "`{} version` did not answer within {bound:?}", + bin.display() + ), + }), + } +} + +/// One entry that was retrieved and whose digest is settled. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Materialized { + /// The entry's `cor-` id. + pub entry_id: String, + /// The entry's human name. + pub name: String, + /// The materialized workspace root — what both implementations are pointed at. + pub workspace: PathBuf, + /// The digest computed over the materialized content, `sha256:<64-hex>`. + pub digest: String, + /// **True only on first materialization**, when the manifest carried `null` and this + /// run recorded the digest. Every later fetch verifies instead, so a `true` here on a + /// second run means the digest was removed — the second clause of **D4**. + pub recorded: bool, +} + +/// What happened to one corpus entry. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum EntryStatus { + /// Retrieved, and its digest either verified or recorded for the first time. + Materialized(Materialized), + /// **FR-051** — retrieved, but the content digest disagrees with the recorded one. + /// The entry is not compared: comparing against unexpected content would attribute a + /// difference in the *upstream workspace* to a difference between the implementations. + DigestMismatch { + entry_id: String, + name: String, + expected: String, + actual: String, + }, + /// **FR-052** — the snapshot could not be retrieved. Distinguished from an entry that + /// ran and produced no finding, because the two are opposites: one says nothing was + /// compared, the other says everything compared agreed. + Unreachable { + entry_id: String, + name: String, + cause: String, + }, +} + +impl EntryStatus { + /// The entry this status is about. + pub fn entry_id(&self) -> &str { + match self { + EntryStatus::Materialized(m) => &m.entry_id, + EntryStatus::DigestMismatch { entry_id, .. } + | EntryStatus::Unreachable { entry_id, .. } => entry_id, + } + } + + /// A one-line rendering for a campaign log or report. + pub fn summary(&self) -> String { + match self { + EntryStatus::Materialized(m) => format!( + "{} ({}): {} {}", + m.name, + m.entry_id, + if m.recorded { "recorded" } else { "verified" }, + m.digest + ), + EntryStatus::DigestMismatch { + entry_id, + name, + expected, + actual, + } => format!( + "{name} ({entry_id}): DIGEST MISMATCH — recorded {expected}, materialized \ + {actual}" + ), + EntryStatus::Unreachable { + entry_id, + name, + cause, + } => format!("{name} ({entry_id}): UNREACHABLE — {cause}"), + } + } +} + +/// Materialize one corpus entry under `root` and settle its digest. +/// +/// `Err` is reserved for machinery failures — a destination that cannot be created is a +/// statement about the run, not about the entry. Everything an entry can do wrong comes +/// back as an [`EntryStatus`]. +pub async fn materialize( + entry: &CorpusEntry, + root: &Path, + bound: Duration, +) -> Result { + // A mutable reference never reaches the network. **D4** already rejects it + // hermetically, but the fetch refuses it too: this function is reachable from a caller + // holding a hand-built entry, and fetching a branch would record a digest over content + // that is different tomorrow — a verification that quietly means nothing. + if !corpus::is_immutable_reference(&entry.commit) { + return Ok(EntryStatus::Unreachable { + entry_id: entry.id.clone(), + name: entry.name.clone(), + cause: format!( + "`{}` is not a 40-character lowercase-hex object name, so it names \ + different content over time and is never fetched", + entry.commit + ), + }); + } + + let workspace = entry.workspace_dir(root); + if workspace.exists() { + std::fs::remove_dir_all(&workspace).map_err(|e| HarnessError::Report { + cause: format!("could not clear {}: {e}", workspace.display()), + })?; + } + std::fs::create_dir_all(&workspace).map_err(|e| HarnessError::Report { + cause: format!("could not create {}: {e}", workspace.display()), + })?; + + if let Err(cause) = clone_sparse(entry, &workspace, bound).await { + return Ok(EntryStatus::Unreachable { + entry_id: entry.id.clone(), + name: entry.name.clone(), + cause, + }); + } + + let files = collect_content(&workspace).map_err(|e| HarnessError::Report { + cause: format!( + "could not read the materialized workspace {}: {e}", + workspace.display() + ), + })?; + if files.is_empty() { + // The commit resolved and the checkout succeeded, but the pinned path holds no + // devcontainer configuration. That is "the workspace this entry names is not + // there", which is unreachability — NOT an agreement between two implementations + // that were never invoked. + return Ok(EntryStatus::Unreachable { + entry_id: entry.id.clone(), + name: entry.name.clone(), + cause: format!( + "the pinned path `{}` carries no devcontainer configuration at commit {}", + if entry.path.is_empty() { + "" + } else { + &entry.path + }, + entry.commit + ), + }); + } + + let digest = digest_of(&files); + match &entry.content_digest { + None => Ok(EntryStatus::Materialized(Materialized { + entry_id: entry.id.clone(), + name: entry.name.clone(), + workspace, + digest, + recorded: true, + })), + Some(expected) if *expected == digest => Ok(EntryStatus::Materialized(Materialized { + entry_id: entry.id.clone(), + name: entry.name.clone(), + workspace, + digest, + recorded: false, + })), + Some(expected) => Ok(EntryStatus::DigestMismatch { + entry_id: entry.id.clone(), + name: entry.name.clone(), + expected: expected.clone(), + actual: digest, + }), + } +} + +/// Fold the digests a run settled back into the manifest, and write it. +/// +/// Only **first** materializations change anything: a verified digest is already what the +/// file says, and a mismatched one is deliberately not written (FR-051 — the recorded +/// digest is the claim under test, so overwriting it with whatever was fetched would turn +/// every mismatch into a silent re-baseline). +/// +/// The write is refused outright if it would drop a digest the committed manifest already +/// carries — **D4**'s second clause, checked here because this is the one caller that +/// holds both the baseline and the successor (see [`corpus::check_drift`]). +pub fn record_digests( + discovery_dir: &Path, + committed: &[CorpusEntry], + statuses: &[EntryStatus], +) -> Result, HarnessError> { + let recorded: BTreeMap<&str, &str> = statuses + .iter() + .filter_map(|s| match s { + EntryStatus::Materialized(m) if m.recorded => { + Some((m.entry_id.as_str(), m.digest.as_str())) + } + _ => None, + }) + .collect(); + if recorded.is_empty() { + return Ok(Vec::new()); + } + + let mut next = committed.to_vec(); + let mut written = Vec::new(); + for entry in &mut next { + if let Some(digest) = recorded.get(entry.id.as_str()) { + entry.content_digest = Some((*digest).to_string()); + written.push(entry.id.clone()); + } + } + + let drift = corpus::check_drift(committed, &next); + if !drift.is_empty() { + return Err(HarnessError::Report { + cause: format!( + "refusing to write the corpus manifest: {}", + drift + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("; ") + ), + }); + } + + corpus::write(discovery_dir, &next).map_err(|e| HarnessError::Report { + cause: format!("could not write the corpus manifest: {e}"), + })?; + Ok(written) +} + +// --------------------------------------------------------------------------- +// git +// --------------------------------------------------------------------------- + +/// The sparse-checkout patterns for one entry: the devcontainer subtree, and nothing else. +/// +/// `/` anchors each pattern at the repository root, so a `vendor/src/go/.devcontainer/` +/// buried in a monorepo cannot be matched by an entry pinned at `src/go`. +fn sparse_patterns(path: &str) -> Vec { + let prefix = path.trim_matches('/'); + if prefix.is_empty() { + vec![ + "/.devcontainer/".to_string(), + "/.devcontainer.json".to_string(), + ] + } else { + vec![ + format!("/{prefix}/.devcontainer/"), + format!("/{prefix}/.devcontainer.json"), + ] + } +} + +/// Materialize the entry's devcontainer subtree into `dest`. +/// +/// `Err(String)` is the *cause of unreachability*, never a run failure: every step depends +/// on a third party being up and a pin still resolving, and neither is something this +/// repository controls. +async fn clone_sparse(entry: &CorpusEntry, dest: &Path, bound: Duration) -> Result<(), String> { + let git = git_binary(); + let url = format!("https://github.com/{}.git", entry.repository); + + run_git(&git, dest, &["init", "--quiet"], bound).await?; + run_git(&git, dest, &["remote", "add", "origin", &url], bound).await?; + run_git( + &git, + dest, + &["config", "core.sparseCheckout", "true"], + bound, + ) + .await?; + + let mut patterns = sparse_patterns(&entry.path).join("\n"); + patterns.push('\n'); + let sparse_file = dest.join(".git").join("info").join("sparse-checkout"); + if let Some(parent) = sparse_file.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| format!("could not create {}: {e}", parent.display()))?; + } + std::fs::write(&sparse_file, patterns) + .map_err(|e| format!("could not write {}: {e}", sparse_file.display()))?; + + // `--filter=blob:none` keeps the fetch to commits and trees; the checkout below then + // pulls only the blobs the sparse patterns select. `--depth 1` on an explicit object + // name is what makes a monorepo cost the same as a sample repository. + run_git( + &git, + dest, + &[ + "fetch", + "--quiet", + "--depth", + "1", + "--filter=blob:none", + "origin", + &entry.commit, + ], + bound, + ) + .await?; + run_git(&git, dest, &["checkout", "--quiet", "FETCH_HEAD"], bound).await?; + Ok(()) +} + +async fn run_git(git: &Path, cwd: &Path, args: &[&str], bound: Duration) -> Result<(), String> { + let mut cmd = tokio::process::Command::new(git); + cmd.args(args) + .current_dir(cwd) + // A credential prompt would hang the campaign on a private or renamed repository + // until the bound expired, and report an auth problem as a timeout. + .env("GIT_TERMINAL_PROMPT", "0") + .env("GIT_ASKPASS", "") + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .kill_on_drop(true); + + let rendered = args.join(" "); + match tokio::time::timeout(bound, cmd.output()).await { + Ok(Ok(output)) if output.status.success() => Ok(()), + Ok(Ok(output)) => Err(format!( + "`git {rendered}` exited with {}: {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + )), + Ok(Err(e)) => Err(format!("could not run `git {rendered}`: {e}")), + Err(_elapsed) => Err(format!("`git {rendered}` exceeded {bound:?}")), + } +} + +// --------------------------------------------------------------------------- +// digest +// --------------------------------------------------------------------------- + +/// One materialized tree, keyed by workspace-relative POSIX path. +type Content = BTreeMap>; + +/// Collect the materialized content, excluding `.git/`. +/// +/// A `BTreeMap` rather than the walk order: the digest must not depend on the order a +/// filesystem happens to enumerate directories in, or the same snapshot would digest +/// differently on two machines and every verification would fail as a mismatch. +fn collect_content(root: &Path) -> std::io::Result { + let mut out = Content::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + for entry in std::fs::read_dir(&dir)? { + let entry = entry?; + let path = entry.path(); + if path.file_name().is_some_and(|n| n == ".git") { + continue; + } + // `symlink_metadata`, not `metadata`: a symlink is digested as the link it is. + // Following it would either digest the same bytes twice or escape the + // materialized tree entirely. + let meta = std::fs::symlink_metadata(&path)?; + if meta.is_dir() { + stack.push(path); + continue; + } + let Ok(relative) = path.strip_prefix(root) else { + continue; + }; + let key = relative + .components() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/"); + let bytes = if meta.is_symlink() { + let target = std::fs::read_link(&path)?; + let mut marked = b"symlink:".to_vec(); + marked.extend_from_slice(target.to_string_lossy().as_bytes()); + marked + } else { + std::fs::read(&path)? + }; + out.insert(key, bytes); + } + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use deacon_conformance::discovery::corpus::DIGEST_PREFIX; + + fn entry(path: &str, digest: Option<&str>) -> CorpusEntry { + CorpusEntry { + id: CorpusEntry::derive_id( + "devcontainers/images", + "31b61b521d55926d62c748b659f24ae71774c0e3", + path, + ), + name: format!("images-{}", path.replace('/', "-")), + repository: "devcontainers/images".to_string(), + commit: "31b61b521d55926d62c748b659f24ae71774c0e3".to_string(), + path: path.to_string(), + content_digest: digest.map(str::to_string), + notes: String::new(), + } + } + + #[test] + fn sparse_patterns_are_anchored_at_the_repository_root() { + assert_eq!( + sparse_patterns(""), + vec![ + "/.devcontainer/".to_string(), + "/.devcontainer.json".to_string() + ] + ); + assert_eq!( + sparse_patterns("src/go"), + vec![ + "/src/go/.devcontainer/".to_string(), + "/src/go/.devcontainer.json".to_string() + ] + ); + // A stray leading or trailing slash must not produce `//` or an unanchored + // pattern: an unanchored `src/go/.devcontainer/` would also match a nested + // `vendor/src/go/.devcontainer/` in a monorepo. + assert_eq!(sparse_patterns("/src/go/"), sparse_patterns("src/go")); + } + + #[test] + fn the_digest_is_well_formed_and_order_independent() { + let mut a = Content::new(); + a.insert( + ".devcontainer/devcontainer.json".to_string(), + b"{}".to_vec(), + ); + a.insert(".devcontainer/Dockerfile".to_string(), b"FROM x".to_vec()); + + let mut b = Content::new(); + b.insert(".devcontainer/Dockerfile".to_string(), b"FROM x".to_vec()); + b.insert( + ".devcontainer/devcontainer.json".to_string(), + b"{}".to_vec(), + ); + + let digest = digest_of(&a); + assert_eq!(digest, digest_of(&b), "insertion order must not matter"); + assert!( + corpus::is_well_formed_digest(&digest), + "{digest} must satisfy the manifest's own digest format" + ); + } + + #[test] + fn the_digest_distinguishes_content_a_naive_concatenation_would_merge() { + // The injectivity boundary case: the same bytes split differently across path and + // payload. Without length prefixes these two trees hash identically, and a + // verification that cannot tell them apart is one that cannot fail. + let mut left = Content::new(); + left.insert("ab".to_string(), b"c".to_vec()); + let mut right = Content::new(); + right.insert("a".to_string(), b"bc".to_vec()); + assert_ne!(digest_of(&left), digest_of(&right)); + + // And content changes must change the digest — the property FR-051 rests on. + let mut before = Content::new(); + before.insert( + ".devcontainer/devcontainer.json".to_string(), + b"{}".to_vec(), + ); + let mut after = Content::new(); + after.insert( + ".devcontainer/devcontainer.json".to_string(), + b"{\"image\":\"x\"}".to_vec(), + ); + assert_ne!(digest_of(&before), digest_of(&after)); + + // An empty tree has a digest too; it is never mistaken for a populated one. + assert_ne!(digest_of(&Content::new()), digest_of(&before)); + } + + #[tokio::test] + async fn a_mutable_reference_never_reaches_the_network() { + let mut e = entry("src/go", None); + e.commit = "main".to_string(); + let root = tempfile::tempdir().expect("tempdir"); + // No process is spawned at all: the refusal happens before `git` is invoked, + // which is what makes this assertion meaningful with no network available. + let status = materialize(&e, root.path(), Duration::from_secs(1)) + .await + .expect("a refusal is a status, not a run failure"); + match status { + EntryStatus::Unreachable { cause, .. } => { + assert!(cause.contains("40-character lowercase-hex"), "{cause}"); + } + other => panic!("expected Unreachable, got {other:?}"), + } + } + + #[test] + fn recording_only_writes_first_materializations() { + let dir = tempfile::tempdir().expect("tempdir"); + let committed = vec![entry("src/go", None), entry("src/rust", None)]; + corpus::write(dir.path(), &committed).expect("seed the manifest"); + + let digest = format!("{DIGEST_PREFIX}{}", "a".repeat(64)); + let statuses = vec![ + EntryStatus::Materialized(Materialized { + entry_id: committed[0].id.clone(), + name: committed[0].name.clone(), + workspace: dir.path().to_path_buf(), + digest: digest.clone(), + recorded: true, + }), + // A mismatch must never re-baseline: overwriting the recorded digest with + // whatever was fetched would make every FR-051 failure self-healing, and + // therefore invisible. + EntryStatus::DigestMismatch { + entry_id: committed[1].id.clone(), + name: committed[1].name.clone(), + expected: format!("{DIGEST_PREFIX}{}", "b".repeat(64)), + actual: format!("{DIGEST_PREFIX}{}", "c".repeat(64)), + }, + ]; + + let written = + record_digests(dir.path(), &committed, &statuses).expect("the write succeeds"); + assert_eq!(written, vec![committed[0].id.clone()]); + + let reloaded = deacon_conformance::discovery::queue::DiscoveryData::load(dir.path()) + .expect("the manifest reloads"); + assert_eq!( + reloaded.corpus[0].content_digest.as_deref(), + Some(digest.as_str()) + ); + assert_eq!(reloaded.corpus[1].content_digest, None); + assert!( + corpus::check(&reloaded.corpus).is_empty(), + "a recorded digest must leave the manifest D4-clean" + ); + } + + #[test] + fn nothing_to_record_writes_nothing() { + let dir = tempfile::tempdir().expect("tempdir"); + let committed = vec![entry("src/go", None)]; + // Deliberately does NOT seed the file: a run that recorded nothing must not + // create or rewrite the manifest at all. + let written = record_digests(dir.path(), &committed, &[]).expect("no-op"); + assert!(written.is_empty()); + assert!( + !deacon_conformance::discovery::queue::corpus_path(dir.path()).exists(), + "a no-op must not touch the manifest" + ); + } +} diff --git a/crates/parity-harness/src/discovery/differential.rs b/crates/parity-harness/src/discovery/differential.rs new file mode 100644 index 00000000..5854abf6 --- /dev/null +++ b/crates/parity-harness/src/discovery/differential.rs @@ -0,0 +1,872 @@ +//! Differential comparison of deacon against the verified pinned oracle over one +//! candidate (025-exploratory-parity-discovery, T034/T035/T036/T122, +//! FR-013 – FR-017). +//! +//! ## Nothing here is a new mechanism +//! +//! - **Execution** is [`crate::exec::run_and_capture`] — the same bounded invocation with +//! always-on raw capture every parity comparison already uses. A second execution path +//! would be a second set of bounds, captures, and failure modes. +//! - **Oracle resolution and exact-version verification** are [`crate::oracle`] and +//! [`crate::prereq`]. A missing or mismatched oracle fails loudly (FR-003). +//! - **Normalization** is [`crate::normalize`], applied through exactly the rule chain the +//! declarative `chan-structured-output` channel applies (FR-015). A signature computed +//! from independently re-diffed values would be a second opinion on what differs, able +//! to disagree with the one the comparison used — the identical defect class the +//! single-normalizer rule exists to prevent. +//! - **The signature** is derived from `normalize::diff`'s own [`ConfigDivergence`] +//! output by `deacon_conformance::discovery::signature` (T035): a field-for-field move, +//! never a recomputation. +//! +//! ## Outcomes and structured content, never message wording (FR-016, T122) +//! +//! The comparison relates two things and no others: whether each side **accepted or +//! rejected** the candidate, and — when both accepted — the **normalized structured +//! document**. Diagnostic prose is captured to disk for a reviewer and is never compared. +//! +//! Two rejections that differ only in wording therefore produce no finding, by +//! construction rather than by a filter that could be forgotten: there is no code path +//! that reads stderr into a comparison. The exit *class* is compared rather than the +//! numeric code for the same reason — deacon and the reference legitimately spell "I +//! refused this" with different non-zero codes, and treating that as a difference would +//! report the wording of a status rather than its meaning. +//! +//! ## Already-characterized differences never enter the queue as new (FR-017, T036) +//! +//! A difference the project has already recorded — through a case's scoped +//! `allowedDifferences`, or through a `wvr-` waiver — is reported as +//! [`Verdict::Characterized`] naming the record that covers it. It is still *observed* and +//! still counted; it simply is not news. Without this the queue would fill on every run +//! with the strictness family this repository has already characterized in full, and the +//! genuinely new finding would be the one nobody could see. +//! +//! Note the direction: discovery **reads** the tolerance records and never writes one. +//! FR-018 forbids a discovery program authoring an allowed difference, because that would +//! let a difference disappear by being observed. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use deacon_conformance::discovery::queue::ObservedValues; +use deacon_conformance::discovery::signature::{Divergence, DivergenceKind, Signature}; +use deacon_conformance::load::Registry; +use deacon_conformance::model::{CHAN_EXIT_CODE, CHAN_STRUCTURED_OUTPUT, Expect, Scope}; +use serde_json::Value; + +use crate::HarnessError; +use crate::exec::{Invocation, Side, run_and_capture}; +use crate::normalize::{self, ConfigDivergence, DiffKind, DocumentBlock}; +use crate::oracle::VerifiedOracle; + +/// Whether an implementation accepted or rejected a candidate. +/// +/// The *class*, not the numeric exit code: two implementations spell "I refused this" with +/// different non-zero codes, and comparing the numbers would report the wording of a +/// status rather than its meaning (FR-016). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OutcomeClass { + /// The CLI exited successfully. + Accepted, + /// The CLI exited non-zero, or was terminated. + Rejected, +} + +impl OutcomeClass { + /// The stable wire spelling, used as the compared value on `chan-exit-code`. + pub fn as_str(self) -> &'static str { + match self { + OutcomeClass::Accepted => "accepted", + OutcomeClass::Rejected => "rejected", + } + } + + fn of(invocation: &Invocation) -> OutcomeClass { + if invocation.success { + OutcomeClass::Accepted + } else { + OutcomeClass::Rejected + } + } +} + +/// What the comparison decided about one observed difference. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Verdict { + /// Not covered by any recorded case, waiver, or tolerated difference — a queue + /// candidate. + New, + /// Covered by the named record (a `wvr-`/`ext-`/`case-` id). Reported, counted, and + /// **never admitted to the queue as new** (FR-017). + Characterized(String), +} + +/// One observed difference: its signature, the concrete values behind it, and the verdict. +#[derive(Debug, Clone, PartialEq)] +pub struct Observation { + /// The deduplication key. + pub signature: Signature, + /// The concrete values each side produced — **evidence, not identity** (research D3). + pub observed: ObservedValues, + /// Whether this is news. + pub verdict: Verdict, +} + +impl Observation { + /// Whether this observation should be admitted to the findings queue. + pub fn is_new(&self) -> bool { + self.verdict == Verdict::New + } +} + +/// Where one side's raw evidence was preserved (FR-014). +/// +/// Raw and normalized are kept **separate**: the raw bytes live on disk exactly as the CLI +/// produced them, and the normalized document lives in memory alongside. Conflating them +/// would make it impossible to tell a difference the implementations produced from one the +/// normalizer produced — which is precisely the `normalizer-defect` classification the +/// triage vocabulary reserves. +#[derive(Debug, Clone, PartialEq)] +pub struct SideEvidence { + /// Accepted or rejected. + pub outcome: OutcomeClass, + /// The raw exit code, preserved but never compared. + pub exit_code: Option, + /// Absolute path to the verbatim stdout capture. + pub stdout_path: PathBuf, + /// Absolute path to the verbatim stderr capture. + pub stderr_path: PathBuf, + /// The normalized structured document, when the side produced one. + pub normalized: Option, +} + +/// The result of comparing one candidate. +#[derive(Debug, Clone, PartialEq)] +pub struct DifferentialResult { + /// The candidate that was compared. + pub candidate_id: String, + /// deacon's evidence. + pub deacon: SideEvidence, + /// The reference's evidence. + pub reference: SideEvidence, + /// **Neither** side reached configuration resolution — the SC-002 numerator. + /// + /// Defined as "no side produced a structured document", which is exactly the + /// complement of FR-007's "reach configuration resolution or a later stage". It is + /// deliberately *not* "deacon rejected": a rejection by one side while the other + /// resolves is a real difference, and counting it as a trivial failure would hide the + /// most interesting candidate the generator can produce. + pub parse_stage_failure: bool, + /// Every observed difference, in the diff's deterministic order. + pub observations: Vec, +} + +impl DifferentialResult { + /// The observations that are news. + pub fn new_observations(&self) -> impl Iterator { + self.observations.iter().filter(|o| o.is_new()) + } + + /// How many observations were already characterized. + pub fn characterized_count(&self) -> usize { + self.observations.iter().filter(|o| !o.is_new()).count() + } +} + +/// Everything one comparison needs. Bundled so the call site reads as the fact it is — +/// "compare this candidate, in this workspace, with these two binaries, under this bound". +#[derive(Debug, Clone, Copy)] +pub struct DifferentialInput<'a> { + /// The candidate's `cnd-` id, used as the raw-artifact case name. + pub candidate_id: &'a str, + /// The materialized workspace both sides are pointed at. + pub workspace: &'a Path, + /// The deacon binary under test. + pub deacon: &'a Path, + /// The **verified** pinned oracle. Taking the verified type rather than a path is what + /// makes "never compare against an unverified reference" (FR-003) a type-level fact + /// at this call site rather than a rule the caller has to remember. + pub oracle: &'a VerifiedOracle, + /// The per-candidate bound (60 s hermetic, 5 min container-backed). + pub bound: Duration, + /// Where raw artifacts are written. + pub report_root: &'a Path, + /// Whether the candidate was **deliberately** made invalid — a near-valid draw that + /// violates a `required` key, or a mutated document. + /// + /// This is what narrows the strictness characterization from a channel-wide ignore + /// list to a scoped one. deacon deliberately rejects malformed configuration the + /// reference accepts, and that family is characterized in full — but only *for + /// malformed input*. deacon refusing a candidate the grammar says is **valid** is a + /// `deacon-regression`, and it is exactly the finding this whole feature exists to + /// surface, so it must never be swallowed by the same waiver. + pub deliberately_invalid: bool, +} + +/// Compare one candidate across both implementations. +pub async fn compare( + input: DifferentialInput<'_>, + characterization: &Characterization, +) -> Result { + let workspace = input.workspace.to_string_lossy().into_owned(); + let args: Vec<&str> = vec!["read-configuration", "--workspace-folder", &workspace]; + + let deacon_run = run_and_capture( + Side::Deacon, + "discovery_campaign", + input.candidate_id, + input.deacon, + &args, + input.workspace, + input.bound, + input.report_root, + ) + .await?; + let oracle_run = run_and_capture( + Side::Oracle, + "discovery_campaign", + input.candidate_id, + &input.oracle.path, + &args, + input.workspace, + input.bound, + input.report_root, + ) + .await?; + + // Structured content is read only from a side that ACCEPTED. A rejecting CLI's stdout + // is diagnostic prose or nothing, and treating it as a document would smuggle message + // wording into the comparison through the back door (FR-016). + let deacon_doc = structured_document(&deacon_run, input.workspace, Side::Deacon); + let reference_doc = structured_document(&oracle_run, input.workspace, Side::Oracle); + + let deacon = SideEvidence { + outcome: OutcomeClass::of(&deacon_run), + exit_code: deacon_run.exit_code, + stdout_path: deacon_run.stdout_path(), + stderr_path: deacon_run.stderr_path(), + normalized: deacon_doc.clone(), + }; + let reference = SideEvidence { + outcome: OutcomeClass::of(&oracle_run), + exit_code: oracle_run.exit_code, + stdout_path: oracle_run.stdout_path(), + stderr_path: oracle_run.stderr_path(), + normalized: reference_doc.clone(), + }; + + Ok(result_from_sides( + input.candidate_id, + deacon, + reference, + characterization, + input.deliberately_invalid, + )) +} + +/// Relate two sides' evidence into a [`DifferentialResult`]. +/// +/// Extracted from [`compare`] rather than duplicated, and it is the **only** place a +/// comparison is defined. The pipeline proof (`super::pipeline_proof`) acquires its two +/// sides differently — one of them is perturbed at the sealed evidence-source boundary — +/// but it must relate them exactly as a live campaign does, or the proof would be asserting +/// that *a* comparison propagates a difference rather than that *this* one does. +pub(crate) fn result_from_sides( + candidate_id: &str, + deacon: SideEvidence, + reference: SideEvidence, + characterization: &Characterization, + deliberately_invalid: bool, +) -> DifferentialResult { + let observations = + observations_between(&deacon, &reference, characterization, deliberately_invalid); + DifferentialResult { + candidate_id: candidate_id.to_string(), + parse_stage_failure: deacon.normalized.is_none() && reference.normalized.is_none(), + deacon, + reference, + observations, + } +} + +/// Every difference between two sides' evidence, in the diff's deterministic order. +/// +/// Three branches, and the middle one is the reason this is not simply "diff the two +/// documents": a CLI that exits zero having emitted nothing parseable would otherwise be +/// reported as agreement, which is the quietest possible failure and exactly the kind this +/// feature exists to find. +fn observations_between( + deacon: &SideEvidence, + reference: &SideEvidence, + characterization: &Characterization, + deliberately_invalid: bool, +) -> Vec { + let mut observations = Vec::new(); + + // `chan-exit-code`: the OUTCOME CLASS, never the numeric code and never the message. + if deacon.outcome != reference.outcome { + let deacon_value = Value::String(deacon.outcome.as_str().to_string()); + let reference_value = Value::String(reference.outcome.as_str().to_string()); + let signature = Signature::derive( + CHAN_EXIT_CODE, + &Divergence { + kind: DivergenceKind::Value, + path: "outcome", + deacon: Some(&deacon_value), + reference: Some(&reference_value), + }, + ); + let stricter = if deacon.outcome == OutcomeClass::Rejected { + Stricter::Deacon + } else { + Stricter::Reference + }; + let verdict = characterization.verdict( + &signature, + ObservationContext { + deliberately_invalid, + stricter: Some(stricter), + }, + ); + observations.push(Observation { + signature, + observed: ObservedValues { + deacon: Some(deacon_value), + reference: Some(reference_value), + }, + verdict, + }); + } + + // One side succeeded and produced a structured document while the other succeeded and + // did not. Without this branch that difference is invisible: the outcome classes agree + // (both accepted), and the document comparison below needs both documents, so a CLI + // that exited zero having emitted nothing parseable would be reported as agreement. + // That is the quietest possible failure and exactly the kind this feature exists to + // find, so it is a difference in its own right. + if deacon.outcome == reference.outcome + && deacon.normalized.is_some() != reference.normalized.is_some() + { + let (kind, deacon_value, reference_value) = if deacon.normalized.is_some() { + ( + DivergenceKind::DeaconOnly, + Some(Value::String("structured document".to_string())), + None, + ) + } else { + ( + DivergenceKind::RefOnly, + None, + Some(Value::String("structured document".to_string())), + ) + }; + let signature = Signature::derive( + CHAN_STRUCTURED_OUTPUT, + &Divergence { + kind, + path: "document", + deacon: deacon_value.as_ref(), + reference: reference_value.as_ref(), + }, + ); + let verdict = characterization.verdict( + &signature, + ObservationContext { + deliberately_invalid, + stricter: None, + }, + ); + observations.push(Observation { + signature, + observed: ObservedValues { + deacon: deacon_value, + reference: reference_value, + }, + verdict, + }); + } + + // `chan-structured-output`: the normalized documents, compared only when BOTH sides + // produced one. Comparing a document against an absence would re-report the presence + // difference already recorded above, once per key. + if let (Some(d), Some(r)) = (&deacon.normalized, &reference.normalized) { + for divergence in normalize::diff(d, r) { + let signature = signature_of(CHAN_STRUCTURED_OUTPUT, &divergence); + let verdict = characterization.verdict( + &signature, + ObservationContext { + deliberately_invalid, + // Not an outcome divergence: both sides resolved, so neither was + // stricter than the other about accepting the input. + stricter: None, + }, + ); + observations.push(Observation { + signature, + observed: ObservedValues { + deacon: divergence.deacon.clone(), + reference: divergence.reference.clone(), + }, + verdict, + }); + } + } + + observations +} + +/// **T035** — derive a signature from `normalize::diff`'s own output. +/// +/// A field-for-field move, with no recomputation of what differs. `ConfigDivergence` and +/// `Divergence` carry the same four fields, and this is the single place the two +/// vocabularies meet — which is what keeps "derive only, never re-diff" true in the code +/// rather than only in a comment (research D3). +pub fn signature_of(channel: &str, divergence: &ConfigDivergence) -> Signature { + let kind = match divergence.kind { + DiffKind::RefOnly => DivergenceKind::RefOnly, + DiffKind::DeaconOnly => DivergenceKind::DeaconOnly, + DiffKind::Value => DivergenceKind::Value, + }; + Signature::derive( + channel, + &Divergence { + kind, + path: &divergence.path, + deacon: divergence.deacon.as_ref(), + reference: divergence.reference.as_ref(), + }, + ) +} + +/// The normalized structured document a side produced, or `None` when it produced none. +/// +/// Applies exactly the `chan-structured-output` rule chain from [`crate::normalize`] — +/// `path_token` then `config_document_rules`, in that order, because that is the chain +/// `normalize`'s own per-channel dispatch applies for this channel. Restating the rules +/// here would be the second normalization path FR-015 forbids; *calling* them in the +/// declared order is the reuse it requires. +fn structured_document(invocation: &Invocation, workspace: &Path, side: Side) -> Option { + structured_document_bytes(invocation.success, &invocation.stdout, workspace, side) +} + +/// The same normalization, over raw captured bytes rather than an [`Invocation`]. +/// +/// The pipeline proof needs this seam: its deacon side is the stdout of a real run **after** +/// a perturbation has been applied to it at the sealed evidence-source boundary, so there is +/// no `Invocation` carrying those bytes — and reconstructing one would mean inventing a +/// capture that never happened. Taking the bytes keeps the proof on the single normalization +/// chain FR-015 permits instead of growing a second one beside it. +pub(crate) fn structured_document_bytes( + success: bool, + stdout: &[u8], + workspace: &Path, + side: Side, +) -> Option { + if !success { + return None; + } + let text = String::from_utf8_lossy(stdout); + let raw: Value = serde_json::from_str(text.trim()).ok()?; + let tokens = normalize::tokens_for_channel(CHAN_STRUCTURED_OUTPUT, workspace); + Some(normalize::config_document_rules( + &normalize::path_token(&raw, &tokens), + side, + DocumentBlock::Wrapper, + )) +} + +// --------------------------------------------------------------------------- +// T036 — already-characterized suppression +// --------------------------------------------------------------------------- + +/// The project's recorded tolerances, indexed for lookup by signature (FR-017). +/// +/// Built from records the registry already owns. Discovery **reads** them and never writes +/// one: FR-018 forbids a discovery program authoring an allowed difference, because a +/// difference that could be tolerated by being observed would disappear precisely when it +/// was found. +#[derive(Debug, Clone, Default)] +pub struct Characterization { + /// `(channel, path prefix, covering id)` from every case's scoped + /// `allowedDifferences`. The `observablePath` is a dotted path **within** a channel, + /// never a bare channel — the registry rejects a bare-channel scope at load (V19), so + /// there is no global ignore list to inherit here. + scoped_paths: Vec<(String, String, String)>, + /// `(field pattern, waiver id)` from every `StateField`-scoped waiver. The pattern + /// supports an exact match or a trailing `*`, matched by the SAME + /// [`crate::waiver::field_matches`] the parity comparison uses. + field_patterns: Vec<(String, String)>, + /// `(which side is stricter, waiver id)` for every waiver characterizing an + /// accept/reject **strictness** divergence. + /// + /// deacon's deliberate strictness on malformed configuration is characterized in full + /// (`bhv-readconfig-*-rejected` and their waivers), and a near-valid generator + /// reproduces it constantly — so without this the queue would fill on every run with + /// the one divergence family the project has most thoroughly reviewed. + /// + /// It is deliberately **not** a channel-wide tolerance, which would be the global + /// ignore list the registry itself forbids (V19). Two conditions narrow it, and both + /// are load-bearing: + /// + /// 1. **The direction must match.** A `deacon-stricter` waiver says deacon refuses + /// what the reference accepts. It says nothing about deacon *accepting* what the + /// reference refuses, which is the opposite defect. + /// 2. **The candidate must be deliberately invalid.** These waivers characterize + /// strictness *on malformed input*. deacon refusing a candidate the grammar says is + /// valid is a `deacon-regression` — precisely the finding this feature exists to + /// surface — and swallowing it under a malformed-input waiver would make the + /// machinery quietest exactly where it should be loudest. + strictness_waivers: Vec<(Stricter, String)>, +} + +/// Which implementation refused an input the other accepted. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Stricter { + /// deacon rejected; the reference accepted. + Deacon, + /// The reference rejected; deacon accepted. + Reference, +} + +/// What the comparison knows about a candidate when it asks whether a difference is +/// already characterized. +/// +/// A separate parameter rather than fields on [`Signature`] on purpose: a signature is the +/// deduplication *identity*, and folding "was this input deliberately malformed?" into it +/// would split one defect across two signatures depending on how the candidate that +/// surfaced it happened to be produced. +#[derive(Debug, Clone, Copy, Default)] +pub struct ObservationContext { + /// Whether the candidate was deliberately made invalid (a near-valid draw or a mutated + /// document). + pub deliberately_invalid: bool, + /// For an outcome divergence, which side refused. `None` for every other channel. + pub stricter: Option, +} + +impl Characterization { + /// Build the index from a loaded registry. + pub fn from_registry(registry: &Registry) -> Characterization { + let mut scoped_paths = Vec::new(); + for case in ®istry.cases { + for allowed in &case.allowed_differences { + let Some((channel, path)) = allowed.observable_path.split_once('.') else { + // A bare channel is a global ignore list, which the registry rejects at + // load (V19). Skipping it here rather than treating it as a + // channel-wide tolerance means a record that somehow got through + // suppresses nothing, instead of silently suppressing everything. + continue; + }; + let covering = allowed + .waiver_id + .clone() + .or_else(|| allowed.divergence_id.clone()) + .unwrap_or_else(|| case.id.clone()); + scoped_paths.push((channel.to_string(), path.to_string(), covering)); + } + } + + let mut field_patterns = Vec::new(); + let mut strictness_waivers = Vec::new(); + for waiver in ®istry.waivers { + match &waiver.scope { + Scope::StateField { field, .. } => { + field_patterns.push((field.clone(), waiver.id.clone())); + } + Scope::CorpusCase { .. } => {} + } + match waiver.expect { + Expect::DeaconStricter { .. } => { + strictness_waivers.push((Stricter::Deacon, waiver.id.clone())); + } + Expect::ReferenceStricter { .. } => { + strictness_waivers.push((Stricter::Reference, waiver.id.clone())); + } + _ => {} + } + } + strictness_waivers.sort_by(|a, b| a.1.cmp(&b.1)); + strictness_waivers.dedup(); + + Characterization { + scoped_paths, + field_patterns, + strictness_waivers, + } + } + + /// The verdict for a signature: [`Verdict::Characterized`] naming the covering record, + /// or [`Verdict::New`]. + pub fn verdict(&self, signature: &Signature, context: ObservationContext) -> Verdict { + match self.covering(signature, context) { + Some(id) => Verdict::Characterized(id), + None => Verdict::New, + } + } + + /// The id of the record covering `signature`, if any. + pub fn covering(&self, signature: &Signature, context: ObservationContext) -> Option { + // A scoped allowed difference covers its exact path and everything beneath it — + // `chan-structured-output.configuration` covers `…configuration.remoteUser`. It + // does NOT cover a sibling whose name merely shares a prefix, which is why the + // match requires a `.` boundary rather than a bare `starts_with`. + for (channel, path, covering) in &self.scoped_paths { + if channel != &signature.channel { + continue; + } + if &signature.path == path || signature.path.starts_with(&format!("{path}.")) { + return Some(covering.clone()); + } + } + + for (pattern, waiver) in &self.field_patterns { + if crate::waiver::field_matches(&signature.path, pattern) { + return Some(waiver.clone()); + } + } + + // A strictness waiver characterizes deacon-vs-reference acceptance ON MALFORMED + // INPUT, in one direction. Both conditions must hold, or the tolerance would be a + // channel-wide ignore list wearing a waiver id. + if signature.channel == CHAN_EXIT_CODE + && context.deliberately_invalid + && let Some(stricter) = context.stricter + { + return self + .strictness_waivers + .iter() + .find(|(side, _)| *side == stricter) + .map(|(_, id)| id.clone()); + } + + None + } + + /// Whether anything at all is indexed — a campaign logs this, because an empty index + /// against a populated registry means every known divergence would be re-reported as + /// news. + pub fn is_empty(&self) -> bool { + self.scoped_paths.is_empty() + && self.field_patterns.is_empty() + && self.strictness_waivers.is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn signature(channel: &str, path: &str) -> Signature { + let d = json!("a"); + let r = json!("b"); + Signature::derive( + channel, + &Divergence { + kind: DivergenceKind::Value, + path, + deacon: Some(&d), + reference: Some(&r), + }, + ) + } + + #[test] + fn the_signature_is_derived_from_the_diffs_own_output() { + // T035: a field-for-field move. Re-deriving `kind`, `path`, or the value shape here + // would be a second opinion on what differs, able to disagree with the one the + // comparison used. + for (kind, expected) in [ + (DiffKind::RefOnly, DivergenceKind::RefOnly), + (DiffKind::DeaconOnly, DivergenceKind::DeaconOnly), + (DiffKind::Value, DivergenceKind::Value), + ] { + let divergence = ConfigDivergence { + kind, + path: "configuration.remoteUser".to_string(), + deacon: Some(json!("vscode")), + reference: Some(json!("root")), + }; + let sig = signature_of(CHAN_STRUCTURED_OUTPUT, &divergence); + assert_eq!(sig.kind, expected); + assert_eq!(sig.path, "configuration.remoteUser"); + assert_eq!(sig.channel, CHAN_STRUCTURED_OUTPUT); + assert_eq!(sig.derived_id(), sig.id); + } + } + + /// The context of an ordinary structured-output observation. + fn plain() -> ObservationContext { + ObservationContext::default() + } + + /// The context of an outcome divergence on a deliberately malformed candidate. + fn refused(stricter: Stricter) -> ObservationContext { + ObservationContext { + deliberately_invalid: true, + stricter: Some(stricter), + } + } + + #[test] + fn an_empty_characterization_calls_everything_new() { + let c = Characterization::default(); + assert!(c.is_empty()); + assert_eq!( + c.verdict( + &signature(CHAN_STRUCTURED_OUTPUT, "configuration.remoteUser"), + plain() + ), + Verdict::New + ); + assert_eq!( + c.verdict( + &signature(CHAN_EXIT_CODE, "outcome"), + refused(Stricter::Deacon) + ), + Verdict::New + ); + } + + #[test] + fn a_scoped_allowed_difference_covers_its_subtree_but_not_a_prefix_sibling() { + let c = Characterization { + scoped_paths: vec![( + CHAN_STRUCTURED_OUTPUT.to_string(), + "configuration.remoteEnv".to_string(), + "wvr-example".to_string(), + )], + ..Characterization::default() + }; + assert_eq!( + c.verdict( + &signature(CHAN_STRUCTURED_OUTPUT, "configuration.remoteEnv"), + plain() + ), + Verdict::Characterized("wvr-example".to_string()) + ); + assert_eq!( + c.verdict( + &signature(CHAN_STRUCTURED_OUTPUT, "configuration.remoteEnv.PATH"), + plain() + ), + Verdict::Characterized("wvr-example".to_string()) + ); + assert_eq!( + c.verdict( + &signature(CHAN_STRUCTURED_OUTPUT, "configuration.remoteEnvironment"), + plain() + ), + Verdict::New, + "a sibling that merely shares a textual prefix is a different path, and \ + tolerating it would silence a difference nobody reviewed" + ); + assert_eq!( + c.verdict( + &signature(CHAN_EXIT_CODE, "configuration.remoteEnv"), + plain() + ), + Verdict::New, + "a tolerance is scoped to its channel" + ); + } + + #[test] + fn a_state_field_waiver_pattern_covers_its_field() { + let c = Characterization { + field_patterns: vec![("mounts.*".to_string(), "wvr-mounts".to_string())], + ..Characterization::default() + }; + assert_eq!( + c.verdict( + &signature(CHAN_STRUCTURED_OUTPUT, "mounts.0.source"), + plain() + ), + Verdict::Characterized("wvr-mounts".to_string()) + ); + assert_eq!( + c.verdict(&signature(CHAN_STRUCTURED_OUTPUT, "remoteUser"), plain()), + Verdict::New + ); + } + + #[test] + fn a_strictness_waiver_covers_only_its_direction_on_deliberately_invalid_input() { + let c = Characterization { + strictness_waivers: vec![(Stricter::Deacon, "wvr-readconfig-strict".to_string())], + ..Characterization::default() + }; + let outcome = signature(CHAN_EXIT_CODE, "outcome"); + + assert_eq!( + c.verdict(&outcome, refused(Stricter::Deacon)), + Verdict::Characterized("wvr-readconfig-strict".to_string()), + "deacon's deliberate strictness on malformed input is characterized in full; a \ + near-valid generator reproduces it constantly and it is not news" + ); + + // The opposite direction is the opposite defect, and this waiver says nothing + // about it. + assert_eq!( + c.verdict(&outcome, refused(Stricter::Reference)), + Verdict::New, + "a `deacon-stricter` waiver does not characterize deacon ACCEPTING what the \ + reference refuses" + ); + + // The condition that matters most: deacon refusing a candidate the grammar says is + // VALID is a regression, and must not be swallowed by a malformed-input waiver. + assert_eq!( + c.verdict( + &outcome, + ObservationContext { + deliberately_invalid: false, + stricter: Some(Stricter::Deacon), + } + ), + Verdict::New, + "deacon rejecting a VALID candidate is the finding this whole feature exists to \ + surface — silencing it under a malformed-input waiver would make the machinery \ + quietest exactly where it should be loudest" + ); + + assert_eq!( + c.verdict( + &signature(CHAN_STRUCTURED_OUTPUT, "configuration.remoteUser"), + refused(Stricter::Deacon) + ), + Verdict::New, + "a strictness waiver says nothing about a value difference between two \ + successful resolutions" + ); + } + + #[test] + fn the_workspace_registry_indexes_real_tolerances() { + // A guard against the index silently going empty: an empty index against a + // populated registry would re-report every already-characterized divergence as + // news, which is the flood FR-017 exists to prevent. + let registry = Registry::load(&crate::conformance_registry_root()) + .expect("the committed registry loads"); + let c = Characterization::from_registry(®istry); + assert!( + !c.is_empty(), + "the committed registry records tolerances; an empty index means the reader \ + stopped seeing them" + ); + assert!( + !c.strictness_waivers.is_empty(), + "the read-configuration strictness family is characterized in the registry" + ); + } + + #[test] + fn the_outcome_class_is_the_comparison_not_the_exit_code() { + // FR-016 / T122 in miniature: the values compared on `chan-exit-code` are the two + // class names, so two rejections are equal whatever their numeric codes or + // messages. + assert_eq!(OutcomeClass::Accepted.as_str(), "accepted"); + assert_eq!(OutcomeClass::Rejected.as_str(), "rejected"); + assert_eq!(OutcomeClass::Rejected, OutcomeClass::Rejected); + assert_ne!(OutcomeClass::Accepted, OutcomeClass::Rejected); + } +} diff --git a/crates/parity-harness/src/discovery/metamorphic_run.rs b/crates/parity-harness/src/discovery/metamorphic_run.rs new file mode 100644 index 00000000..bab4dc2d --- /dev/null +++ b/crates/parity-harness/src/discovery/metamorphic_run.rs @@ -0,0 +1,1498 @@ +//! Deacon-only metamorphic relation evaluation +//! (025-exploratory-parity-discovery, US6, T095/T127). +//! +//! This is the only discovery tier that needs **neither** the oracle **nor** Docker **nor** +//! the network (research D12), which makes it the cheapest complete vertical slice through +//! generation → comparison → signature → candidate. It also catches what the differential +//! structurally cannot: if deacon and the reference are *consistently* wrong, the +//! differential is clean and the defect is invisible, whereas a sensitivity relation +//! asserts the result **must** change and so fails on consistent wrongness. +//! +//! ## What is data and what is code +//! +//! The **catalogue** is registry data — `conformance/registry/metamorphic.json`, validated +//! by V31/V32. The **transformations** are code, because "reindent this document" is not +//! expressible as data without inventing a rewriting language, and a rewriting language is +//! a second thing to get wrong. [`TRANSFORMATIONS`] is therefore a closed table keyed by +//! relation id, and [`evaluate_catalogue`] fails loudly with +//! [`HarnessError::RelationUnevaluable`] for a declared relation the table does not cover. +//! +//! That failure mode is the whole point. A relation the harness cannot apply reports +//! nothing, and *reporting nothing is byte-identical to holding*. SC-011 requires zero +//! inert relations, so an unimplemented relation must turn the run red naming itself, +//! never contribute a silent pass. +//! +//! ## The evidence document +//! +//! Each side is observed as one document: +//! +//! ```json +//! { "exitCode": 0, "structuredOutput": { "configuration": { … }, "workspace": { … } } } +//! ``` +//! +//! Both declared channel families are then paths within it — `chan-exit-code` at +//! `exitCode`, `chan-structured-output` under `structuredOutput` — so one comparison +//! covers both, and a run that *failed* is still comparable (its `structuredOutput` is +//! `null`, which is a value, not an absence of evidence). Normalization is +//! [`crate::normalize`] and nothing else: FR-015 permits exactly one normalization +//! definition, and a second one here could disagree with the one every other comparison +//! uses. +//! +//! ## Both sides are deacon +//! +//! [`crate::normalize::ConfigDivergence`]'s two sides are named `deacon` and `reference` +//! because it was built for the differential. Here both sides *are* deacon, so the mapping is fixed once +//! and stated: **`reference` is the original run, `deacon` is the transformed run**. That +//! makes the derived [`Signature`]'s `kind` read the way it should — `deacon-only` means +//! the transformation *introduced* something, `ref-only` means it *lost* something. +//! +//! ## The accounting, and why a residual is the interesting output +//! +//! An invariance relation holds when the two normalized documents are equal *after* the +//! relation's declared [`Accounting`] absorbs the differences it legitimately explains — +//! the workspace-path tokenization for relocation (FR-046), or the exact site the +//! transformation rewrote. Everything the accounting does not absorb is a **residual**, +//! and a residual is reported rather than tolerated. +//! +//! For `mrl-path-relocation` that residual is simultaneously a check on deacon and a check +//! on the normalizer: an absolute path the tokenizer missed surfaces here. Triage it +//! carefully — it is as likely to be a `normalizer-defect` as a `deacon-regression`, and +//! misfiling it as the latter sends someone to fix code that is correct. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +use deacon_conformance::discovery::metamorphic::{MetamorphicRelation, RelationEffect}; +use deacon_conformance::discovery::signature::{Divergence, DivergenceKind, Signature}; + +use crate::HarnessError; +use crate::exec::{ExecKind, Invocation, Side, run_and_capture}; +use crate::normalize::{self, DiffKind, DocumentBlock, TokenMap}; + +/// Where the configuration document lives inside a [`Fixture`]. +/// +/// One of the two spec discovery locations. A plain `devcontainer.json` at the workspace +/// root is NOT one, and a fixture that put it there would fail to resolve for a reason +/// that has nothing to do with the relation under test. +pub const CONFIG_PATH: &str = ".devcontainer/devcontainer.json"; + +/// The workspace directory name every materialized fixture uses. +/// +/// Fixed, and deliberately shared by both sides of a relocation: deacon derives +/// container-side paths (and `${localWorkspaceFolderBasename}`) from the workspace +/// directory's basename, so two differently-*named* directories would differ for a reason +/// that is not relocation. Holding the basename constant makes the relocation a pure +/// change of absolute path, which is what the relation asserts about. +pub const WORKSPACE_DIR_NAME: &str = "metamorphic-workspace"; + +// --------------------------------------------------------------------------- +// Fixture +// --------------------------------------------------------------------------- + +/// A workspace tree, held as authored **text** rather than parsed values. +/// +/// Text, because three of the seven relations transform things a parsed value cannot +/// represent — indentation, comments, and member order are all lost the moment a document +/// becomes a `serde_json::Value`. A fixture that round-tripped through a value could not +/// express the transformation whose invariance it is meant to test. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct Fixture { + /// Workspace-relative path → file contents, in sorted order for deterministic + /// materialization and byte-stable candidate rendering. + files: BTreeMap, +} + +impl Fixture { + /// An empty fixture. + pub fn new() -> Fixture { + Fixture::default() + } + + /// A fixture whose only file is the configuration document. + pub fn with_config(config: &str) -> Fixture { + Fixture::new().with_file(CONFIG_PATH, config) + } + + /// Add (or replace) one file. + pub fn with_file(mut self, rel: &str, contents: &str) -> Fixture { + self.files.insert(rel.to_string(), contents.to_string()); + self + } + + /// Remove one file, if present. + pub fn without_file(mut self, rel: &str) -> Fixture { + self.files.remove(rel); + self + } + + /// The files, workspace-relative path → contents. + pub fn files(&self) -> &BTreeMap { + &self.files + } + + /// The authored configuration document. + /// + /// A fixture with no configuration at [`CONFIG_PATH`] is a fixture defect, not a + /// relation failure, so it is an error rather than an empty string. + pub fn config(&self, relation: &str) -> Result<&str, HarnessError> { + self.files + .get(CONFIG_PATH) + .map(String::as_str) + .ok_or_else(|| HarnessError::RelationUnevaluable { + relation: relation.to_string(), + cause: format!("the base fixture has no `{CONFIG_PATH}`"), + }) + } + + /// Write the fixture into `dir`, removing anything already there. + /// + /// The wipe matters: a non-relocating relation materializes both sides into the SAME + /// directory, and `mrl-extends-flattening`'s transformed side *removes* a file. Writing + /// over the previous tree without clearing it would leave the parent document behind, + /// and the relation would compare a chain against a chain. + pub fn materialize(&self, dir: &Path) -> Result<(), HarnessError> { + if dir.exists() { + std::fs::remove_dir_all(dir).map_err(|e| HarnessError::Report { + cause: format!("could not clear workspace {dir:?}: {e}"), + })?; + } + for (rel, contents) in &self.files { + let path = dir.join(rel); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|e| HarnessError::Report { + cause: format!("could not create {parent:?}: {e}"), + })?; + } + std::fs::write(&path, contents).map_err(|e| HarnessError::Report { + cause: format!("could not write {path:?}: {e}"), + })?; + } + Ok(()) + } +} + +// --------------------------------------------------------------------------- +// The closed transformation table +// --------------------------------------------------------------------------- + +/// How a relation reconciles the two normalized documents before deciding whether it holds. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Accounting { + /// Nothing is explained away: the documents must be equal outright. + Identity, + /// The workspace moved, so compare modulo the declared `` tokenization and + /// report whatever the tokenization does not account for (FR-046). + PathTokenization, + /// The transformation rewrote exactly these observable paths (and their descendants). + /// A difference confined to them is the transformation itself; anything else is + /// residual. + /// + /// This is a **scoped** accounting, not an ignore list: the paths are named, finite, + /// and specific to one relation, in the same spirit as the scoped allowed-differences + /// of 022 (a bare channel would be a global ignore and is exactly what FR-032 forbids + /// there). + TransformedSites(&'static [&'static str]), +} + +/// What a transformation does to the fixture. +pub enum TransformKind { + /// Rewrite the authored configuration text. + RewriteConfigText(fn(&str) -> Result), + /// Rewrite the whole fixture (adding or removing files). + RewriteFixture(fn(&Fixture) -> Result), + /// Leave the fixture alone and materialize it at a different absolute path. + Relocate, +} + +impl std::fmt::Debug for TransformKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TransformKind::RewriteConfigText(_) => f.write_str("RewriteConfigText"), + TransformKind::RewriteFixture(_) => f.write_str("RewriteFixture"), + TransformKind::Relocate => f.write_str("Relocate"), + } + } +} + +/// Which part of deacon's output a relation is asserted over. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ObservedScope { + /// The whole CLI document. + WholeDocument, + /// One named top-level block of it (e.g. `mergedConfiguration`). + /// + /// `mrl-extends-flattening` needs this: the `configuration` block is an **echo** of the + /// authored document, and a chain and its flattening are authored differently by + /// construction. The claim the relation makes is about the *resolved* configuration, so + /// that is the block it compares. + Block(&'static str), +} + +/// Where an invariance relation's deliberate break is applied (see [`Sabotage`]). +/// +/// The break must land on **exactly one** side and must survive to the output, and which +/// order achieves that depends on the transformation — so it is declared per relation +/// rather than guessed. Guessing would mean a silent fallback in the one place a silent +/// fallback is least affordable: a break that quietly failed to land would leave the +/// relation holding, and a relation that holds under its own break reads as a healthy +/// relation rather than an inert one. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BreakPoint { + /// Perturb the base first, then transform it. + /// + /// Required when the transformation *emits* something the perturbation could not then + /// be applied to — `mrl-comment-invariance` produces JSONC, and re-parsing that as + /// strict JSON to insert a change would fail. + BeforeTransform, + /// Transform first, then perturb the result. + /// + /// The default, and required when the transformation does not carry its input's + /// content through — `mrl-extends-flattening` emits the hand-written flattening, so a + /// perturbation applied to its input would simply be discarded and the two sides would + /// still agree. + AfterTransform, +} + +/// One relation's executable half. +pub struct Transformation { + /// The `mrl-` id this implements. + pub relation: &'static str, + /// The base workspace the relation is evaluated over. + pub base: fn() -> Fixture, + /// Extra argv appended after `read-configuration --workspace-folder `. + pub argv: &'static [&'static str], + /// Which part of the output is compared. + pub scope: ObservedScope, + /// What is applied to the input. + pub kind: TransformKind, + /// How the two results are reconciled. + pub accounting: Accounting, + /// Where an invariance break is applied. + pub break_point: BreakPoint, +} + +impl Transformation { + /// Whether this transformation materializes its two sides at different absolute paths. + pub fn relocates(&self) -> bool { + matches!(self.kind, TransformKind::Relocate) + } + + /// Apply the transformation to `base`. + pub fn apply(&self, base: &Fixture) -> Result { + match &self.kind { + TransformKind::RewriteConfigText(f) => { + let rewritten = f(base.config(self.relation)?)?; + Ok(base.clone().with_file(CONFIG_PATH, &rewritten)) + } + TransformKind::RewriteFixture(f) => f(base), + TransformKind::Relocate => Ok(base.clone()), + } + } +} + +/// The closed transformation table — one entry per mandated relation family (FR-044). +/// +/// Closed on purpose. Adding a relation to `metamorphic.json` without adding its +/// transformation here is caught by [`evaluate_catalogue`] as +/// [`HarnessError::RelationUnevaluable`], because the alternative — skipping it — would +/// make a declared relation contribute a silent pass. +pub const TRANSFORMATIONS: &[Transformation] = &[ + Transformation { + relation: "mrl-formatting-invariance", + base: rich_base_fixture, + argv: &[], + scope: ObservedScope::WholeDocument, + kind: TransformKind::RewriteConfigText(reindent), + accounting: Accounting::Identity, + break_point: BreakPoint::AfterTransform, + }, + Transformation { + relation: "mrl-comment-invariance", + base: rich_base_fixture, + argv: &[], + scope: ObservedScope::WholeDocument, + kind: TransformKind::RewriteConfigText(insert_comments_and_trailing_commas), + accounting: Accounting::Identity, + break_point: BreakPoint::BeforeTransform, + }, + Transformation { + relation: "mrl-key-order-invariance", + base: rich_base_fixture, + argv: &[], + scope: ObservedScope::WholeDocument, + kind: TransformKind::RewriteConfigText(reverse_object_members), + accounting: Accounting::Identity, + break_point: BreakPoint::AfterTransform, + }, + Transformation { + relation: "mrl-path-relocation", + base: rich_base_fixture, + argv: &[], + scope: ObservedScope::WholeDocument, + kind: TransformKind::Relocate, + accounting: Accounting::PathTokenization, + break_point: BreakPoint::AfterTransform, + }, + Transformation { + relation: "mrl-lifecycle-equivalence", + base: rich_base_fixture, + argv: &[], + scope: ObservedScope::WholeDocument, + kind: TransformKind::RewriteConfigText(lifecycle_to_named_object), + accounting: Accounting::TransformedSites(&[ + "structuredOutput.configuration.postCreateCommand", + ]), + break_point: BreakPoint::AfterTransform, + }, + Transformation { + relation: "mrl-extends-flattening", + base: extends_chain_fixture, + argv: &["--include-merged-configuration"], + scope: ObservedScope::Block("mergedConfiguration"), + kind: TransformKind::RewriteFixture(flatten_extends_chain), + accounting: Accounting::Identity, + break_point: BreakPoint::AfterTransform, + }, + Transformation { + relation: "mrl-declaration-order-sensitivity", + base: compose_overlay_fixture, + argv: &[], + scope: ObservedScope::WholeDocument, + kind: TransformKind::RewriteConfigText(reverse_compose_overlay_list), + accounting: Accounting::Identity, + break_point: BreakPoint::AfterTransform, + }, +]; + +/// The transformation implementing `relation`, or `None`. +pub fn transformation_for(relation: &str) -> Option<&'static Transformation> { + TRANSFORMATIONS.iter().find(|t| t.relation == relation) +} + +// --------------------------------------------------------------------------- +// Base fixtures +// --------------------------------------------------------------------------- + +/// The configuration the four document-level relations are evaluated over. +/// +/// Deliberately rich in the shapes that make the relations non-vacuous: nested objects +/// (`customizations`), ordered arrays (`forwardPorts`, `runArgs`, +/// `overrideFeatureInstallOrder`), an unordered map with more than one member (`features`, +/// `containerEnv`), a lifecycle command in its bare string form, and a +/// `${localWorkspaceFolder}` substitution that puts a host absolute path into the result — +/// which is what gives `mrl-path-relocation` something to be invariant *about*. +/// +/// The Feature ids are echoed, never resolved: `read-configuration` without +/// `--include-features-configuration` performs no fetch, so this tier stays hermetic. +fn rich_base_fixture() -> Fixture { + Fixture::with_config( + r#"{ + "name": "metamorphic-base", + "image": "alpine:3.18", + "workspaceFolder": "/workspace", + "features": { + "ghcr.io/devcontainers/features/common-utils:2": {}, + "ghcr.io/devcontainers/features/git:1": { "version": "latest" } + }, + "overrideFeatureInstallOrder": [ + "ghcr.io/devcontainers/features/common-utils", + "ghcr.io/devcontainers/features/git" + ], + "containerEnv": { "ZULU": "z", "ALPHA": "${localWorkspaceFolder}" }, + "forwardPorts": [3000, 3001], + "runArgs": ["--cap-add", "SYS_PTRACE"], + "postCreateCommand": "echo hi", + "customizations": { "vscode": { "settings": { "editor.tabSize": 2 }, "extensions": ["a.b"] } } +} +"#, + ) +} + +/// A two-link `extends` chain: a parent contributing an image, an environment variable and +/// a port, and a child contributing a name and a second environment variable. +fn extends_chain_fixture() -> Fixture { + Fixture::new() + .with_file( + ".devcontainer/base.json", + r#"{ + "image": "alpine:3.18", + "containerEnv": { "FROM_BASE": "1" }, + "forwardPorts": [3000] +} +"#, + ) + .with_file( + CONFIG_PATH, + r#"{ + "extends": "./base.json", + "name": "child", + "containerEnv": { "FROM_CHILD": "2" } +} +"#, + ) +} + +/// The hand-flattened equal of [`extends_chain_fixture`] — one document, no parent. +fn flatten_extends_chain(base: &Fixture) -> Result { + // Guard the correspondence rather than assume it: a flattening that silently stopped + // matching its chain would make the relation compare two unrelated documents and + // report a difference that says nothing about deacon. + if !base.files().contains_key(".devcontainer/base.json") { + return Err(HarnessError::RelationUnevaluable { + relation: "mrl-extends-flattening".to_string(), + cause: "the base fixture declares no parent document to flatten".to_string(), + }); + } + Ok(base + .clone() + .without_file(".devcontainer/base.json") + .with_file( + CONFIG_PATH, + r#"{ + "name": "child", + "image": "alpine:3.18", + "containerEnv": { "FROM_BASE": "1", "FROM_CHILD": "2" }, + "forwardPorts": [3000] +} +"#, + )) +} + +/// A Compose configuration whose `dockerComposeFile` is an ordered overlay list — the +/// collection the cited clause says order matters for. +fn compose_overlay_fixture() -> Fixture { + Fixture::with_config( + r#"{ + "name": "compose-overlay", + "dockerComposeFile": ["docker-compose.yml", "docker-compose.override.yml"], + "service": "app", + "workspaceFolder": "/workspace" +} +"#, + ) +} + +// --------------------------------------------------------------------------- +// Text transformations +// --------------------------------------------------------------------------- + +/// **`mrl-formatting-invariance`** — re-emit the document with different indentation and +/// line wrapping, changing no token. +/// +/// Every inter-token space is dropped and replaced with our own, which is token-preserving +/// **by construction** rather than by care: in JSON, whitespace appears only *between* +/// tokens, and two value tokens are never adjacent without a structural character +/// (`{}[]:,`) between them, so no re-spacing can fuse or split a token. The test asserts +/// the consequence non-circularly — both texts parse to the same value. +pub fn reindent(text: &str) -> Result { + let mut out = String::with_capacity(text.len() * 2); + let mut depth: usize = 0; + let mut in_string = false; + let mut escaped = false; + + // A deliberately unusual layout: tab indentation, a blank line after every `{`, and a + // space on both sides of `:`. Nothing about it is a token. + let indent = |out: &mut String, depth: usize| { + out.push('\n'); + for _ in 0..depth { + out.push('\t'); + } + }; + + for c in text.chars() { + if in_string { + out.push(c); + if escaped { + escaped = false; + } else if c == '\\' { + escaped = true; + } else if c == '"' { + in_string = false; + } + continue; + } + match c { + '"' => { + in_string = true; + out.push(c); + } + w if w.is_whitespace() => {} + '{' | '[' => { + out.push(c); + depth += 1; + indent(&mut out, depth); + } + '}' | ']' => { + depth = depth.saturating_sub(1); + indent(&mut out, depth); + out.push(c); + } + ':' => out.push_str(" : "), + ',' => { + out.push(c); + indent(&mut out, depth); + } + other => out.push(other), + } + } + out.push('\n'); + Ok(out) +} + +/// **`mrl-comment-invariance`** — insert JSONC line and block comments, and a trailing +/// comma in every non-empty object and array. +pub fn insert_comments_and_trailing_commas(text: &str) -> Result { + let mut out = String::with_capacity(text.len() * 2); + out.push_str("// injected: a leading line comment\n"); + let mut in_string = false; + let mut escaped = false; + + for c in text.chars() { + if in_string { + out.push(c); + if escaped { + escaped = false; + } else if c == '\\' { + escaped = true; + } else if c == '"' { + in_string = false; + } + continue; + } + match c { + '"' => { + in_string = true; + out.push(c); + } + '{' | '[' => { + out.push(c); + out.push_str(" /* injected: an opening block comment */ "); + } + '}' | ']' => { + // A trailing comma only where the container has content: `{,}` and `[,]` + // are not JSONC, and emitting one would test the parser's error handling + // rather than its comment tolerance. + if last_meaningful(&out).is_some_and(|p| !matches!(p, '{' | '[' | ',')) { + out.push(','); + } + out.push_str("\n// injected: a trailing line comment\n"); + out.push(c); + } + other => out.push(other), + } + } + out.push_str("\n// injected: a closing line comment\n"); + Ok(out) +} + +/// The last emitted character that is neither whitespace nor part of an injected comment. +/// +/// Scans backwards over whitespace and over any `//`-comment line we just wrote, which is +/// enough because this rewriter only ever appends comments in those two shapes. +fn last_meaningful(out: &str) -> Option { + for line in out.lines().rev() { + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with("//") { + continue; + } + // Drop a trailing block comment we injected on the same line. + let head = match trimmed.rfind("*/") { + Some(idx) => trimmed[..trimmed[..idx].rfind("/*").unwrap_or(0)].trim_end(), + None => trimmed, + }; + if let Some(c) = head.chars().next_back() { + return Some(c); + } + } + None +} + +/// **`mrl-key-order-invariance`** — reverse the member order of every JSON object. +pub fn reverse_object_members(text: &str) -> Result { + let value = parse_config(text, "mrl-key-order-invariance")?; + Ok(render_config(&reverse_members(&value))) +} + +fn reverse_members(value: &Value) -> Value { + match value { + Value::Object(map) => { + // `preserve_order` makes a `Map` insertion-ordered, so rebuilding it in reverse + // genuinely changes the document's member order — which is exactly the + // transformation. It does NOT change the value: `Map`'s equality is map + // equality. + let mut out = Map::new(); + for (k, v) in map.iter().rev() { + out.insert(k.clone(), reverse_members(v)); + } + Value::Object(out) + } + // Arrays keep their order: reversing one would be the *sensitivity* relation, and + // conflating the two would make this relation assert something it does not mean. + Value::Array(items) => Value::Array(items.iter().map(reverse_members).collect()), + other => other.clone(), + } +} + +/// **`mrl-lifecycle-equivalence`** — rewrite `postCreateCommand` from its bare form into the +/// single-entry named-object form denoting the same command. +pub fn lifecycle_to_named_object(text: &str) -> Result { + const RELATION: &str = "mrl-lifecycle-equivalence"; + let mut value = parse_config(text, RELATION)?; + let object = value + .as_object_mut() + .ok_or_else(|| HarnessError::RelationUnevaluable { + relation: RELATION.to_string(), + cause: "the configuration is not a JSON object".to_string(), + })?; + let existing = object + .get("postCreateCommand") + .cloned() + .filter(|v| !v.is_null() && !v.is_object()) + .ok_or_else(|| HarnessError::RelationUnevaluable { + relation: RELATION.to_string(), + cause: "the base fixture declares no bare-form `postCreateCommand` to rewrite; \ + with nothing to transform the relation would compare a document with \ + itself and hold vacuously" + .to_string(), + })?; + let mut named = Map::new(); + named.insert("solo".to_string(), existing); + object.insert("postCreateCommand".to_string(), Value::Object(named)); + Ok(render_config(&value)) +} + +/// **`mrl-declaration-order-sensitivity`** — reverse the `dockerComposeFile` overlay list. +pub fn reverse_compose_overlay_list(text: &str) -> Result { + const RELATION: &str = "mrl-declaration-order-sensitivity"; + let mut value = parse_config(text, RELATION)?; + let object = value + .as_object_mut() + .ok_or_else(|| HarnessError::RelationUnevaluable { + relation: RELATION.to_string(), + cause: "the configuration is not a JSON object".to_string(), + })?; + let list = object + .get_mut("dockerComposeFile") + .and_then(Value::as_array_mut) + .ok_or_else(|| HarnessError::RelationUnevaluable { + relation: RELATION.to_string(), + cause: "the base fixture declares no `dockerComposeFile` array".to_string(), + })?; + if list.len() < 2 { + // A one-element list has exactly one order, so reversing it changes nothing. A + // sensitivity relation over it could never be satisfied, and reporting that as a + // *failed relation* would blame deacon for a defective fixture. + return Err(HarnessError::RelationUnevaluable { + relation: RELATION.to_string(), + cause: format!( + "`dockerComposeFile` has {} entr(y/ies); a declaration-ordered collection \ + needs at least two for a permutation to exist", + list.len() + ), + }); + } + list.reverse(); + Ok(render_config(&value)) +} + +/// Parse an authored configuration document, tolerating nothing. +/// +/// The base fixtures are plain JSON by construction (the comment-bearing document is +/// *produced* by a transformation, never consumed by one), so a parse failure here is a +/// fixture defect and is reported as one rather than as a relation failure. +fn parse_config(text: &str, relation: &str) -> Result { + serde_json::from_str(text).map_err(|e| HarnessError::RelationUnevaluable { + relation: relation.to_string(), + cause: format!("the base configuration is not parseable JSON: {e}"), + }) +} + +fn render_config(value: &Value) -> String { + let mut out = serde_json::to_string_pretty(value) + .unwrap_or_else(|e| unreachable!("re-rendering a parsed document is infallible: {e}")); + out.push('\n'); + out +} + +// --------------------------------------------------------------------------- +// Sabotage — the SC-011 anti-inert probe +// --------------------------------------------------------------------------- + +/// Whether to evaluate a relation honestly, or to deliberately break it. +/// +/// The break is applied to the **input**, never to an observation: perturbing what the +/// comparison *returns* would make a comparison that ignores its arguments look alive, the +/// same defect 024's regression harness forbids by sealing its injection boundary. Here the +/// two breaks are the two ways a relation can genuinely be violated, so a relation that +/// still "holds" under one is not observing its own transformation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Sabotage { + /// Evaluate the relation as declared. + None, + /// Break it: give an invariance relation a genuinely different input, and take a + /// sensitivity relation's transformation away. + Break, +} + +/// The saboteur for an invariance relation: change something the resolved configuration +/// must reflect, so the two sides genuinely differ. +fn perturb_invariant_input(fixture: &Fixture, relation: &str) -> Result { + let mut value = parse_config(fixture.config(relation)?, relation)?; + let object = value + .as_object_mut() + .ok_or_else(|| HarnessError::RelationUnevaluable { + relation: relation.to_string(), + cause: "the configuration is not a JSON object".to_string(), + })?; + object.insert( + "name".to_string(), + Value::String("sabotaged-metamorphic-input".to_string()), + ); + Ok(fixture + .clone() + .with_file(CONFIG_PATH, &render_config(&value))) +} + +// --------------------------------------------------------------------------- +// Evaluation +// --------------------------------------------------------------------------- + +/// One difference between the two runs that the relation's accounting did not absorb. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Residual { + /// The observable path within the evidence document. + pub path: String, + /// The difference kind, in the wire spelling the signature uses. + pub kind: String, + /// What the original run produced. + pub original: Option, + /// What the transformed run produced. + pub transformed: Option, +} + +/// One side's evidence: the fixture that produced it, and what deacon did with it. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SideEvidence { + /// The materialized workspace tree. + pub input: Fixture, + /// The absolute workspace path it was materialized at. + pub workspace: String, + /// The RAW evidence document, before normalization. + pub raw: Value, + /// The NORMALIZED evidence document. Held separately from [`raw`](Self::raw), never + /// derived on read: raw and normalized evidence must never be conflated (FR-014, the + /// FR-016 precedent from 022). + pub normalized: Value, +} + +/// The result of evaluating one relation over one base fixture. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RelationOutcome { + /// The `mrl-` id. + pub relation: String, + /// The transformation as the catalogue words it — what a reviewer reproduces. + pub transformation: String, + /// What the relation asserts. + pub effect: RelationEffect, + /// Whether the assertion held. + pub holds: bool, + /// The original run. + pub original: SideEvidence, + /// The transformed run. + pub transformed: SideEvidence, + /// Differences the accounting did not absorb. + pub residual: Vec, + /// Differences the accounting DID absorb, retained rather than discarded: for + /// `mrl-path-relocation` this is the evidence that the tokenization did its job, and + /// for a scoped-site relation it is the transformation showing up where it should. + pub accounted: Vec, +} + +impl RelationOutcome { + /// The reviewable candidate for a failed relation (FR-047), or `None` when it held. + pub fn candidate(&self) -> Option { + if self.holds { + return None; + } + Some(MetamorphicCandidate { + relation: self.relation.clone(), + transformation: self.transformation.clone(), + effect: self.effect, + original_input: self.original.input.clone(), + transformed_input: self.transformed.input.clone(), + original_normalized: self.original.normalized.clone(), + transformed_normalized: self.transformed.normalized.clone(), + residual: self.residual.clone(), + }) + } +} + +/// A metamorphic failure, in the same reviewable shape a differential finding takes so both +/// enter one triage pipeline (FR-047, contracts/metamorphic-catalogue.md § Failure output). +/// +/// Names the relation, the transformation applied, **both** inputs, and **both** normalized +/// outputs — every one of them required, because a candidate a reviewer cannot reproduce +/// from its own contents is a bug report with the evidence left behind. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MetamorphicCandidate { + /// The relation that failed. + pub relation: String, + /// The transformation that was applied. + pub transformation: String, + /// What the relation asserted. + pub effect: RelationEffect, + /// The input the original run saw. + pub original_input: Fixture, + /// The input the transformed run saw. + pub transformed_input: Fixture, + /// The original run's normalized evidence. + pub original_normalized: Value, + /// The transformed run's normalized evidence. + pub transformed_normalized: Value, + /// The differences the accounting did not absorb (empty for a sensitivity failure, + /// where the absence of a difference IS the failure). + pub residual: Vec, +} + +impl MetamorphicCandidate { + /// The deduplication keys for this candidate. + /// + /// One [`Signature`] per residual difference, derived by the SAME function the + /// differential uses, so a metamorphic finding and a differential finding at the same + /// path deduplicate against each other — which is correct: they are the same defect + /// observed two ways. + /// + /// A **sensitivity** failure yields none, and that is not an oversight: its failure is + /// the *absence* of a difference, and there is nothing to key on. Keying it on the site + /// the transformation touched would collide with a genuine value difference at that + /// path and merge two unrelated defects. Such a candidate is identified by its relation. + pub fn signatures(&self, channel: &str) -> Vec { + self.residual + .iter() + .filter_map(|r| { + let kind = DivergenceKind::parse(&r.kind)?; + Some(Signature::derive( + channel, + // `reference` is the original run and `deacon` is the transformed one, + // fixed once here (see the module docs) so `deacon-only` reads as "the + // transformation introduced this". + &Divergence { + kind, + path: &r.path, + deacon: r.transformed.as_ref(), + reference: r.original.as_ref(), + }, + )) + }) + .collect() + } +} + +/// Evaluate every relation in `catalogue`, in catalogue order. +/// +/// Fails loudly — never skips — for a declared relation with no transformation: see +/// [`HarnessError::RelationUnevaluable`]. +pub async fn evaluate_catalogue( + deacon: &Path, + root: &Path, + catalogue: &[MetamorphicRelation], + sabotage: Sabotage, +) -> Result, HarnessError> { + let mut out = Vec::with_capacity(catalogue.len()); + for (index, relation) in catalogue.iter().enumerate() { + // A per-relation subdirectory so one relation's wipe cannot reach another's tree, + // and an index prefix so the layout is stable and readable after a failure. + let relation_root = root.join(format!("{index:02}-{}", relation.id)); + out.push(evaluate(deacon, &relation_root, relation, sabotage).await?); + } + Ok(out) +} + +/// Evaluate one relation against deacon alone. +pub async fn evaluate( + deacon: &Path, + root: &Path, + relation: &MetamorphicRelation, + sabotage: Sabotage, +) -> Result { + let transformation = + transformation_for(&relation.id).ok_or_else(|| HarnessError::RelationUnevaluable { + relation: relation.id.clone(), + cause: "no transformation is registered for it in `TRANSFORMATIONS`".to_string(), + })?; + + let base = (transformation.base)(); + let breaking_invariance = + sabotage == Sabotage::Break && relation.effect == RelationEffect::Invariance; + + // Breaking an INVARIANCE relation means giving the transformed side a genuinely + // different input; breaking a SENSITIVITY relation means taking its transformation + // away, so the two sides become identical and a relation that still reports "changed" + // is not observing its own transformation. Which side of the transformation the + // invariance perturbation goes on is declared per relation (see [`BreakPoint`]) rather + // than guessed, because a break that quietly failed to land would leave the relation + // holding — and a relation holding under its own break is indistinguishable from a + // healthy one. + let source = if breaking_invariance && transformation.break_point == BreakPoint::BeforeTransform + { + perturb_invariant_input(&base, &relation.id)? + } else { + base.clone() + }; + let mut transformed_input = + if sabotage == Sabotage::Break && relation.effect == RelationEffect::Sensitivity { + source.clone() + } else { + transformation.apply(&source)? + }; + if breaking_invariance && transformation.break_point == BreakPoint::AfterTransform { + transformed_input = perturb_invariant_input(&transformed_input, &relation.id)?; + } + + let (ws_original, ws_transformed) = workspace_paths(root, transformation.relocates()); + + let original = run_side( + deacon, + relation, + transformation, + &base, + &ws_original, + "original", + sabotage, + ) + .await?; + let transformed = run_side( + deacon, + relation, + transformation, + &transformed_input, + &ws_transformed, + "transformed", + sabotage, + ) + .await?; + + let (residual, accounted) = reconcile( + &original.normalized, + &transformed.normalized, + transformation.accounting, + ); + + let holds = match relation.effect { + RelationEffect::Invariance => residual.is_empty(), + // Sensitivity asks whether ANY difference appeared, including one the accounting + // would have explained away for an invariance relation — the assertion is "the + // result changed", not "the result changed somewhere unaccounted". + RelationEffect::Sensitivity => !residual.is_empty() || !accounted.is_empty(), + }; + + Ok(RelationOutcome { + relation: relation.id.clone(), + transformation: relation.transformation.clone(), + effect: relation.effect, + holds, + original, + transformed, + residual, + accounted, + }) +} + +/// Where each side is materialized. +/// +/// A non-relocating relation uses **one** directory for both runs, so the only thing that +/// differs between the two sides is the transformation. Giving them separate directories +/// would put a different absolute path into every result and force every relation to +/// tokenize — which would quietly turn `mrl-path-relocation` into the same relation as the +/// others and delete the FR-046 residual check. +fn workspace_paths(root: &Path, relocates: bool) -> (PathBuf, PathBuf) { + if relocates { + ( + root.join("origin").join(WORKSPACE_DIR_NAME), + root.join("relocated").join(WORKSPACE_DIR_NAME), + ) + } else { + let single = root.join(WORKSPACE_DIR_NAME); + (single.clone(), single) + } +} + +/// Materialize one side, run deacon over it, and capture raw + normalized evidence. +#[allow(clippy::too_many_arguments)] +async fn run_side( + deacon: &Path, + relation: &MetamorphicRelation, + transformation: &Transformation, + fixture: &Fixture, + workspace: &Path, + side_name: &str, + sabotage: Sabotage, +) -> Result { + fixture.materialize(workspace)?; + let workspace_arg = workspace.to_string_lossy().into_owned(); + let mut args: Vec<&str> = vec!["read-configuration", "--workspace-folder", &workspace_arg]; + args.extend_from_slice(transformation.argv); + + // The sabotage state is part of the artifact name, not only the outcome: the SC-011 + // probe evaluates each relation twice — once clean, once broken — and a shared name + // would have the second run silently overwrite the first's raw stdout/stderr. The one + // pair a reviewer most needs side by side is exactly the pair that would be lost. + let case = match sabotage { + Sabotage::None => format!("{}-{side_name}", relation.id), + Sabotage::Break => format!("{}-broken-{side_name}", relation.id), + }; + let invocation = run_and_capture( + Side::Deacon, + "discovery_metamorphic", + &case, + deacon, + &args, + workspace, + ExecKind::Config.bound(), + &crate::discovery_report_root(), + ) + .await?; + + let raw = raw_evidence(&invocation, transformation.scope); + let normalized = normalize_evidence(&raw, workspace, transformation.accounting); + Ok(SideEvidence { + input: fixture.clone(), + workspace: workspace_arg, + raw, + normalized, + }) +} + +/// Build the raw evidence document from one invocation. +/// +/// A run that failed, or whose stdout is not JSON, yields `structuredOutput: null` rather +/// than an error: the exit code is itself a declared channel, and a relation whose two +/// sides disagree about whether the configuration resolves is exactly the failure the +/// relation exists to catch. Turning that into a harness error would report deacon's +/// divergence as the harness being broken. +fn raw_evidence(invocation: &Invocation, scope: ObservedScope) -> Value { + let document = invocation + .stdout_json() + .ok() + .map(|doc| match scope { + ObservedScope::WholeDocument => doc, + ObservedScope::Block(name) => doc.get(name).cloned().unwrap_or(Value::Null), + }) + .unwrap_or(Value::Null); + let mut out = Map::new(); + out.insert( + "exitCode".to_string(), + match invocation.exit_code { + Some(code) => Value::from(code), + None => Value::Null, + }, + ); + out.insert("structuredOutput".to_string(), document); + Value::Object(out) +} + +/// Normalize one side's raw evidence: the single [`crate::normalize`] rule chain, plus the +/// workspace-path tokenization when — and only when — the relation's accounting declares it. +/// +/// The chain is reused whole rather than composed from a metamorphic-specific subset. +/// FR-015 permits exactly one normalization definition, and a bespoke chain here would be a +/// second one — free to disagree with the one every other comparison uses, which is the +/// defect that rule exists to prevent. +/// +/// The cost is worth naming. `drop_absent_optional` was written to reconcile a *serializer* +/// difference between deacon and the reference, and here both sides are deacon, so it can +/// only ever hide — specifically, a transformation that turned an enumerated key from +/// `[]` into absent would compare equal. It applies identically to both sides, so it cannot +/// manufacture a difference, only mask one; and the relation that most needs the strictness, +/// `mrl-extends-flattening`, observes the `mergedConfiguration` block on its own, where the +/// rule's scope check finds no wrapper key and elides nothing at all. +fn normalize_evidence(raw: &Value, workspace: &Path, accounting: Accounting) -> Value { + let mut out = Map::new(); + out.insert( + "exitCode".to_string(), + raw.get("exitCode").cloned().unwrap_or(Value::Null), + ); + let document = raw.get("structuredOutput").cloned().unwrap_or(Value::Null); + let ruled = normalize::config_document_rules(&document, Side::Deacon, DocumentBlock::Wrapper); + let document = if accounting == Accounting::PathTokenization { + normalize::path_token(&ruled, &workspace_tokens(workspace)) + } else { + ruled + }; + out.insert("structuredOutput".to_string(), document); + Value::Object(out) +} + +/// The `` token map for one side of a relocation. +/// +/// Carries BOTH the path as given and its canonicalized form. deacon canonicalizes the +/// workspace folder before substituting it, so on a host where the temp root is a symlink +/// the emitted path is the resolved one and a map built from the given path alone would +/// substitute nothing — producing a residual that looks like a leaked path and is really a +/// gap in the token map. +fn workspace_tokens(workspace: &Path) -> TokenMap { + let mut tokens = TokenMap::workspace(workspace); + if let Ok(canonical) = std::fs::canonicalize(workspace) + && canonical != workspace + { + tokens.insert(canonical.to_string_lossy(), ""); + } + tokens +} + +/// Split the differences between two normalized documents into those the accounting +/// absorbs and those it does not. +fn reconcile( + original: &Value, + transformed: &Value, + accounting: Accounting, +) -> (Vec, Vec) { + let mut residual = Vec::new(); + let mut accounted = Vec::new(); + // `diff`'s two sides are named for the differential: `reference` is the original run, + // `deacon` the transformed one (see the module docs). + for divergence in normalize::diff(transformed, original) { + let entry = Residual { + path: divergence.path.clone(), + kind: kind_str(divergence.kind).to_string(), + original: divergence.reference.clone(), + transformed: divergence.deacon.clone(), + }; + if accounting_absorbs(accounting, &divergence.path) { + accounted.push(entry); + } else { + residual.push(entry); + } + } + (residual, accounted) +} + +/// Whether `accounting` explains a difference at `path`. +/// +/// `PathTokenization` absorbs **nothing** here on purpose. The tokenization has already run +/// on both sides during normalization, so anything still differing is precisely the +/// residual FR-046 requires be reported — a path the tokenizer did not account for. Making +/// this arm absorb path-shaped differences would delete the check the relation exists for. +fn accounting_absorbs(accounting: Accounting, path: &str) -> bool { + match accounting { + Accounting::Identity | Accounting::PathTokenization => false, + Accounting::TransformedSites(sites) => sites + .iter() + .any(|site| path == *site || path.starts_with(&format!("{site}."))), + } +} + +fn kind_str(kind: DiffKind) -> &'static str { + match kind { + DiffKind::RefOnly => "ref-only", + DiffKind::DeaconOnly => "deacon-only", + DiffKind::Value => "value", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn relation(id: &str, effect: RelationEffect) -> MetamorphicRelation { + MetamorphicRelation { + id: id.to_string(), + transformation: "t".to_string(), + effect, + ground: "bhv-x".to_string(), + channels: vec!["chan-structured-output".to_string()], + rationale: "r".to_string(), + } + } + + #[test] + fn every_mandated_family_has_a_transformation() { + // The anti-inert guarantee at the table level: a family declared in the registry + // with no implementation here would be skipped, and a skipped relation reports + // nothing — which is indistinguishable from holding (SC-011). + for family in deacon_conformance::discovery::metamorphic::MANDATED_RELATIONS { + assert!( + transformation_for(family).is_some(), + "no transformation implements mandated relation `{family}`" + ); + } + assert_eq!( + TRANSFORMATIONS.len(), + deacon_conformance::discovery::metamorphic::MANDATED_RELATIONS.len() + ); + } + + #[tokio::test] + async fn an_unimplemented_relation_fails_loudly_rather_than_skipping() { + let dir = tempfile::tempdir().expect("tempdir"); + let err = evaluate( + Path::new("/nonexistent/deacon"), + dir.path(), + &relation("mrl-invented", RelationEffect::Invariance), + Sabotage::None, + ) + .await + .expect_err("an unimplemented relation must not be skipped"); + match err { + HarnessError::RelationUnevaluable { relation, .. } => { + assert_eq!(relation, "mrl-invented") + } + other => panic!("expected RelationUnevaluable, got {other:?}"), + } + } + + #[test] + fn reindent_changes_layout_and_no_token() { + let original = (rich_base_fixture)() + .config("t") + .expect("config") + .to_string(); + let rewritten = reindent(&original).expect("reindents"); + assert_ne!(rewritten, original, "the layout must actually change"); + assert!(rewritten.contains('\t'), "the rewrite uses tab indentation"); + // Token preservation, checked the only way that is not circular: both texts parse + // to the same value. + let a: Value = serde_json::from_str(&original).expect("original parses"); + let b: Value = serde_json::from_str(&rewritten).expect("rewritten parses"); + assert_eq!(a, b, "reindentation must preserve every token"); + } + + #[test] + fn comment_injection_produces_valid_jsonc_with_no_empty_container_comma() { + let original = (rich_base_fixture)() + .config("t") + .expect("config") + .to_string(); + let rewritten = insert_comments_and_trailing_commas(&original).expect("rewrites"); + assert!(rewritten.contains("// injected")); + assert!(rewritten.contains("/* injected")); + // `{}` must not become `{,}`: the fixture's `common-utils` options object is empty, + // and a comma there would test the parser's error handling rather than its comment + // tolerance. + assert!( + !rewritten.replace(char::is_whitespace, "").contains("{,"), + "an empty object must not receive a trailing comma:\n{rewritten}" + ); + assert!(!rewritten.replace(char::is_whitespace, "").contains("[,")); + // A non-empty container DOES get one. + assert!( + rewritten.contains(','), + "a non-empty container must receive a trailing comma" + ); + } + + #[test] + fn member_reversal_changes_order_but_not_the_value() { + let original = (rich_base_fixture)() + .config("t") + .expect("config") + .to_string(); + let rewritten = reverse_object_members(&original).expect("rewrites"); + assert_ne!(rewritten, original); + let a: Value = serde_json::from_str(&original).expect("parses"); + let b: Value = serde_json::from_str(&rewritten).expect("parses"); + assert_eq!(a, b, "objects compare as maps, so the VALUE is unchanged"); + // The rendered order really did change: `name` led the original document. + assert!(original.trim_start().starts_with('{')); + let first_key = |s: &str| s.split('"').nth(1).unwrap_or_default().to_string(); + assert_ne!(first_key(&original), first_key(&rewritten)); + // Arrays keep their order — reversing one would be the sensitivity relation. + assert_eq!( + b.get("forwardPorts"), + a.get("forwardPorts"), + "array order is not a member order" + ); + } + + #[test] + fn lifecycle_rewrite_wraps_the_bare_form_and_refuses_a_missing_one() { + let rewritten = + lifecycle_to_named_object((rich_base_fixture)().config("t").expect("config")) + .expect("rewrites"); + let value: Value = serde_json::from_str(&rewritten).expect("parses"); + assert_eq!( + value.get("postCreateCommand"), + Some(&serde_json::json!({ "solo": "echo hi" })) + ); + + // Nothing to transform is a fixture defect, not a vacuously holding relation. + let err = lifecycle_to_named_object(r#"{ "image": "alpine:3.18" }"#) + .expect_err("a fixture with no bare lifecycle command must fail loudly"); + assert!(matches!(err, HarnessError::RelationUnevaluable { .. })); + } + + #[test] + fn compose_reversal_needs_at_least_two_entries() { + let rewritten = + reverse_compose_overlay_list((compose_overlay_fixture)().config("t").expect("config")) + .expect("rewrites"); + let value: Value = serde_json::from_str(&rewritten).expect("parses"); + assert_eq!( + value.get("dockerComposeFile"), + Some(&serde_json::json!([ + "docker-compose.override.yml", + "docker-compose.yml" + ])) + ); + + // A one-element list has one order, so a sensitivity relation over it could never + // be satisfied. Reporting that as a failed relation would blame deacon for a + // defective fixture. + let err = reverse_compose_overlay_list(r#"{ "dockerComposeFile": ["only.yml"] }"#) + .expect_err("a single-entry list must fail loudly"); + assert!(matches!(err, HarnessError::RelationUnevaluable { .. })); + } + + #[test] + fn flattening_matches_the_chain_it_replaces() { + let chain = (extends_chain_fixture)(); + let flat = flatten_extends_chain(&chain).expect("flattens"); + assert!( + !flat.files().contains_key(".devcontainer/base.json"), + "the flattened fixture has no parent" + ); + let flat_config: Value = + serde_json::from_str(flat.config("t").expect("config")).expect("parses"); + assert!(flat_config.get("extends").is_none()); + // A fixture with nothing to flatten fails loudly rather than comparing a document + // with itself. + let err = flatten_extends_chain(&Fixture::with_config("{}")) + .expect_err("a chain-less fixture must fail loudly"); + assert!(matches!(err, HarnessError::RelationUnevaluable { .. })); + } + + #[test] + fn the_break_point_matches_what_each_transformation_can_carry() { + // `mrl-comment-invariance` emits JSONC, so a perturbation applied AFTER it would + // have to re-parse comments as strict JSON and would fail — the exact failure this + // declaration exists to prevent. + assert_eq!( + transformation_for("mrl-comment-invariance") + .expect("declared") + .break_point, + BreakPoint::BeforeTransform + ); + // `mrl-extends-flattening` emits the hand-written flattening and discards its + // input's content, so a perturbation applied BEFORE it would vanish and both sides + // would still agree — a break that never landed, reported as a healthy relation. + assert_eq!( + transformation_for("mrl-extends-flattening") + .expect("declared") + .break_point, + BreakPoint::AfterTransform + ); + // Every transformation whose output is strict JSON perturbs afterwards. + for transformation in TRANSFORMATIONS { + if transformation.relation != "mrl-comment-invariance" { + assert_eq!( + transformation.break_point, + BreakPoint::AfterTransform, + "{}", + transformation.relation + ); + } + } + } + + #[test] + fn a_non_relocating_relation_reuses_one_workspace() { + let root = Path::new("/tmp/x"); + let (a, b) = workspace_paths(root, false); + assert_eq!(a, b, "one directory, so only the transformation differs"); + let (a, b) = workspace_paths(root, true); + assert_ne!(a, b); + assert_eq!( + a.file_name(), + b.file_name(), + "the basename is held constant so relocation is a pure change of absolute path" + ); + } + + #[test] + fn the_scoped_accounting_absorbs_only_its_named_sites() { + let sites = + Accounting::TransformedSites(&["structuredOutput.configuration.postCreateCommand"]); + assert!(accounting_absorbs( + sites, + "structuredOutput.configuration.postCreateCommand" + )); + assert!(accounting_absorbs( + sites, + "structuredOutput.configuration.postCreateCommand.solo" + )); + // A sibling whose name merely starts the same way is NOT absorbed — the site is a + // path, not a prefix match. + assert!(!accounting_absorbs( + sites, + "structuredOutput.configuration.postCreateCommandExtra" + )); + assert!(!accounting_absorbs( + sites, + "structuredOutput.configuration.name" + )); + // The path-tokenization accounting absorbs nothing: tokenization already ran, so + // whatever still differs IS the FR-046 residual. + assert!(!accounting_absorbs( + Accounting::PathTokenization, + "anything" + )); + assert!(!accounting_absorbs(Accounting::Identity, "anything")); + } + + #[test] + fn a_candidate_names_the_relation_the_transformation_both_inputs_and_both_outputs() { + let outcome = RelationOutcome { + relation: "mrl-formatting-invariance".to_string(), + transformation: "reindent the configuration document".to_string(), + effect: RelationEffect::Invariance, + holds: false, + original: SideEvidence { + input: Fixture::with_config("{\"name\":\"a\"}"), + workspace: "/ws".to_string(), + raw: serde_json::json!({ "exitCode": 0 }), + normalized: serde_json::json!({ "structuredOutput": { "name": "a" } }), + }, + transformed: SideEvidence { + input: Fixture::with_config("{\n \"name\": \"b\"\n}"), + workspace: "/ws".to_string(), + raw: serde_json::json!({ "exitCode": 0 }), + normalized: serde_json::json!({ "structuredOutput": { "name": "b" } }), + }, + residual: vec![Residual { + path: "structuredOutput.name".to_string(), + kind: "value".to_string(), + original: Some(Value::String("a".to_string())), + transformed: Some(Value::String("b".to_string())), + }], + accounted: vec![], + }; + let candidate = outcome.candidate().expect("a failed relation yields one"); + assert_eq!(candidate.relation, "mrl-formatting-invariance"); + assert!(candidate.transformation.contains("reindent")); + assert_ne!(candidate.original_input, candidate.transformed_input); + assert_ne!( + candidate.original_normalized, + candidate.transformed_normalized + ); + + // The signature is derived by the same function the differential uses, so the two + // deduplicate against each other. + let signatures = candidate.signatures("chan-structured-output"); + assert_eq!(signatures.len(), 1); + assert_eq!(signatures[0].path, "structuredOutput.name"); + assert_eq!(signatures[0].derived_id(), signatures[0].id); + + // A relation that held yields no candidate at all. + let mut held = outcome.clone(); + held.holds = true; + assert!(held.candidate().is_none()); + } +} diff --git a/crates/parity-harness/src/discovery/minimize.rs b/crates/parity-harness/src/discovery/minimize.rs new file mode 100644 index 00000000..6af4c91f --- /dev/null +++ b/crates/parity-harness/src/discovery/minimize.rs @@ -0,0 +1,234 @@ +//! The live reproduction predicate supplied to the hermetic shrinker +//! (025-exploratory-parity-discovery, T048, FR-019 – FR-023). +//! +//! [`deacon_conformance::discovery::shrink`] takes its predicate as a **parameter** +//! precisely so the reduction strategy can be unit-tested against a synthetic one; this +//! module is the real one — it re-runs both implementations over a reduced input and reports +//! whether the *signature under reduction* is still there (research D4/D5). +//! +//! ## Nothing here is a new mechanism +//! +//! A probe is one call to [`super::differential::compare`]: the same bounded execution, the +//! same single normalization definition, the same tolerance index, the same signature +//! derivation. Re-implementing any of it would give minimization a second opinion on what +//! differs, able to disagree with the comparison that produced the finding — at which point +//! a "reduced input that still reproduces" would be a claim about a comparison nobody ran. +//! +//! ## Two things held constant across every probe, and why +//! +//! - **The workspace shape.** A probe materializes the reduced document into the *same* +//! tree shape [`super::campaign`] materializes a candidate into. A probe workspace whose +//! Compose or Dockerfile scaffolding differed would be measuring the scaffold. +//! - **`deliberately_invalid`.** It is a fact about the *candidate's provenance* — a +//! near-valid draw, or a mutated document — not about the document currently under the +//! knife. Recomputing it as the reduction un-applies mutations would move the tolerance +//! boundary mid-reduction, so "the signature was preserved" would silently start meaning +//! something different halfway through. +//! +//! ## What each of the three answers means here +//! +//! | Probe answer | Live meaning | +//! |---|---| +//! | [`Preserved`](Reproduction::Preserved) | the target signature is among the observations | +//! | [`Drifted`](Reproduction::Drifted) | it is not, but other **new** differences are — captured as candidate findings (FR-023) | +//! | [`Absent`](Reproduction::Absent) | it is not, and nothing else is news either | +//! +//! An already-*characterized* observation never counts as drift. It is not a candidate +//! finding by definition (FR-017), and admitting it here would let minimization author +//! findings the differential itself refuses to raise. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use deacon_conformance::discovery::shrink::{Reproduction, ReproductionProbe}; +use deacon_conformance::discovery::signature::Signature; +use serde_json::Value; + +use crate::HarnessError; +use crate::oracle::VerifiedOracle; + +use super::campaign::materialize_document; +use super::differential::{self, Characterization, DifferentialInput}; + +/// Everything a probe needs, resolved once by the caller. +/// +/// Owned rather than borrowed for the two path fields so a probe can be constructed at the +/// call site without threading a lifetime through the campaign loop; the oracle and the +/// tolerance index stay borrowed because both are shared, immutable, and expensive to clone. +pub struct DifferentialProbe<'a> { + /// The signature the reduction must preserve. + target: Signature, + /// The candidate the finding came from — the artifact-tree prefix, so a reviewer can + /// find a probe's raw output beside the candidate's own. + candidate_id: String, + deacon: PathBuf, + /// The **verified** pinned oracle. Taking the verified type rather than a path is what + /// makes "never compare against an unverified reference" (FR-003) a type-level fact at + /// this call site rather than a rule the caller has to remember. + oracle: &'a VerifiedOracle, + bound: Duration, + report_root: PathBuf, + characterization: &'a Characterization, + /// Constant for the whole reduction — see the module docs. + deliberately_invalid: bool, + /// How many probes have been made, so each writes to its own artifact directory rather + /// than overwriting the previous one's raw capture. + probes: u64, +} + +impl<'a> DifferentialProbe<'a> { + /// Build a probe for one finding under reduction. + #[allow(clippy::too_many_arguments)] + pub fn new( + target: Signature, + candidate_id: &str, + deacon: &Path, + oracle: &'a VerifiedOracle, + bound: Duration, + report_root: &Path, + characterization: &'a Characterization, + deliberately_invalid: bool, + ) -> DifferentialProbe<'a> { + DifferentialProbe { + target, + candidate_id: candidate_id.to_string(), + deacon: deacon.to_path_buf(), + oracle, + bound, + report_root: report_root.to_path_buf(), + characterization, + deliberately_invalid, + probes: 0, + } + } + + /// How many probes this instance made — the expensive unit, reported so a campaign can + /// say what minimization cost it. + pub fn probes(&self) -> u64 { + self.probes + } + + async fn run(&mut self, document: &Value) -> Result { + let workspace = materialize_document(document).map_err(|e| HarnessError::Report { + cause: format!( + "could not materialize a reduced workspace for `{}`: {e}", + self.candidate_id + ), + })?; + // A per-probe artifact name, so the raw capture of every probe survives. Reusing the + // candidate's own name would leave only the LAST probe's output on disk, and the + // reduction's most interesting artifact is usually the probe that failed. + let case = format!("{}-shrink-{:04}", self.candidate_id, self.probes); + self.probes += 1; + + let result = differential::compare( + DifferentialInput { + candidate_id: &case, + workspace: workspace.path(), + deacon: &self.deacon, + oracle: self.oracle, + bound: self.bound, + report_root: &self.report_root, + deliberately_invalid: self.deliberately_invalid, + }, + self.characterization, + ) + .await; + + // A per-candidate timeout during minimization is a fact about *this probe*, not a + // failure of the campaign: the reduced input happened to be pathological. It is + // reported as `Absent` — the reduction did not demonstrate reproduction — so the + // step is rejected and the reduction continues, exactly as the campaign loop treats + // a timed-out candidate rather than letting one input consume the whole budget. + let result = match result { + Ok(r) => r, + Err(HarnessError::OracleTimeout { .. }) => { + tracing::debug!( + candidate = %self.candidate_id, + probe = %case, + "a minimization probe exceeded its bound; the step is rejected" + ); + return Ok(Reproduction::Absent); + } + Err(other) => return Err(other), + }; + + if result + .observations + .iter() + .any(|o| o.signature.id == self.target.id) + { + return Ok(Reproduction::Preserved); + } + + // FR-023: what appeared *instead*. Only genuinely new observations — an + // already-characterized difference is not a candidate finding (FR-017), and + // capturing one here would let minimization raise what the differential refuses to. + let drifted: Vec = result + .new_observations() + .map(|o| o.signature.clone()) + .collect(); + Ok(if drifted.is_empty() { + Reproduction::Absent + } else { + Reproduction::Drifted(drifted) + }) + } +} + +impl ReproductionProbe for DifferentialProbe<'_> { + type Error = HarnessError; + + async fn probe(&mut self, document: &Value) -> Result { + self.run(document).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use deacon_conformance::discovery::signature::{Divergence, DivergenceKind}; + use deacon_conformance::model::CHAN_STRUCTURED_OUTPUT; + + fn signature(path: &str) -> Signature { + Signature::derive( + CHAN_STRUCTURED_OUTPUT, + &Divergence { + kind: DivergenceKind::Value, + path, + deacon: None, + reference: None, + }, + ) + } + + /// The probe's own bookkeeping is asserted here; its *verdict* is asserted live, in + /// `discovery_campaign`, because the verdict is exactly the thing that cannot be + /// established without the reference. + #[test] + fn a_fresh_probe_has_spent_nothing() { + let oracle = VerifiedOracle { + path: PathBuf::from("/nonexistent/devcontainer"), + source: crate::oracle::OracleSource::Override, + version: "0.0.0".to_string(), + }; + let characterization = Characterization::default(); + let probe = DifferentialProbe::new( + signature("configuration.remoteUser"), + "cnd-11111111", + Path::new("/nonexistent/deacon"), + &oracle, + Duration::from_secs(1), + Path::new("/tmp/does-not-matter"), + &characterization, + true, + ); + assert_eq!(probe.probes(), 0); + assert_eq!(probe.target.path, "configuration.remoteUser"); + assert!( + probe.deliberately_invalid, + "the provenance flag is held constant for the whole reduction, so it must be \ + what the caller passed rather than something recomputed per probe" + ); + } +} diff --git a/crates/parity-harness/src/discovery/mod.rs b/crates/parity-harness/src/discovery/mod.rs new file mode 100644 index 00000000..5ed9c47d --- /dev/null +++ b/crates/parity-harness/src/discovery/mod.rs @@ -0,0 +1,50 @@ +//! Exploratory parity discovery — the **live** half +//! (025-exploratory-parity-discovery, plan.md § Project Structure). +//! +//! Everything that touches the pinned oracle, Docker, or the network lives here; the +//! pure data-to-data logic (grammar, generation, mutation, reduction strategy, +//! signature, queue, relations, reports) lives in `deacon_conformance::discovery`. This +//! is the 022 hermetic/live split applied unchanged (research D4), and it is what keeps +//! the hermetic half runnable in the fast lane with no external dependency. +//! +//! ## What this half must not re-implement +//! +//! - **Process execution** — reuse [`crate::exec`]; a second execution path would be a +//! second set of bounds, captures, and failure modes. +//! - **Oracle resolution and exact-version verification** — reuse [`crate::oracle`] and +//! [`crate::prereq`]; a missing or mismatched oracle fails loudly, never silently +//! (FR-003). +//! - **Normalization** — reuse [`crate::normalize`]. FR-015 permits exactly one +//! normalization definition, and a signature computed from independently re-diffed +//! values would be a second opinion on what differs, able to disagree with the one the +//! comparison used (research D3). +//! - **Injection** — reuse [`crate::inject`]'s sealed `EvidenceSource` boundary, which +//! already makes injecting into an observer's *return* value fail to compile +//! (research D7). +//! +//! ## Exit-status discipline +//! +//! Every discovery command's exit status reflects **whether it ran**, never **what it +//! found** (contracts/discovery-cli.md, FR-058). A campaign that finds forty differences +//! exits `0`; a campaign that cannot verify the oracle exits non-zero. The single +//! exception is `discovery-proof`, whose status asserts a property of the *machinery*. +//! +//! ## Module map +//! +//! | Module | Owns | +//! |---|---| +//! | [`campaign`] | the driver: seed, tier, budget, per-candidate timeout, admission cap, outcome accumulation | +//! | [`differential`] | deacon vs the verified pinned oracle over one candidate | +//! | [`metamorphic_run`] | deacon-only relation evaluation (needs no oracle, Docker, or network) | +//! | [`minimize`] | supplies the live reproduction predicate to the hermetic shrinker | +//! | [`candidate`] | assembles the reviewable candidate under `target/discovery/candidates/` | +//! | [`corpus_fetch`] | network-lane fetch + content-digest verification | +//! | [`pipeline_proof`] | the injected-difference traversal proof | + +pub mod campaign; +pub mod candidate; +pub mod corpus_fetch; +pub mod differential; +pub mod metamorphic_run; +pub mod minimize; +pub mod pipeline_proof; diff --git a/crates/parity-harness/src/discovery/pipeline_proof.rs b/crates/parity-harness/src/discovery/pipeline_proof.rs new file mode 100644 index 00000000..ad6f6168 --- /dev/null +++ b/crates/parity-harness/src/discovery/pipeline_proof.rs @@ -0,0 +1,1209 @@ +//! The injected-difference pipeline proof +//! (025-exploratory-parity-discovery, US5, T081/T082, FR-042a, research D7). +//! +//! Injects a known difference through [`crate::inject::perturb_source`] — the existing +//! **sealed** `EvidenceSource` boundary — and requires it to traverse +//! **generation → comparison → minimization → candidate → classification → promotable**. +//! +//! # Why the sealed boundary, and what reusing it buys +//! +//! `inject.rs` already establishes the one property this proof cannot cheaply +//! re-establish: every perturbation entry point is generic over a **sealed** +//! [`EvidenceSource`](crate::inject::EvidenceSource) trait that only +//! [`RunContext`] — the RAW captured artifact — implements. No +//! observer's return type can implement it, and its supertrait lives in a private module, +//! so injecting *downstream* of the comparison is not a rule to remember: it does not +//! compile. +//! +//! That matters here more than it does for `coverage-regressions`. A proof that could plant +//! a difference past the comparison would demonstrate nothing at all — it would be asserting +//! on data it wrote itself, downstream of the part under test. Reusing the boundary inherits +//! the guarantee rather than re-arguing it. +//! +//! It inherits [`HarnessError::InjectionInapplicable`] too, and that is the other half of +//! FR-042a: **a perturbation that never landed is reported as a harness fault, never as "the +//! pipeline found nothing"**. Those are opposite conclusions — one says the machinery is +//! mis-authored, the other says the implementations agree — and a proof that merged them +//! would be the most comfortable possible way for this feature to be broken. +//! +//! # What it compares against, and why it is not the oracle +//! +//! The counterpart is **deacon's own unperturbed run**, not the pinned reference. +//! +//! This is deliberate and it is what makes the proof mean anything. A reference comparison +//! has a baseline nobody controls: deacon and the oracle may already differ on a generated +//! candidate, and an injected difference landing on top of that cannot be attributed to the +//! injection. Comparing a run against itself makes the baseline empty — and [`establish`] +//! fails loudly rather than assuming it, so **every observation that appears afterwards is +//! the one that was planted**. That is the whole attribution argument, and it is the same +//! clean-baseline requirement [`crate::inject::detects`] imposes on a channel verdict. +//! +//! It also means the proof needs **no oracle, no Docker, and no network**: it asserts a +//! property of the *machinery*, and the reference takes no part in whether a difference +//! propagates from evidence to a promotable finding. That is the same argument research D12 +//! makes for the metamorphic tier having no prerequisite, applied to the one command whose +//! exit status is supposed to mean "the pipeline is broken" rather than "a prerequisite was +//! missing". +//! +//! Two consequences follow, both load-bearing: +//! +//! - **Both sides are normalized as [`Side::Deacon`]**. The single normalizer is +//! deliberately side-asymmetric (`drop_absent_optional` runs on deacon's `configuration` +//! block only, 024 T123), so labelling one copy of deacon's output as the reference would +//! apply rules written for a different serializer and manufacture differences the pipeline +//! never found. Note what the empty-baseline check does and does not establish: because +//! both sides are normalized identically it is *satisfied by construction*, so it +//! confirms attribution — the two sides as this module builds them diff to nothing — but +//! it does **not** independently vindicate the side choice. The argument for that is the +//! one above, and a refactor that made the two sides asymmetric would be caught by the +//! check rather than by the reasoning. +//! - **The tolerance index is not consulted.** [`Characterization`] answers "has this +//! deacon-vs-reference difference already been characterized?", a question with no meaning +//! for a self-comparison — and a scoped waiver that happened to cover the injected path +//! would suppress the planted difference and report a working pipeline as broken. Left out +//! visibly, for the same reason `run_metamorphic` leaves it out. +//! +//! # What it must never do +//! +//! Write anything the registry owns (FR-036). It emits reviewable candidates under +//! `/proof/` and a report at `target/discovery/proof.json`, and it reaches the +//! promotion stage by *refusing* to promote: the stage passes when the state machine permits +//! promotion **and** [`promote::validate_promotion`] still rejects the scaffolded record for +//! every missing axis. Review-only promotion is proven by showing the gate holds, not by +//! going through it. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use deacon_conformance::discovery::generate::{ + Candidate, Generator, required_keys, unsafe_reasons, +}; +use deacon_conformance::discovery::grammar::Grammar; +use deacon_conformance::discovery::promote::{self, PromotionError}; +use deacon_conformance::discovery::queue::{ + Campaign, CampaignLane, CampaignTier, Classification, Finding, FindingState, ObservedValues, + PinnedInputSet, Witness, +}; +use deacon_conformance::discovery::shrink::{ + self, ReductionInput, Reproduction, ReproductionProbe, +}; +use deacon_conformance::discovery::signature::Signature; +use deacon_conformance::load::Registry; +use deacon_conformance::regression::{RegressionFile, RegressionRecord}; +use serde_json::Value; + +use crate::HarnessError; +use crate::exec::{Side, run_and_capture}; +use crate::inject::{RegressionHarness, perturb_source}; +use crate::observe::{ProcessOutcome, RunContext}; + +use super::campaign::{CandidateWorkspace, materialize_document, pinned_input_set}; +use super::candidate::{self, CANDIDATE_PARTS, CandidateInputs, ReferenceProvenance}; +use super::differential::{ + self, Characterization, DifferentialResult, Observation, OutcomeClass, SideEvidence, +}; + +/// The artifact-tree binary name every proof invocation captures under. +const PROOF_BINARY: &str = "discovery_proof"; + +/// The operation id the perturbation is applied to. One operation per run +/// (`read-configuration`), so the name is fixed. +const PROOF_OPERATION: &str = "read-configuration"; + +/// The injections the proof requires to traverse, as strict-JSON [`RegressionRecord`]s. +/// +/// **Exactly the two channels this comparison reads.** The configuration differential +/// relates two things and no others (FR-016): whether each side *accepted or rejected* the +/// candidate, and — when both accepted — the *normalized structured document*. A record +/// perturbing stderr would be correctly invisible, because no code path reads stderr into a +/// comparison, and requiring it to surface would demand a defect rather than a proof. +/// +/// These are **not** registry records. They are never loaded by `validate`, carry no `reg-` +/// obligation, and `expectedDetectingCases` — required non-empty by the shape — plays no +/// part here: the proof reports which *stages* a difference reached, not which cases caught +/// it. +const PROOF_INJECTIONS_JSON: &str = r#"{ + "records": [ + { + "id": "reg-proof-structured-output", + "channel": "chan-structured-output", + "target": "structured-output-document", + "perturbation": { + "kind": "set-json-pointer", + "pointer": "/configuration/deaconPipelineProofMarker", + "value": "injected-by-the-pipeline-proof" + }, + "expectedDetectingCases": ["discovery-proof"], + "notes": "Writes through /configuration, which every accepted read-configuration document carries, so the perturbation lands on any candidate the proof can establish a clean baseline from. The key is not in ABSENT_OPTIONAL_KEYS and its value is non-empty, so no normalization rule can elide it — the difference the pipeline must surface is the one that was planted." + }, + { + "id": "reg-proof-exit-code", + "channel": "chan-exit-code", + "target": "process-result", + "perturbation": { "kind": "set-exit-code", "exitCode": 9 }, + "expectedDetectingCases": ["discovery-proof"], + "notes": "Flips the perturbed side to REJECTED while the unperturbed counterpart accepted, which is an outcome-class divergence. The comparison compares the class, never the numeric code, so 9 is arbitrary and the assertion is about meaning rather than spelling." + } + ] +}"#; + +/// The proof's declared injections, parsed from `PROOF_INJECTIONS_JSON`. +/// +/// Parsed rather than constructed so the records go through the SAME strict loader a +/// committed `reg-` record does: a proof whose perturbations were built by hand could +/// declare a shape the real harness would reject. +pub fn proof_injections() -> Result, HarnessError> { + let file: RegressionFile = + serde_json::from_str(PROOF_INJECTIONS_JSON).map_err(|e| HarnessError::Report { + cause: format!("the proof's own injection records do not load: {e}"), + })?; + if file.records.is_empty() { + return Err(HarnessError::Report { + cause: "the proof declares no injections; a proof over zero injections reports \ + the same thing as one in which every injection traversed" + .to_string(), + }); + } + Ok(file.records) +} + +// --------------------------------------------------------------------------- +// Stages and verdicts +// --------------------------------------------------------------------------- + +/// The pipeline stages FR-042a requires an injected difference to traverse, in order. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Stage { + /// A candidate was drawn from the real constrained generator. + Generation, + /// The difference surfaced as an observation on the record's declared channel, against + /// a provably clean baseline. + Comparison, + /// The real structural shrinker reduced the input while the signature held. + Minimization, + /// All six parts of a reviewable candidate were emitted. + Candidate, + /// The finding took exactly one classification through the real state machine. + Classification, + /// Promotion is permitted by the state machine **and** still gated on a human. + Promotable, +} + +impl Stage { + /// The stable wire spelling. + pub fn as_str(self) -> &'static str { + match self { + Stage::Generation => "generation", + Stage::Comparison => "comparison", + Stage::Minimization => "minimization", + Stage::Candidate => "candidate", + Stage::Classification => "classification", + Stage::Promotable => "promotable", + } + } + + /// Every stage, in traversal order. + pub fn all() -> &'static [Stage] { + &[ + Stage::Generation, + Stage::Comparison, + Stage::Minimization, + Stage::Candidate, + Stage::Classification, + Stage::Promotable, + ] + } +} + +/// One stage's outcome, with the evidence behind it. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StageOutcome { + pub stage: Stage, + /// What was actually observed at this stage — recorded whether it passed or failed, so + /// a failure names the thing that was wrong rather than only the stage it was in. + pub detail: String, +} + +/// What one injection's traversal concluded. +/// +/// The three variants are three genuinely different facts, and keeping them apart is the +/// point of FR-042a: the pipeline worked; the pipeline swallowed a difference that was +/// there; or the proof itself was mis-authored and never planted anything. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(tag = "verdict", rename_all = "kebab-case")] +pub enum TraversalVerdict { + /// Every stage was reached. + Traversed, + /// The perturbation LANDED and the difference did not reach the end of the pipeline. + /// **A pipeline defect.** + FailedToSurface { stage: Stage, cause: String }, + /// The perturbation never landed, so nothing at all was proven about the pipeline. + /// **A proof defect** — deliberately not `failed-to-surface`, and deliberately not a + /// pass: a mis-authored record must never masquerade as a working pipeline. + InjectionInapplicable { cause: String }, +} + +/// One injection's traversal. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Traversal { + /// The perturbation record's id. + pub injection: String, + /// The channel it was declared on — the channel the difference had to surface on. + pub channel: String, + /// How many raw artifacts the perturbation was applied to (zero is impossible: it is + /// [`TraversalVerdict::InjectionInapplicable`] before it gets here). + pub applied: usize, + /// The signature the difference produced, once the comparison surfaced it. + #[serde(skip_serializing_if = "Option::is_none")] + pub signature: Option, + /// The finding id the signature derives — what a campaign would have admitted. + #[serde(skip_serializing_if = "Option::is_none")] + pub finding: Option, + /// The stages reached, in order. Truncated at the first failure. + pub stages: Vec, + #[serde(flatten)] + pub verdict: TraversalVerdict, +} + +impl Traversal { + /// Whether this traversal counts as a pass. + pub fn traversed(&self) -> bool { + self.verdict == TraversalVerdict::Traversed + } +} + +/// The `target/discovery/proof.json` document. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ProofReport { + pub schema_version: u32, + /// The candidate the proof established its clean baseline from — the `cnd-` id, so a + /// failure can be reproduced from the same seed. + pub baseline_candidate: String, + /// The seed, recorded verbatim, for the same reason a campaign records one. + pub seed: String, + /// One entry per declared injection, in declaration order. + pub injections: Vec, + /// Injections that landed and did not traverse — pipeline defects. + pub failed_count: usize, + /// Injections that never landed — proof defects. + pub inapplicable_count: usize, +} + +impl ProofReport { + /// Roll traversals up into the report. + pub fn build(seed: &str, baseline_candidate: &str, injections: Vec) -> ProofReport { + let failed_count = injections + .iter() + .filter(|t| matches!(t.verdict, TraversalVerdict::FailedToSurface { .. })) + .count(); + let inapplicable_count = injections + .iter() + .filter(|t| matches!(t.verdict, TraversalVerdict::InjectionInapplicable { .. })) + .count(); + ProofReport { + schema_version: 1, + baseline_candidate: baseline_candidate.to_string(), + seed: seed.to_string(), + injections, + failed_count, + inapplicable_count, + } + } + + /// The run's exit status per contracts/discovery-cli.md: `0` when every injection + /// traversed, `1` when any failed to surface **or** any was inapplicable. + /// + /// An **empty** injection set is also `1`. This is the one discovery command whose + /// status depends on an outcome, and a proof over nothing reports exactly what a proof + /// over everything-passing reports — the indistinguishability this whole feature exists + /// to refuse. + /// + /// A function rather than a rule the bin restates, so the test asserting the status and + /// the bin producing it are asserting the same decision. + pub fn exit_status(&self) -> u8 { + if self.injections.is_empty() { + return 1; + } + if self.failed_count == 0 && self.inapplicable_count == 0 { + 0 + } else { + 1 + } + } + + /// Byte-stable pretty JSON with a trailing newline. + pub fn render(&self) -> Result { + let mut s = serde_json::to_string_pretty(self).map_err(|e| HarnessError::Report { + cause: format!("could not serialize the proof report: {e}"), + })?; + s.push('\n'); + Ok(s) + } +} + +// --------------------------------------------------------------------------- +// Request and context +// --------------------------------------------------------------------------- + +/// Everything one proof run needs, resolved by the caller (the bin, or a test). +/// +/// Every knob is explicit rather than read from the environment, matching +/// [`super::campaign::CampaignRequest`]: mutating process-global state is `unsafe` under +/// this workspace's edition and hostile to a parallel runner besides. +#[derive(Debug, Clone)] +pub struct ProofRequest { + /// The deacon binary under test — the ONLY implementation this proof runs. + pub deacon_binary: PathBuf, + /// The registry root, for the pins and the candidate's suggested behavior mapping. + pub registry_dir: PathBuf, + /// Where raw artifacts, proof candidates, and the report are written. + pub report_root: PathBuf, + /// The seed, exactly as recorded. A proof is as reproducible as a campaign. + pub seed_hex: String, + /// The seed's numeric value, feeding the generator's stream. + pub seed: u64, + /// The certification profile the run records itself under (a `prof-` id). + pub profile: String, + /// The per-invocation bound. + pub bound: Duration, + /// How many probes minimization may spend per injection. + pub shrink_budget: u64, + /// How many candidates may be drawn while looking for one deacon accepts. + pub max_draws: u64, +} + +/// The clean baseline every injection is traversed against. +/// +/// Established once and shared: the baseline is a property of the *candidate*, not of the +/// injection, and re-establishing it per injection would spend a deacon invocation to reach +/// the same document. +pub struct ProofContext { + registry: Registry, + grammar: Grammar, + pins: PinnedInputSet, + campaign_id: String, + /// The generated candidate that produced a clean baseline. + candidate: Candidate, + /// Its materialized workspace, reclaimed when the context drops. + workspace: CandidateWorkspace, + /// The unperturbed run's evidence — the counterpart every injected side is compared to. + baseline: SideEvidence, + /// The unperturbed run's raw process outcome, re-perturbed per injection. + baseline_outcome: ProcessOutcome, + request: ProofRequest, +} + +impl ProofContext { + /// The candidate the baseline was established from. + pub fn candidate_id(&self) -> &str { + &self.candidate.id + } +} + +/// Establish a clean baseline: draw candidates until one is safe **and** deacon accepts it +/// **and** comparing its evidence against itself yields nothing. +/// +/// Every failure here is fail-loud. In particular, running out of draws is an error rather +/// than an empty report: a proof that could not find an input to plant a difference in has +/// proven nothing, and reporting that as "no injections failed" would be the exact +/// indistinguishability FR-042a forbids. +pub async fn establish(request: &ProofRequest) -> Result { + let registry = Registry::load(&request.registry_dir).map_err(|e| HarnessError::Report { + cause: format!( + "could not load the conformance registry at {}: {e}", + request.registry_dir.display() + ), + })?; + let grammar = Grammar::load_default().map_err(|e| HarnessError::Report { + cause: format!("could not load the generation grammar: {e}"), + })?; + // No oracle is acquired — see the module docs. The pinned oracle *version* still enters + // the pinned input set, because it is part of what makes any finding checkable, and the + // pin is read from the committed `oracle.json` rather than from a binary. + let pins = pinned_input_set(&grammar, None)?; + let campaign_id = Campaign::derive_id( + &request.seed_hex, + &pins, + CampaignLane::Invoked, + &request.profile, + // The closest declared tier: the proof draws generated configuration documents and + // compares `read-configuration` evidence, exactly as the nightly tier does. Nothing + // is persisted, so this id names a run rather than a record. + CampaignTier::ConfigDifferential, + ); + + let mut generator = Generator::new(&grammar, request.seed); + let mut rejected: Vec = Vec::new(); + + for _ in 0..request.max_draws { + let candidate = generator.next_candidate(); + + // FR-011: the same safety predicates a campaign applies. The proof runs real + // deacon over a real generated document and must not be the one code path that + // executes what a campaign would refuse. + let refusals = unsafe_reasons(&candidate.document, /* container_backed */ false); + if !refusals.is_empty() { + rejected.push(format!( + "{}: unsafe ({})", + candidate.id, + refusals.join("; ") + )); + continue; + } + + let workspace = + materialize_document(&candidate.document).map_err(|e| HarnessError::Report { + cause: format!( + "could not materialize a proof workspace for `{}`: {e}", + candidate.id + ), + })?; + + let outcome = run_deacon(request, &candidate.id, workspace.path()).await?; + let evidence = side_evidence( + &outcome, + workspace.path(), + &request.report_root, + &candidate.id, + ); + + // The baseline must be ACCEPTED and structured: a rejected run produces no document + // for the structured-output injection to perturb, and no outcome-class difference + // for the exit-code injection to create. + if evidence.outcome != OutcomeClass::Accepted || evidence.normalized.is_none() { + rejected.push(format!( + "{}: deacon did not produce an accepted structured document", + candidate.id + )); + continue; + } + + // And it must be CLEAN against itself. This is what makes every later observation + // attributable to the injection rather than to the candidate — the same + // clean-baseline requirement `inject::detects` imposes, established here rather + // than assumed. + let self_comparison = differential::result_from_sides( + &candidate.id, + evidence.clone(), + evidence.clone(), + &Characterization::default(), + false, + ); + if !self_comparison.observations.is_empty() { + return Err(HarnessError::Report { + cause: format!( + "candidate `{}` differs from ITSELF in {} place(s) before anything was \ + injected ({}). The comparison is not deterministic over one run's \ + evidence, so no observation after an injection could be attributed to \ + it — this is a comparison defect, not a proof that found nothing.", + candidate.id, + self_comparison.observations.len(), + self_comparison + .observations + .iter() + .map(|o| o.signature.path.clone()) + .collect::>() + .join(", ") + ), + }); + } + + return Ok(ProofContext { + registry, + grammar, + pins, + campaign_id, + candidate, + workspace, + baseline: evidence, + baseline_outcome: outcome, + request: request.clone(), + }); + } + + Err(HarnessError::Report { + cause: format!( + "no clean baseline in {} draw(s) from seed {}: {}. A proof with no input to \ + plant a difference in has proven nothing, so this is an error rather than a \ + report with no failures.", + request.max_draws, + request.seed_hex, + rejected.join(" | ") + ), + }) +} + +/// Traverse one injected difference through the whole pipeline. +/// +/// Public so a hermetic guard can drive an arbitrary record — including a deliberately +/// inapplicable one, which is how the "an injection that never lands fails LOUDLY rather +/// than reading as found-nothing" half of FR-042a is asserted rather than merely described. +pub async fn traverse( + ctx: &ProofContext, + record: &RegressionRecord, +) -> Result { + let mut stages: Vec = Vec::new(); + let mut traversal = Traversal { + injection: record.id.clone(), + channel: record.channel.clone(), + applied: 0, + signature: None, + finding: None, + stages: Vec::new(), + verdict: TraversalVerdict::Traversed, + }; + + // ---- generation ----------------------------------------------------- + stages.push(StageOutcome { + stage: Stage::Generation, + detail: format!( + "candidate `{}` (draw {}, kind {:?}) drawn from seed {} by the real constrained \ + generator", + ctx.candidate.id, ctx.candidate.index, ctx.candidate.kind, ctx.request.seed_hex + ), + }); + + // ---- comparison ----------------------------------------------------- + let injected = match inject_and_compare(ctx, record, &ctx.candidate.document).await { + Ok(injected) => injected, + // The distinction FR-042a draws, and the reason this is not folded into the failure + // path: the perturbation never landed, so NOTHING was proven about the pipeline. A + // mis-authored record must not masquerade as a working one — and it must not + // masquerade as a broken one either. + Err(HarnessError::InjectionInapplicable { cause, .. }) => { + traversal.stages = stages; + traversal.verdict = TraversalVerdict::InjectionInapplicable { cause }; + return Ok(traversal); + } + Err(other) => return Err(other), + }; + traversal.applied = injected.applied; + + let Some(observation) = injected.on_declared_channel(&record.channel) else { + traversal.stages = stages; + traversal.verdict = TraversalVerdict::FailedToSurface { + stage: Stage::Comparison, + cause: format!( + "the perturbation was applied to {} raw artifact(s) and produced no NEW \ + observation on `{}` (observed instead: {}). The baseline was clean, so a \ + difference that is present in the evidence and absent from the comparison \ + is the comparison losing it.", + injected.applied, + record.channel, + describe(&injected.result) + ), + }; + return Ok(traversal); + }; + let signature = observation.signature.clone(); + let finding_id = signature.finding_id(); + traversal.signature = Some(signature.id.clone()); + traversal.finding = Some(finding_id.clone()); + stages.push(StageOutcome { + stage: Stage::Comparison, + detail: format!( + "surfaced as signature `{}` at `{}.{}` ({} / {}) against a provably empty \ + baseline", + signature.id, + signature.channel, + signature.path, + signature.kind.as_str(), + signature.value_shape_class.as_str() + ), + }); + + // ---- minimization --------------------------------------------------- + let reduction_input = ReductionInput { + document: ctx.candidate.document.clone(), + mutations: ctx.candidate.mutations.clone(), + required_keys: ctx + .candidate + .branch + .map(|branch| required_keys(&ctx.grammar, branch)) + .unwrap_or_default(), + }; + let mut probe = InjectedProbe { + ctx, + record, + target: signature.clone(), + probes: 0, + }; + let reduction = shrink::reduce(&reduction_input, ctx.request.shrink_budget, &mut probe).await?; + if reduction.probes == 0 { + traversal.stages = stages; + traversal.verdict = TraversalVerdict::FailedToSurface { + stage: Stage::Minimization, + cause: "the reduction spent zero probes, so nothing was reduced and nothing was \ + tested; a reduction that never runs cannot establish that the signature \ + is stable under it" + .to_string(), + }; + return Ok(traversal); + } + stages.push(StageOutcome { + stage: Stage::Minimization, + detail: format!( + "reduced {} → {} node(s) in {} probe(s) via [{}]; isMinimal={}{}", + reduction.original_size, + reduction.reduced_size, + reduction.probes, + reduction.steps.join(", "), + reduction.is_minimal, + reduction + .not_minimal_reason + .as_ref() + .map(|r| format!(" ({r})")) + .unwrap_or_default(), + ), + }); + + // ---- candidate ------------------------------------------------------ + let witness = Witness { + id: Witness::derived_id(&ctx.campaign_id, &ctx.candidate.id), + campaign_id: ctx.campaign_id.clone(), + candidate_id: ctx.candidate.id.clone(), + minimal_input: reduction.document.clone(), + is_minimal: reduction.is_minimal, + reduction_steps: reduction.steps.clone(), + observed_values: ObservedValues { + deacon: observation.observed.deacon.clone(), + reference: observation.observed.reference.clone(), + }, + mutation_operators: ctx.candidate.operator_ids(), + }; + let candidates_root = ctx.request.report_root.join("proof").join("candidates"); + let dir = candidate::write(CandidateInputs { + finding_id: &finding_id, + signature: &signature, + observation, + campaign_id: &ctx.campaign_id, + seed_hex: &ctx.request.seed_hex, + lane: CampaignLane::Invoked.as_str(), + tier: CampaignTier::ConfigDifferential.as_str(), + profile: &ctx.request.profile, + pinned_input_set: &ctx.pins, + candidate_id: &ctx.candidate.id, + operations: &ctx.candidate.operations, + mutation_operators: &ctx.candidate.operator_ids(), + reduction: &reduction, + result: &injected.result, + // NOT a reference comparison, and the provenance says so in its own file rather + // than leaving a reviewer to infer it. + reference: ReferenceProvenance::InjectedSelfComparison { + injection: &record.id, + }, + registry: &ctx.registry, + root: &candidates_root, + })?; + if let Some(missing) = missing_parts(&dir) { + traversal.stages = stages; + traversal.verdict = TraversalVerdict::FailedToSurface { + stage: Stage::Candidate, + cause: format!( + "the reviewable candidate at {} is missing {missing}; FR-027's claim is that \ + reproducing needs only the candidate, and a missing part makes that false", + dir.display() + ), + }; + return Ok(traversal); + } + stages.push(StageOutcome { + stage: Stage::Candidate, + detail: format!( + "emitted all {} part(s) at {}", + CANDIDATE_PARTS.len(), + dir.display() + ), + }); + + // ---- classification ------------------------------------------------- + let mut finding = Finding::newly_admitted(signature, witness, &ctx.campaign_id); + // Any of the four promotable classifications would do; what is asserted is that the + // REAL state machine accepts exactly one and then permits promotion, not which one a + // reviewer would choose. A finding cannot tell you that, which is why promotion is a + // human act in the first place. + let classification = Classification::DeaconRegression; + match finding.triage( + classification, + Some("injected by the FR-042a pipeline proof"), + ) { + Ok(FindingState::Triaged) => stages.push(StageOutcome { + stage: Stage::Classification, + detail: format!( + "finding `{finding_id}` took classification `{}` and reached `triaged`", + classification.as_str() + ), + }), + Ok(other) => { + traversal.stages = stages; + traversal.verdict = TraversalVerdict::FailedToSurface { + stage: Stage::Classification, + cause: format!( + "triage left the finding in `{}` rather than `triaged`", + other.as_str() + ), + }; + return Ok(traversal); + } + Err(e) => { + traversal.stages = stages; + traversal.verdict = TraversalVerdict::FailedToSurface { + stage: Stage::Classification, + cause: format!("the state machine refused a classification: {e}"), + }; + return Ok(traversal); + } + } + + // ---- promotable ----------------------------------------------------- + // + // The stage passes when BOTH halves hold, and the second half is the one that matters: + // the machinery offers a skeleton and the state machine permits the transition, while + // the pre-flight still refuses the scaffolded record on every axis. Promotion stays a + // human act, and the proof demonstrates that by showing the gate holding rather than by + // walking through it. + let skeleton = match promote::promotion_skeleton(&finding) { + Ok(skeleton) => skeleton, + Err(e) => { + traversal.stages = stages; + traversal.verdict = TraversalVerdict::FailedToSurface { + stage: Stage::Promotable, + cause: format!("no promotion skeleton could be produced: {e}"), + }; + return Ok(traversal); + } + }; + let refusals = promote::validate_promotion(&finding, &skeleton["behavior"], None, &[]); + let axes_named = promote::BEHAVIOR_DISPOSITION_AXES.iter().all(|axis| { + refusals + .iter() + .any(|e| matches!(e, PromotionError::MissingDisposition { axis: a, .. } if a == axis)) + }); + if !axes_named { + traversal.stages = stages; + traversal.verdict = TraversalVerdict::FailedToSurface { + stage: Stage::Promotable, + cause: format!( + "the promotion pre-flight did not refuse the scaffolded record on every \ + disposition axis ({refusals:?}); a skeleton that passes validation is a \ + skeleton a machine could commit" + ), + }; + return Ok(traversal); + } + if let Err(e) = finding.promote("case-pipeline-proof-not-a-real-case") { + traversal.stages = stages; + traversal.verdict = TraversalVerdict::FailedToSurface { + stage: Stage::Promotable, + cause: format!("the state machine refused the promotion transition: {e}"), + }; + return Ok(traversal); + } + stages.push(StageOutcome { + stage: Stage::Promotable, + detail: format!( + "the state machine permits promotion and the pre-flight still refuses the \ + scaffold on all {} disposition axes plus its unresolved case — promotion \ + remains a human act", + promote::BEHAVIOR_DISPOSITION_AXES.len() + ), + }); + + traversal.stages = stages; + Ok(traversal) +} + +/// Establish a baseline and traverse every declared injection. +/// +/// Takes the [`RegressionHarness`] capability by reference so the *call site* shows the +/// process took it out: [`perturb_source`] fails closed without it (FR-070), and a proof +/// that could run without declaring it would be a proof of nothing. +pub async fn run( + request: &ProofRequest, + _capability: &RegressionHarness, +) -> Result { + let injections = proof_injections()?; + let ctx = establish(request).await?; + let mut traversals = Vec::with_capacity(injections.len()); + for record in &injections { + traversals.push(traverse(&ctx, record).await?); + } + Ok(ProofReport::build( + &request.seed_hex, + ctx.candidate_id(), + traversals, + )) +} + +// --------------------------------------------------------------------------- +// The injected comparison +// --------------------------------------------------------------------------- + +/// One injected comparison: the differential result plus how many artifacts were perturbed. +struct Injected { + result: DifferentialResult, + applied: usize, +} + +impl Injected { + /// The NEW observation on `channel`, if the comparison produced one. + fn on_declared_channel(&self, channel: &str) -> Option<&Observation> { + self.result + .new_observations() + .find(|o| o.signature.channel == channel) + } +} + +/// Run deacon over `document`, perturb the captured artifact at the **sealed** evidence-source +/// boundary, and relate the perturbed side to the unperturbed one. +/// +/// The perturbation is applied to a [`RunContext`] — the raw captured artifact — through +/// [`perturb_source`], whose signature is generic over the sealed +/// [`EvidenceSource`](crate::inject::EvidenceSource). Handing it an observer's output does +/// not compile, so this function cannot plant a difference downstream of the part it is +/// testing even by mistake (research D7). +async fn inject_and_compare( + ctx: &ProofContext, + record: &RegressionRecord, + document: &Value, +) -> Result { + // The candidate's own workspace when the document is the candidate's; a fresh one for a + // reduction proposal. Materializing the same tree shape either way is load-bearing: a + // probe workspace whose scaffolding differed would be measuring the scaffold. + let owned; + let (workspace, case): (&Path, String) = if document == &ctx.candidate.document { + (ctx.workspace.path(), ctx.candidate.id.clone()) + } else { + owned = materialize_document(document).map_err(|e| HarnessError::Report { + cause: format!("could not materialize a proof probe workspace: {e}"), + })?; + ( + owned.path(), + format!("{}-probe-{}", ctx.candidate.id, record.id), + ) + }; + + // ONE invocation, two sides. The unperturbed capture is the counterpart; the perturbed + // copy is the side under test. Running deacon twice would introduce a second source of + // difference into a comparison whose whole value is that it has exactly one. + // + // The baseline candidate reuses the evidence `establish` already proved clean, rather + // than re-running and re-normalizing it: the counterpart every injection is measured + // against must be the *same* evidence the empty-baseline check passed, or the guarantee + // that check establishes would not be the guarantee this comparison relies on. + let (outcome, reference) = if document == &ctx.candidate.document { + (ctx.baseline_outcome.clone(), ctx.baseline.clone()) + } else { + let outcome = run_deacon(&ctx.request, &case, workspace).await?; + let reference = side_evidence(&outcome, workspace, &ctx.request.report_root, &case); + (outcome, reference) + }; + + let mut source = RunContext::for_side(workspace.to_path_buf(), Side::Deacon); + source.record_outcome(PROOF_OPERATION, outcome); + // ===== THE SEALED BOUNDARY (research D7) ===== + let applied = perturb_source(&mut source, record)?; + let perturbed = source.outcome(PROOF_OPERATION).cloned().ok_or_else(|| { + HarnessError::InjectionInapplicable { + record: record.id.clone(), + cause: "the perturbed run context no longer carries the operation's outcome" + .to_string(), + } + })?; + let deacon = side_evidence(&perturbed, workspace, &ctx.request.report_root, &case); + + Ok(Injected { + result: differential::result_from_sides( + &case, + deacon, + reference, + // Deliberately EMPTY — see the module docs. A tolerance index built for + // deacon-vs-reference differences could suppress the planted one and report a + // working pipeline as broken. + &Characterization::default(), + false, + ), + applied, + }) +} + +/// The live reproduction predicate for the proof's minimization stage. +/// +/// The same shape as [`super::minimize::DifferentialProbe`] and for the same reason: the +/// shrinker takes its predicate as a parameter (research D4/D5), so the *strategy* stays +/// hermetic while the caller supplies the real one. What differs is only where the +/// counterpart comes from — deacon's own unperturbed run rather than the oracle. +struct InjectedProbe<'a> { + ctx: &'a ProofContext, + record: &'a RegressionRecord, + target: Signature, + probes: u64, +} + +impl ReproductionProbe for InjectedProbe<'_> { + type Error = HarnessError; + + async fn probe(&mut self, document: &Value) -> Result { + self.probes += 1; + let injected = match inject_and_compare(self.ctx, self.record, document).await { + Ok(injected) => injected, + // A proposal the perturbation cannot be applied to has said "this reduction + // removed the artifact the difference lives in", which is a fact about the + // PROPOSAL, not about the record — the record already landed on the baseline + // before minimization began. Rejecting the step is therefore correct, and + // treating it as a fault would make every reduction that removes the document a + // reported pipeline defect. This is the same accommodation + // `DifferentialProbe` makes for a per-probe timeout. + Err(HarnessError::InjectionInapplicable { cause, .. }) => { + tracing::debug!( + record = %self.record.id, + %cause, + "a reduction proposal removed the artifact the perturbation targets; the \ + step is rejected" + ); + return Ok(Reproduction::Absent); + } + Err(other) => return Err(other), + }; + + if injected + .result + .observations + .iter() + .any(|o| o.signature.id == self.target.id) + { + return Ok(Reproduction::Preserved); + } + let drifted: Vec = injected + .result + .new_observations() + .map(|o| o.signature.clone()) + .collect(); + Ok(if drifted.is_empty() { + Reproduction::Absent + } else { + Reproduction::Drifted(drifted) + }) + } +} + +// --------------------------------------------------------------------------- +// Small helpers +// --------------------------------------------------------------------------- + +/// Run deacon's `read-configuration` over `workspace`, capturing raw stdout/stderr. +async fn run_deacon( + request: &ProofRequest, + case: &str, + workspace: &Path, +) -> Result { + let folder = workspace.to_string_lossy().into_owned(); + let args: Vec<&str> = vec!["read-configuration", "--workspace-folder", &folder]; + let invocation = run_and_capture( + Side::Deacon, + PROOF_BINARY, + case, + &request.deacon_binary, + &args, + workspace, + request.bound, + &request.report_root, + ) + .await?; + Ok(ProcessOutcome { + exit_code: invocation.exit_code, + success: invocation.success, + stdout: invocation.stdout.clone(), + stderr: invocation.stderr.clone(), + failure_phase: None, + }) +} + +/// Build one side's evidence from a raw process outcome. +/// +/// **Both sides are normalized as [`Side::Deacon`]**, because both sides ARE deacon. The +/// single normalizer is side-asymmetric on purpose (024 T123), so labelling one copy of +/// deacon's output as the reference would apply rules written for a different serializer and +/// manufacture differences nothing found. The empty-baseline check in [`establish`] is +/// satisfied by construction while this holds — which is the point: it is the guard that +/// would catch a refactor making the two sides asymmetric again, not the argument for the +/// choice. +fn side_evidence( + outcome: &ProcessOutcome, + workspace: &Path, + report_root: &Path, + case: &str, +) -> SideEvidence { + let normalized = differential::structured_document_bytes( + outcome.success, + &outcome.stdout, + workspace, + Side::Deacon, + ); + let raw_dir = report_root.join("raw").join(PROOF_BINARY).join(case); + SideEvidence { + outcome: if outcome.success { + OutcomeClass::Accepted + } else { + OutcomeClass::Rejected + }, + exit_code: outcome.exit_code, + stdout_path: raw_dir.join("deacon.stdout"), + stderr_path: raw_dir.join("deacon.stderr"), + normalized, + } +} + +/// The candidate parts absent from `dir`, or `None` when all six are present. +fn missing_parts(dir: &Path) -> Option { + let missing: Vec<&str> = CANDIDATE_PARTS + .iter() + .copied() + .filter(|part| !dir.join(part).exists()) + .collect(); + if missing.is_empty() { + None + } else { + Some(missing.join(", ")) + } +} + +/// What the comparison DID see, for a failure message that names the shortfall rather than +/// only reporting one. +fn describe(result: &DifferentialResult) -> String { + if result.observations.is_empty() { + return "nothing at all".to_string(); + } + result + .observations + .iter() + .map(|o| { + format!( + "{}.{} [{}]", + o.signature.channel, + o.signature.path, + if o.is_new() { "new" } else { "characterized" } + ) + }) + .collect::>() + .join(", ") +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The proof's own records go through the strict `reg-` loader, so a mis-shaped + /// perturbation fails here rather than at run time — and every one names a channel the + /// configuration differential actually reads. + #[test] + fn the_declared_injections_load_and_target_channels_the_comparison_reads() { + let records = proof_injections().expect("the proof's injections load"); + assert_eq!(records.len(), 2); + let channels: Vec<&str> = records.iter().map(|r| r.channel.as_str()).collect(); + assert!(channels.contains(&"chan-structured-output")); + assert!(channels.contains(&"chan-exit-code")); + for record in &records { + assert!( + record.notes.is_some(), + "`{}` must say why its perturbation is a meaningful difference on its \ + channel", + record.id + ); + } + } + + /// The status rule, asserted against the same function the bin calls. + #[test] + fn the_exit_status_distinguishes_all_four_outcomes() { + let traversed = |id: &str| Traversal { + injection: id.to_string(), + channel: "chan-exit-code".to_string(), + applied: 1, + signature: Some("sig-1".to_string()), + finding: Some("fnd-1".to_string()), + stages: Vec::new(), + verdict: TraversalVerdict::Traversed, + }; + let mut failed = traversed("reg-b"); + failed.verdict = TraversalVerdict::FailedToSurface { + stage: Stage::Comparison, + cause: "swallowed".to_string(), + }; + let mut inapplicable = traversed("reg-c"); + inapplicable.verdict = TraversalVerdict::InjectionInapplicable { + cause: "never landed".to_string(), + }; + + assert_eq!( + ProofReport::build("0x1", "cnd-1", vec![traversed("reg-a")]).exit_status(), + 0 + ); + assert_eq!( + ProofReport::build("0x1", "cnd-1", vec![traversed("reg-a"), failed]).exit_status(), + 1, + "a difference that landed and did not traverse is a pipeline defect" + ); + assert_eq!( + ProofReport::build("0x1", "cnd-1", vec![traversed("reg-a"), inapplicable]) + .exit_status(), + 1, + "an injection that never landed must FAIL, never count as found-nothing" + ); + assert_eq!( + ProofReport::build("0x1", "cnd-1", Vec::new()).exit_status(), + 1, + "a proof over zero injections reports exactly what a fully-passing one does" + ); + } + + #[test] + fn the_report_renders_byte_stably_with_the_verdict_flattened() { + let report = ProofReport::build( + "0xseed", + "cnd-11111111", + vec![Traversal { + injection: "reg-proof-exit-code".to_string(), + channel: "chan-exit-code".to_string(), + applied: 1, + signature: Some("sig-abcd1234".to_string()), + finding: Some("fnd-abcd1234".to_string()), + stages: vec![StageOutcome { + stage: Stage::Comparison, + detail: "surfaced".to_string(), + }], + verdict: TraversalVerdict::Traversed, + }], + ); + let rendered = report.render().expect("renders"); + assert!(rendered.ends_with('\n')); + assert!(rendered.contains("\"verdict\": \"traversed\"")); + assert_eq!(rendered, report.render().expect("renders")); + let round_tripped: ProofReport = + serde_json::from_str(&rendered).expect("the report round-trips"); + assert_eq!(round_tripped, report); + } + + #[test] + fn every_stage_has_a_stable_spelling_and_the_list_is_the_traversal_order() { + assert_eq!( + Stage::all() + .iter() + .map(|s| s.as_str()) + .collect::>(), + vec![ + "generation", + "comparison", + "minimization", + "candidate", + "classification", + "promotable", + ], + "FR-042a names these stages in this order; the report is read against that list" + ); + } +} diff --git a/crates/parity-harness/src/lib.rs b/crates/parity-harness/src/lib.rs index 0d11e9fa..9e385997 100644 --- a/crates/parity-harness/src/lib.rs +++ b/crates/parity-harness/src/lib.rs @@ -22,6 +22,7 @@ use std::time::Duration; pub mod aggregate; pub mod compare; +pub mod discovery; pub mod driver; pub mod equivalence; pub mod evidence; @@ -181,6 +182,23 @@ pub enum HarnessError { )] DockerUnavailable { cause: String }, + /// The **network-backed** corpus tier could not reach the network at all + /// (025-exploratory-parity-discovery, US7). + /// + /// Distinct from an *unreachable entry*, which is a per-entry status the campaign + /// counts and reports (FR-052). This is the tier's prerequisite failing before any + /// entry was attempted, and it is the same fail-loud discipline + /// [`DockerMissing`](Self::DockerMissing) applies: a corpus campaign that quietly + /// reported zero entries because it had no network would be byte-identical to one in + /// which the whole ecosystem agreed with deacon. + #[error( + "the corpus tier requires network access and a working `git`, and neither could be \ + established: {cause}. Remedy: run this tier in the network-backed lane (or provide a \ + working `git` via DEACON_DISCOVERY_GIT) — a corpus campaign with no network reports \ + the same thing as one in which every entry agreed." + )] + NetworkUnavailable { cause: String }, + /// The runner requires Node (for the pinned oracle) but it is unavailable. #[error( "Node is unavailable for the conformance runner: {cause}. Remedy: install the Node \ @@ -272,11 +290,103 @@ pub enum HarnessError { budget and report as an unattributable lane failure." )] CaseTimeout { case: String, bound: Duration }, + + // --- Exploratory parity discovery (025-exploratory-parity-discovery, T005) ------ + // Every variant below is a MACHINERY failure, never a finding. That distinction is + // the whole exit-status contract (contracts/discovery-cli.md): a campaign that finds + // forty differences exits `0`; a campaign that could not verify its oracle exits + // non-zero. Anything that would make the status depend on WHAT was found is a defect, + // because a stochastic gate makes green non-reproducible. + /// A campaign that needs the pinned reference could not verify it (FR-003). + /// + /// Deliberately DISTINCT from [`OracleMissing`](Self::OracleMissing) / + /// [`OracleVersionMismatch`](Self::OracleVersionMismatch), which name *which* check + /// failed: this names the CONSEQUENCE for the campaign — no findings were produced + /// and none may be attributed to this run. Collapsing the two would let "we compared + /// against an unverified reference" read as "we compared and found nothing", which is + /// the exact confusion a discovery run must never create. + #[error( + "discovery campaign cannot run: the pinned oracle is unverified ({cause}). Remedy: \ + install the pinned `@devcontainers/cli` version — a campaign never reports findings \ + against an unverified reference, and never silently skips." + )] + OracleUnverified { cause: String }, + + /// One candidate exceeded its per-candidate bound and was discarded and counted + /// (60 s hermetic / 5 min container-backed). + /// + /// Discarding rather than failing the campaign is deliberate: one pathological + /// generated input must not consume the tier's whole budget, and the count is + /// reported so a *rising* discard rate is visible rather than silent. + #[error( + "discovery candidate `{candidate}` exceeded its {bound:?} bound and was discarded. \ + Remedy: none required — the candidate is counted in the campaign outcome; \ + investigate only if the discard rate rises." + )] + CandidateTimeout { candidate: String, bound: Duration }, + + /// Minimization ran out of shrink steps before reaching a minimal input (FR-022). + /// + /// The best reduction found is still emitted, with `isMinimal: false` and this + /// reason — never silently presented as minimal. A reviewer who believes an input is + /// minimal when it is not will look for the defect in the wrong place. + #[error( + "shrink budget exhausted for finding `{finding}` after {steps} step(s); the best \ + reduction is reported with `isMinimal: false`. Remedy: raise the per-finding shrink \ + budget if the input warrants it — a partially reduced input is never presented as \ + minimal." + )] + ShrinkBudgetExhausted { finding: String, steps: usize }, + + /// A fetched corpus entry's content digest disagrees with the recorded one (FR-051). + /// + /// Fails that entry loudly rather than comparing against unexpected content: an + /// unverified fetch means comparing against content nobody checked, and "expected to + /// be stable" is not "verified". + #[error( + "corpus entry `{entry}` content digest mismatch: recorded {expected}, fetched {actual}. \ + Remedy: investigate the upstream change and re-pin the entry deliberately — never \ + compare against content that does not match its recorded digest." + )] + CorpusDigestMismatch { + entry: String, + expected: String, + actual: String, + }, + + /// A corpus entry could not be fetched (FR-052). + /// + /// Reported as unreachable, NOT as "ran and found nothing" — the two are different + /// facts about the ecosystem and collapsing them makes the canary useless. + #[error( + "corpus entry `{entry}` is unreachable: {cause}. Remedy: check the network lane and the \ + pinned repository/commit — an unreachable entry is reported as unreachable, never as \ + an entry that ran and found nothing." + )] + CorpusUnreachable { entry: String, cause: String }, + + /// A declared metamorphic relation could not be evaluated (025 US6, FR-048). + /// + /// The **anti-inert** guarantee, and the reason this is an error rather than a skipped + /// relation: a relation the harness cannot apply reports nothing, and "reported + /// nothing" is byte-identical to "held". SC-011 requires zero inert relations, so a + /// relation with no transformation implementation, an inapplicable base fixture, or an + /// unparseable configuration must fail the run **naming the relation**, never quietly + /// contribute a pass. + #[error( + "metamorphic relation `{relation}` cannot be evaluated: {cause}. Remedy: implement or \ + repair its transformation — a relation that cannot be applied reports nothing, and \ + reporting nothing is indistinguishable from holding (SC-011)." + )] + RelationUnevaluable { relation: String, cause: String }, } /// Environment override for the report/artifact root (see [`report_root`]). pub const REPORT_DIR_ENV: &str = "DEACON_PARITY_REPORT_DIR"; +/// Environment override for the discovery artifact root (see [`discovery_report_root`]). +pub const DISCOVERY_REPORT_DIR_ENV: &str = "DEACON_DISCOVERY_REPORT_DIR"; + /// Absolute path to the workspace root, derived from this crate's /// `CARGO_MANIFEST_DIR` (`/crates/parity-harness`) so artifact paths are /// stable regardless of the (per-package) cargo-test working directory. @@ -307,6 +417,21 @@ pub fn report_root() -> PathBuf { workspace_root().join("target").join("parity") } +/// The discovery artifact root: `DEACON_DISCOVERY_REPORT_DIR` when set, else +/// `/target/discovery` (025-exploratory-parity-discovery). +/// +/// Separate from [`report_root`] rather than a subdirectory of it, because the two roots +/// answer to different rules. `target/parity` holds the evidence of a **gating** lane; +/// `target/discovery` holds the output of a lane that gates nothing and whose contents are +/// unreviewed candidates. Writing the second into the first would make a discovery artifact +/// look, to anyone reading `target/parity`, like part of a certification run. +pub fn discovery_report_root() -> PathBuf { + if let Some(dir) = std::env::var_os(DISCOVERY_REPORT_DIR_ENV) { + return PathBuf::from(dir); + } + workspace_root().join("target").join("discovery") +} + /// Process-unique suffix source for atomic temp files. static TEMP_SEQ: AtomicU64 = AtomicU64::new(0); @@ -360,6 +485,24 @@ mod tests { assert!(default.ends_with("target/parity") || std::env::var_os(REPORT_DIR_ENV).is_some()); } + #[test] + fn the_discovery_root_is_not_inside_the_parity_root() { + // `target/parity` holds the evidence of a gating lane; `target/discovery` holds + // unreviewed candidates from a lane that gates nothing. Nesting the second inside + // the first would make a discovery artifact read, to anyone looking at + // `target/parity`, as part of a certification run. + let discovery = discovery_report_root(); + assert!( + discovery.ends_with("target/discovery") + || std::env::var_os(DISCOVERY_REPORT_DIR_ENV).is_some() + ); + if std::env::var_os(REPORT_DIR_ENV).is_none() + && std::env::var_os(DISCOVERY_REPORT_DIR_ENV).is_none() + { + assert!(!discovery.starts_with(report_root())); + } + } + #[tokio::test] async fn atomic_write_replaces_shorter_payload() { let dir = tempfile::tempdir().expect("tempdir"); diff --git a/crates/parity-harness/src/prereq.rs b/crates/parity-harness/src/prereq.rs index efa45a4d..c145a0cb 100644 --- a/crates/parity-harness/src/prereq.rs +++ b/crates/parity-harness/src/prereq.rs @@ -100,26 +100,54 @@ pub async fn deacon_binary() -> Result { return Err(HarnessError::FixtureMissing { path }); } + let path = cargo_bin("deacon", "deacon").await?; + eprintln!("deacon binary under test: {}", path.display()); + Ok(path) +} + +/// Build one workspace binary and take its path from **cargo's own artifact report**. +/// +/// The generalization of [`deacon_binary`]'s rule, and it exists for the same reason: a +/// caller outside the owning package cannot expand `env!("CARGO_BIN_EXE_")`, and the +/// tempting fallback — look under `target/{debug,release}` and use whichever file exists — +/// silently judges whatever artifact happened to be lying there. That was a real defect with +/// teeth (023 T115): a three-day-old `target/release/deacon` was compared against the +/// current oracle and manufactured a verdict out of a binary nobody was testing. +/// +/// One implementation rather than one per caller: a second copy of a "do not guess the +/// binary" rule is the copy that rots. +/// +/// `bin` is the **target** name (what `--bin` takes and what the artifact reports), which is +/// not always the package name — `parity-harness` owns `discovery-campaign`. +pub async fn cargo_bin(package: &str, bin: &str) -> Result { let output = tokio::process::Command::new("cargo") - .args(["build", "-p", "deacon", "--message-format", "json"]) + .args([ + "build", + "-p", + package, + "--bin", + bin, + "--message-format", + "json", + ]) .current_dir(crate::workspace_root()) .kill_on_drop(true) .output() .await .map_err(|e| HarnessError::Report { - cause: format!("could not build the deacon binary under test: {e}"), + cause: format!("could not build `{bin}` from `{package}`: {e}"), })?; if !output.status.success() { return Err(HarnessError::Report { cause: format!( - "building the deacon binary under test failed ({}): {}", + "building `{bin}` from `{package}` failed ({}): {}", output.status, String::from_utf8_lossy(&output.stderr) ), }); } - // cargo emits one JSON object per line; the `deacon` bin artifact carries the path. + // cargo emits one JSON object per line; the bin artifact carries the path. let mut executable: Option = None; for line in String::from_utf8_lossy(&output.stdout).lines() { let Ok(value) = serde_json::from_str::(line) else { @@ -128,7 +156,7 @@ pub async fn deacon_binary() -> Result { if value.get("reason").and_then(|v| v.as_str()) != Some("compiler-artifact") { continue; } - if value.pointer("/target/name").and_then(|v| v.as_str()) != Some("deacon") { + if value.pointer("/target/name").and_then(|v| v.as_str()) != Some(bin) { continue; } if let Some(path) = value.get("executable").and_then(|v| v.as_str()) { @@ -136,13 +164,12 @@ pub async fn deacon_binary() -> Result { } } - let path = executable.ok_or_else(|| HarnessError::Report { - cause: "cargo reported no `deacon` executable artifact — refusing to guess which \ - binary to judge" - .to_string(), - })?; - eprintln!("deacon binary under test: {}", path.display()); - Ok(path) + executable.ok_or_else(|| HarnessError::Report { + cause: format!( + "cargo reported no `{bin}` executable artifact for `{package}` — refusing to \ + guess which binary to judge" + ), + }) } #[cfg(test)] diff --git a/crates/parity-harness/src/registry.rs b/crates/parity-harness/src/registry.rs index a71475f3..1057bb19 100644 --- a/crates/parity-harness/src/registry.rs +++ b/crates/parity-harness/src/registry.rs @@ -18,7 +18,8 @@ use std::path::{Path, PathBuf}; use crate::HarnessError; pub use deacon_conformance::parity_corpus::{ - Corpus, LiveBinary, LiveKind, ParityCorpusError, ParityRegistry, REGISTRY_JSON, + Corpus, DiscoveryBinary, DiscoveryRole, LiveBinary, LiveKind, ParityCorpusError, + ParityRegistry, REGISTRY_JSON, }; /// Hermetic harness self-test binaries that intentionally carry the `parity_` @@ -126,28 +127,51 @@ pub fn check_test_files(registry: &ParityRegistry, tests_dir: &Path) -> Vec Vec { let mut problems = Vec::new(); - let selected: std::collections::HashSet = - extract_binary_eq_tokens(filter_expr).into_iter().collect(); + + let selects = |name: &str, problems: &mut Vec| -> Option { + match filter_selects(filter_expr, name) { + Ok(v) => Some(v), + Err(e) => { + problems.push(format!( + "[profile.parity] default-filter could not be evaluated for `{name}`: {e}" + )); + None + } + } + }; for name in registry.live_names() { - if !selected.contains(name) { + if selects(name, &mut problems) == Some(false) { problems.push(format!( "[profile.parity] filter does not select live binary `{name}`" )); } } for name in ®istry.internal_consistency_binaries { - if selected.contains(name.as_str()) { + if selects(name, &mut problems) == Some(true) { problems.push(format!( "[profile.parity] filter selects internal-consistency binary `{name}` (it must not)" )); } } + let live: std::collections::HashSet<&str> = registry.live_names().into_iter().collect(); - for name in &selected { - if !live.contains(name.as_str()) { + let mentioned: std::collections::BTreeSet = + extract_binary_eq_tokens(filter_expr).into_iter().collect(); + for name in &mentioned { + if !live.contains(name.as_str()) && selects(name, &mut problems) == Some(true) { problems.push(format!( "[profile.parity] filter selects `{name}`, which is not a registered live binary" )); @@ -159,9 +183,10 @@ pub fn check_parity_profile_filter(registry: &ParityRegistry, filter_expr: &str) /// Full `.config/nextest.toml` cross-check (research D5; FR-013, FR-014): /// /// - `[profile.parity]` selects EXACTLY the live set and none of the -/// internal-consistency binaries (delegates to [`check_parity_profile_filter`], -/// valid because the parity profile's `default-filter` is a pure `binary(=…)` -/// allow-list); +/// internal-consistency binaries (delegates to [`check_parity_profile_filter`], which +/// EVALUATES the expression — the parity filter is an allow-list with an explicit +/// exclusion group appended, not a pure `binary(=…)` union, so a token-matching check +/// would misread the exclusion as a selection); /// - NO OTHER profile's `default-filter` selects any live parity binary — the /// truthful-by-non-selection invariant (FR-014). This is evaluated by /// [`filter_selects`] over each profile's filter expression, so an exclusion @@ -213,6 +238,238 @@ pub fn check_nextest_profiles( problems } +// --------------------------------------------------------------------------- +// 025-exploratory-parity-discovery — the discovery lane's wiring checks +// (T054/T056; FR-055, FR-057; research D9) +// --------------------------------------------------------------------------- + +/// The one nextest profile permitted to select a live discovery campaign binary. +pub const DISCOVERY_PROFILE: &str = "discovery"; + +/// Every profile a pull request can run through. None of them may select a live +/// discovery binary — the discovery lane gates nothing, so a green PR run must never +/// imply a campaign ran (FR-055/FR-057). +/// +/// `parity` is in this list deliberately. It is not a pull-request-only lane, but it *is* +/// a lane whose result people read as a verdict, and the exclusion is about the two lanes +/// answering different questions on different budgets — not about cost. +pub const PULL_REQUEST_PROFILES: &[&str] = &[ + "default", + "dev-fast", + "full", + "ci", + "mvp-integration", + "parity", +]; + +/// The profiles a hermetic discovery guard MUST be selected by. +/// +/// Only the two fast lanes are required. `full` / `ci` select the guards today as a +/// consequence of their `not (…)` filters, but `mvp-integration` and `parity` are narrow +/// allow-lists that legitimately do not — demanding selection there would force unrelated +/// binaries into two curated lanes to satisfy a rule about guards. +pub const GUARD_REQUIRED_PROFILES: &[&str] = &["default", "dev-fast"]; + +/// Directories that may hold a discovery test binary, checked for *unregistered* sources +/// regardless of what the registry declares. +/// +/// The registry's own `tests_dir` values are unioned in, so declaring a third location +/// automatically extends the scan. These two are listed anyway because the file→registry +/// direction has to work when the registry is the thing that is wrong: a binary dropped +/// into a crate nobody declared is exactly the drift this direction exists to catch. +const DISCOVERY_TEST_DIRS: &[&str] = &["crates/deacon/tests", "crates/conformance/tests"]; + +/// Bidirectional file↔registry match for the discovery lane (FR-057). +/// +/// Registry → file: every registered discovery binary has a source file at its declared +/// `tests_dir`. File → registry: every `discovery_*.rs` under any candidate test +/// directory is registered. Returns human-readable problems (empty = OK). +pub fn check_discovery_files(registry: &ParityRegistry, workspace_root: &Path) -> Vec { + let mut problems = Vec::new(); + + for binary in ®istry.discovery_binaries { + let source = workspace_root + .join(&binary.tests_dir) + .join(format!("{}.rs", binary.name)); + if !source.is_file() { + problems.push(format!( + "registered discovery binary `{}` has no source file {}", + binary.name, + source.display() + )); + } + } + + let registered: std::collections::HashSet<&str> = + registry.discovery_names().into_iter().collect(); + let mut dirs: std::collections::BTreeSet<&str> = DISCOVERY_TEST_DIRS.iter().copied().collect(); + for binary in ®istry.discovery_binaries { + dirs.insert(binary.tests_dir.as_str()); + } + for dir in dirs { + let path = workspace_root.join(dir); + let rd = match std::fs::read_dir(&path) { + Ok(rd) => rd, + Err(e) => { + problems.push(format!("could not read discovery test dir {path:?}: {e}")); + continue; + } + }; + for entry in rd.filter_map(Result::ok) { + let file = entry.file_name(); + let file = file.to_string_lossy(); + let Some(stem) = file.strip_suffix(".rs") else { + continue; + }; + if stem.starts_with("discovery_") && !registered.contains(stem) { + problems.push(format!( + "{dir}/{file} is a discovery test binary but is not registered in \ + fixtures/parity-corpus/registry.json discovery_binaries — an \ + unregistered binary has no declared lane, so nothing checks which \ + profiles select it" + )); + } + } + } + + problems +} + +/// Cross-check `.config/nextest.toml` for the discovery lane (FR-057). +/// +/// Four claims, every one reached by **evaluating** the filter expression rather than +/// token-matching it (the profiles name discovery binaries in order to *exclude* them, +/// which a token match would read as a selection): +/// +/// 1. `[profile.discovery]` exists and declares a `default-filter` — without one it +/// selects every binary in the workspace. +/// 2. It selects every `live` discovery binary. +/// 3. It selects **no** `guard` discovery binary. This is research D9's `discovery_*` +/// glob mistake stated as an assertion: the glob would capture the guards and silently +/// remove them from the fast lane, and the symptom is invisible until it matters. +/// 4. It names no binary it does not select-and-own — the symmetric counterpart of +/// [`check_parity_profile_filter`]'s "selects `{name}`, which is not a registered live +/// binary". Without it the allow-list is only checked for what it is *missing*, and an +/// unrelated binary added to it would run under a 35-minute stochastic profile with +/// nothing objecting. +/// 5. No pull-request profile selects a `live` discovery binary, and both fast lanes +/// select every `guard`. +pub fn check_discovery_profiles( + registry: &ParityRegistry, + profiles: &NextestProfiles, +) -> Vec { + let mut problems = Vec::new(); + + let live: Vec<&str> = registry + .discovery_of_role(DiscoveryRole::Live) + .into_iter() + .map(|b| b.name.as_str()) + .collect(); + let guards: Vec<&str> = registry + .discovery_of_role(DiscoveryRole::Guard) + .into_iter() + .map(|b| b.name.as_str()) + .collect(); + + fn evaluate(profile: &str, expr: &str, name: &str, problems: &mut Vec) -> Option { + match filter_selects(expr, name) { + Ok(v) => Some(v), + Err(e) => { + problems.push(format!( + "[profile.{profile}] default-filter could not be evaluated for `{name}`: {e}" + )); + None + } + } + } + + match profiles.default_filters.get(DISCOVERY_PROFILE) { + None => problems.push(format!( + "nextest.toml has no [profile.{DISCOVERY_PROFILE}] — the discovery lane has no \ + entry point, so its binaries either never run or run in a lane that gates" + )), + Some(None) => problems.push(format!( + "[profile.{DISCOVERY_PROFILE}] declares no default-filter, so it selects every \ + binary in the workspace instead of exactly the discovery campaigns" + )), + Some(Some(expr)) => { + for name in &live { + if evaluate(DISCOVERY_PROFILE, expr, name, &mut problems) == Some(false) { + problems.push(format!( + "[profile.{DISCOVERY_PROFILE}] does not select live discovery binary \ + `{name}` — it is the only lane that may, so nothing would run it" + )); + } + } + for name in &guards { + if evaluate(DISCOVERY_PROFILE, expr, name, &mut problems) == Some(true) { + problems.push(format!( + "[profile.{DISCOVERY_PROFILE}] captures the hermetic guard `{name}`. \ + The filter must be an explicit `binary(=…)` allow-list, never a \ + `discovery_*` glob: the glob silently removes the guards from the \ + fast lane, which is the mistake research D9 exists to prevent" + )); + } + } + // Anything else the filter NAMES and SELECTS. The candidate set has to come + // from the expression's own `binary(=…)` tokens: there is no other way to + // discover a binary the filter mentions but the registry does not know. + let known: std::collections::HashSet<&str> = live.iter().copied().collect(); + for name in extract_binary_eq_tokens(expr) { + if !known.contains(name.as_str()) + && evaluate(DISCOVERY_PROFILE, expr, &name, &mut problems) == Some(true) + { + problems.push(format!( + "[profile.{DISCOVERY_PROFILE}] selects `{name}`, which is not a \ + registered live discovery binary — the discovery lane runs under a \ + long stochastic budget and gates nothing, so admitting an unrelated \ + binary to it both distorts that binary's meaning and hides it from \ + the lane that should be asserting on it" + )); + } + } + } + } + + for profile in PULL_REQUEST_PROFILES { + let Some(filter) = profiles.default_filters.get(*profile) else { + problems.push(format!( + "[profile.{profile}] is missing from nextest.toml, so the discovery lane's \ + exclusion from it cannot be verified" + )); + continue; + }; + let Some(expr) = filter.as_deref() else { + problems.push(format!( + "[profile.{profile}] has no default-filter, so it selects every binary \ + including the live discovery campaigns" + )); + continue; + }; + for name in &live { + if evaluate(profile, expr, name, &mut problems) == Some(true) { + problems.push(format!( + "[profile.{profile}] selects live discovery binary `{name}` — a green \ + pull-request run must never imply a campaign ran (FR-055/FR-057)" + )); + } + } + if GUARD_REQUIRED_PROFILES.contains(profile) { + for name in &guards { + if evaluate(profile, expr, name, &mut problems) == Some(false) { + problems.push(format!( + "[profile.{profile}] does not select the hermetic guard `{name}` — a \ + guard that does not run in the fast lane is a guard nobody notices \ + going stale" + )); + } + } + } + } + + problems +} + /// Extract each `binary(=NAME)` token from a nextest filter expression. fn extract_binary_eq_tokens(expr: &str) -> Vec { let mut out = Vec::new(); @@ -536,6 +793,50 @@ mod tests { ); } + #[test] + fn naming_a_binary_in_order_to_exclude_it_is_not_selecting_it() { + // 025 T007 puts the discovery campaign binaries in the parity filter's `not` + // group, so the parity lane's non-selection of them is structural rather than + // incidental. A token-matching check would read that as "the parity profile + // selects `discovery_campaign`" and fail a correct file — the check has to + // understand the operator it is reading. + let reg = ParityRegistry::load().expect("embedded registry must parse"); + let good = reg + .live_names() + .iter() + .map(|n| format!("binary(={n})")) + .collect::>() + .join(" | "); + + let excluding = format!( + "({good}) & not (binary(=discovery_campaign) | binary(=discovery_metamorphic))" + ); + assert!( + check_parity_profile_filter(®, &excluding).is_empty(), + "an explicit exclusion must not read as a selection: {:?}", + check_parity_profile_filter(®, &excluding) + ); + + // Positively selecting an unregistered binary is still flagged — the exclusion + // tolerance must not become a blanket amnesty for unknown names. + let selecting = format!("{good} | binary(=discovery_campaign)"); + let problems = check_parity_profile_filter(®, &selecting); + assert!( + problems.iter().any(|p| p.contains("discovery_campaign")), + "got: {problems:?}" + ); + + // An exclusion must not hide a MISSING live binary either. + let missing_live = format!("({good}) & not (binary(={}))", reg.live_names()[0]); + let problems = check_parity_profile_filter(®, &missing_live); + assert!( + problems + .iter() + .any(|p| p.contains("does not select live binary")), + "got: {problems:?}" + ); + } + #[test] fn check_test_files_against_real_tree() { let reg = ParityRegistry::load().expect("embedded registry must parse"); @@ -621,6 +922,158 @@ mod tests { ); } + /// A registry carrying one live campaign and one guard, for the discovery checks. + fn discovery_registry() -> ParityRegistry { + ParityRegistry::parse( + r#"{ + "live_binaries": [], + "internal_consistency_binaries": [], + "corpora": [], + "discovery_binaries": [ + { "name": "discovery_campaign", "role": "live", "tests_dir": "crates/deacon/tests", "docker_required": true }, + { "name": "discovery_hermetic", "role": "guard", "tests_dir": "crates/deacon/tests", "docker_required": false } + ] + }"#, + ) + .expect("synthetic discovery registry parses") + } + + fn profiles_from(pairs: &[(&str, &str)]) -> NextestProfiles { + let mut profiles = NextestProfiles::default(); + for (name, expr) in pairs { + profiles + .default_filters + .insert((*name).to_string(), Some((*expr).to_string())); + } + profiles + } + + #[test] + fn the_discovery_allow_list_must_not_capture_a_guard() { + // Research D9's mistake, stated as a test. A `discovery_*` glob selects both the + // campaign and the guard, and the symptom — a hermetic guard silently absent from + // the fast lane — is invisible until it matters. + let reg = discovery_registry(); + let mut pairs = vec![("discovery", "binary(#discovery_*)")]; + for profile in PULL_REQUEST_PROFILES { + pairs.push((profile, "not (binary(=discovery_campaign))")); + } + let problems = check_discovery_profiles(®, &profiles_from(&pairs)); + assert!( + problems + .iter() + .any(|p| p.contains("discovery_hermetic") && p.contains("allow-list")), + "a glob filter must be reported as capturing the guard, got: {problems:?}" + ); + } + + #[test] + fn a_pull_request_profile_selecting_a_campaign_is_flagged() { + let reg = discovery_registry(); + let mut pairs = vec![("discovery", "binary(=discovery_campaign)")]; + for profile in PULL_REQUEST_PROFILES { + // `dev-fast` "forgets" the exclusion — the exact drift FR-057 exists to catch. + pairs.push(( + profile, + if *profile == "dev-fast" { + "not (binary(#smoke_*))" + } else { + "not (binary(=discovery_campaign))" + }, + )); + } + let problems = check_discovery_profiles(®, &profiles_from(&pairs)); + assert!( + problems + .iter() + .any(|p| p.contains("dev-fast") && p.contains("discovery_campaign")), + "got: {problems:?}" + ); + } + + #[test] + fn a_fast_lane_dropping_a_guard_is_flagged() { + let reg = discovery_registry(); + let mut pairs = vec![("discovery", "binary(=discovery_campaign)")]; + for profile in PULL_REQUEST_PROFILES { + pairs.push(( + profile, + if *profile == "default" { + "not (binary(=discovery_campaign) | binary(=discovery_hermetic))" + } else { + "not (binary(=discovery_campaign))" + }, + )); + } + let problems = check_discovery_profiles(®, &profiles_from(&pairs)); + assert!( + problems + .iter() + .any(|p| p.contains("default") && p.contains("discovery_hermetic")), + "got: {problems:?}" + ); + } + + #[test] + fn an_unrelated_binary_in_the_discovery_allow_list_is_flagged() { + // The allow-list must be checked for what it ADDS, not only for what it is + // missing: the discovery lane runs under a 35-minute stochastic budget and gates + // nothing, so a binary quietly moved into it stops being asserted on anywhere. + let reg = discovery_registry(); + let mut pairs = vec![( + "discovery", + "binary(=discovery_campaign) | binary(=integration_up_traditional)", + )]; + for profile in PULL_REQUEST_PROFILES { + pairs.push((profile, "not (binary(=discovery_campaign))")); + } + let problems = check_discovery_profiles(®, &profiles_from(&pairs)); + assert!( + problems + .iter() + .any(|p| p.contains("integration_up_traditional")), + "got: {problems:?}" + ); + } + + #[test] + fn the_real_nextest_toml_wires_the_discovery_lane() { + let reg = ParityRegistry::load().expect("embedded registry must parse"); + let toml_text = + std::fs::read_to_string(crate::workspace_root().join(".config/nextest.toml")) + .expect("read nextest.toml"); + let profiles = parse_nextest_profiles(&toml_text).expect("parse nextest.toml"); + let problems = check_discovery_profiles(®, &profiles); + assert!( + problems.is_empty(), + "discovery lane wiring problems: {problems:?}" + ); + } + + #[test] + fn discovery_files_match_the_registry_both_directions() { + let reg = ParityRegistry::load().expect("embedded registry must parse"); + let problems = check_discovery_files(®, &crate::workspace_root()); + assert!( + problems.is_empty(), + "registry ↔ discovery source mismatch: {problems:?}" + ); + + // An unregistered source is reported: drop the guard from the registry and the + // file→registry direction must notice the file that is still on disk. + let mut trimmed = reg.clone(); + trimmed + .discovery_binaries + .retain(|b| b.name != "discovery_hermetic"); + let problems = check_discovery_files(&trimmed, &crate::workspace_root()); + assert!( + problems + .iter() + .any(|p| p.contains("discovery_hermetic") && p.contains("not registered")), + "got: {problems:?}" + ); + } + #[test] fn check_nextest_profiles_flags_leaked_live_binary() { let reg = ParityRegistry::load().expect("embedded registry must parse"); diff --git a/crates/parity-harness/tests/aggregator.rs b/crates/parity-harness/tests/aggregator.rs index 5d452691..aa3868ee 100644 --- a/crates/parity-harness/tests/aggregator.rs +++ b/crates/parity-harness/tests/aggregator.rs @@ -51,6 +51,9 @@ fn registry() -> ParityRegistry { path: "fixtures/parity-corpus".into(), min_cases: CORPUS_MIN, }], + // The aggregator reports on parity binaries only; the discovery lane is a + // separate surface with its own report (025 T060). + discovery_binaries: vec![], } } diff --git a/crates/parity-harness/tests/injection_faults.rs b/crates/parity-harness/tests/injection_faults.rs index ffe3c049..33b025f0 100644 --- a/crates/parity-harness/tests/injection_faults.rs +++ b/crates/parity-harness/tests/injection_faults.rs @@ -491,8 +491,33 @@ fn a_dead_observer_is_reported_inert_rather_than_falsely_detected() { // T131 — an ordinary conformance run cannot apply a regression (FR-070) // --------------------------------------------------------------------------------- -/// Every source file that could reach the injector, and the one file allowed to. -const CAPABILITY_OWNER: &str = "coverage-regressions.rs"; +/// The **complete** set of files permitted to take out the injection capability, sorted. +/// +/// An explicit allow-list rather than a single owner, because there is now more than one +/// program whose entire purpose is to inject — but the property FR-070 protects is +/// unchanged and the assertion is still exact equality, so a *new* declarer fails this +/// guard until someone justifies it here. +/// +/// Each entry earns its place by being a dedicated injection program, never a driver that +/// happens to want the capability: +/// +/// | File | Why it may declare | +/// |---|---| +/// | `coverage-regressions.rs` | the 024 injected-regression harness: proves each declared channel can fail | +/// | `discovery-proof.rs` | the 025 FR-042a pipeline proof: plants a known difference and requires it to traverse the pipeline | +/// | `injection_faults.rs` | this guard, which must exercise the capability to assert how it behaves | +/// | `discovery_hermetic.rs` | the FR-042a acceptance test, which drives the proof in-process | +/// +/// What must never appear here is a conformance *driver* +/// (`parity_conformance_runner` / `parity_conformance_docker`) or any harness module they +/// go through — a program that can perturb its own evidence can report anything. +/// [`the_ordinary_run_can_only_reach_the_inert_hook`] asserts that half separately. +const CAPABILITY_OWNERS: &[&str] = &[ + "coverage-regressions.rs", + "discovery-proof.rs", + "discovery_hermetic.rs", + "injection_faults.rs", +]; /// Files whose call graph is the ORDINARY conformance run: the two driver test binaries /// and the harness modules they go through. @@ -519,12 +544,16 @@ fn rust_sources(dir: &Path, out: &mut Vec) { } } -/// STRUCTURAL, not conventional: the capability that enables injection is taken out in -/// exactly one program, so no other program's call graph can reach the injector at all. +/// STRUCTURAL, not conventional: the capability that enables injection is taken out only +/// in the programs whose whole purpose is to inject, so no other program's call graph can +/// reach the injector at all. /// /// The alternative — "the drivers just don't do that" — is a comment, and a comment does /// not survive the next person wiring a convenience helper. Enabling injection requires -/// `RegressionHarness::declare`, and this asserts that call exists in one file only. +/// `RegressionHarness::declare`, and this asserts that call exists **only** in +/// [`CAPABILITY_OWNERS`]. The assertion is exact equality in both directions: a new +/// declarer fails, and an owner that stops declaring fails too, because a listed program +/// that no longer injects is an entry nobody would notice going stale. #[test] fn only_the_coverage_regressions_bin_can_enable_injection() { let root = workspace_root(); @@ -555,13 +584,13 @@ fn only_the_coverage_regressions_bin_can_enable_injection() { declaring.dedup(); assert_eq!( declaring, - vec![ - CAPABILITY_OWNER.to_string(), - "injection_faults.rs".to_string() - ], - "the injection capability must be taken out by the `coverage-regressions` bin and \ - by this guard alone — any other program that declares it can perturb its own \ - evidence (FR-070)" + CAPABILITY_OWNERS + .iter() + .map(|s| (*s).to_string()) + .collect::>(), + "the injection capability must be taken out ONLY by the dedicated injection \ + programs listed in `CAPABILITY_OWNERS` — any other program that declares it can \ + perturb its own evidence and then report anything (FR-070)" ); } diff --git a/fixtures/parity-corpus/README.md b/fixtures/parity-corpus/README.md index ed127485..336c7502 100644 --- a/fixtures/parity-corpus/README.md +++ b/fixtures/parity-corpus/README.md @@ -25,7 +25,6 @@ Re-verification of that migration uses **git history**, not a retained copy: the |---|---| | `oracle.json` | the pinned `@devcontainers/cli` version every live comparison verifies against | | `registry.json` | the surviving live-binary enumeration (`parity_build`, `parity_exec`, `parity_up_exec`, `parity_observable_state`, `parity_state_diff`, `parity_conformance_runner`) plus the internal-consistency binaries. `corpora` is now empty — the corpora retired with the binaries that drove them | -| `fetch_realworld_corpus.py` | a fetch utility, never a comparison runner; its 33 pinned entries are recorded as `external-corpus-entry` baseline units and covered by `res-realworld-corpus-not-vendored` (research D8) | | `errors/README.md` | prose describing the error-decision contract, which the declarative `case-errors-decl-*` cases now implement | | `REPORT.md` | the historical findings log | @@ -53,25 +52,50 @@ The single normalization/equivalence definition lives in discovery rule in `crates/conformance/src/parity_corpus.rs`, re-exported by `crates/parity-harness/src/registry.rs`. -## Tier 3 — pinned real-world corpus fetch +## Tier 3 — pinned real-world corpus — MOVED -```bash -python3 fixtures/parity-corpus/fetch_realworld_corpus.py --clean --dest /tmp/realworld-corpus -``` +`fetch_realworld_corpus.py` was **deleted** in 025-exploratory-parity-discovery (US7, +T109). Its 33 pinned entries now live in `conformance/discovery/corpus.json`, a +Rust-owned strict-JSON manifest, and the fetch lives in +`crates/parity-harness/src/discovery/corpus_fetch.rs`. + +The script was not retired for tidiness. Two concrete reasons: + +1. **The immutable-reference check has to be hermetic.** FR-050 — reject any branch, tag, + `HEAD`, or `latest` — is a property of the *manifest*, not of a fetch: nothing needs to + be retrieved to know that `main` names different content tomorrow. In a Python tuple + nothing checked it; as strict JSON, violation class **D4** rejects it on every pull + request with no network at all (research D8). +2. **Two statements of one manifest drift.** The frozen `realworld::` baseline units + are derived from the entry names, and the corpus tier compares against the entry pins. + With two copies, one of those two would eventually be reading the other's stale twin. + +Its documented workflow had also stopped existing: the docstring told you to copy a +fetched snapshot into the corpus root and run `parity_corpus_tier1` — deleted by 023, as +was `parity_corpus_merged`, along with the corpus root itself. A +script whose instructions name removed binaries is not an exploratory aid. -`fetch_realworld_corpus.py` (a fetch utility, NOT a comparison runner — it makes -no pass/fail claim) downloads a pinned set of public workspace snapshots into -`/tmp/realworld-corpus` without vendoring third-party content into this -repository. The current manifest mixes: +What replaced it is strictly more, not less: -- `devcontainers/images` workspace subtrees -- two compose-based `devcontainers/templates` workspace subtrees -- `microsoft/vscode-remote-try-*` sample repos -- a couple of small real OSS repos with checked-in devcontainers +- the entries are validated (**D4**) rather than merely written down; +- each entry carries a **content digest**, recorded at first materialization and verified + on every later fetch (FR-051) — the Python fetcher verified nothing; +- an unreachable entry is reported as unreachable, never as "ran and found nothing" + (FR-052); +- the fetch uses a blob-filtered partial clone plus a sparse checkout instead of the + GitHub contents API, so it needs no `gh`, no token, and no rate-limit budget. + +Corpus content is still **never vendored** (FR-053): the manifest records provenance, not +bytes. To run the canary: + +```bash +cargo run -p parity-harness --bin discovery-campaign -- \ + --seed 0x… --tier corpus --budget-seconds 1800 --lane invoked +``` -The fetched corpus includes a `_manifest.json` recording the exact repos and -commit SHAs used for the run. It is for manual exploration; the pinned, in-repo -corpus above is what the nextest runners drive. +It also runs weekly in `.github/workflows/discovery.yml`, on a schedule of its own — +an ecological canary that runs only when someone remembers to invoke it cannot warn +anyone (FR-056, research D10). ## Tier 2 — up/build (Docker) diff --git a/fixtures/parity-corpus/REPORT.md b/fixtures/parity-corpus/REPORT.md index 801d793e..c37be814 100644 --- a/fixtures/parity-corpus/REPORT.md +++ b/fixtures/parity-corpus/REPORT.md @@ -528,8 +528,10 @@ non-string coercion, and the invalid-JSON error. (`crates/core/src/secrets.rs`.) ## Round 7 — real-world corpus sweep (pinned upstream repos) -First sweep using the pinned real-world fetcher -(`fixtures/parity-corpus/fetch_realworld_corpus.py`), comparing deacon against +First sweep using the pinned real-world fetcher (then +`fixtures/parity-corpus/fetch_realworld_corpus.py`, deleted in 025 US7 — the manifest is +now `conformance/discovery/corpus.json` and the fetch is +`parity_harness::discovery::corpus_fetch`), comparing deacon against `@devcontainers/cli` v0.87.0 on actual `devcontainers/images`, `devcontainers/templates`, and `microsoft/*` devcontainers. Four real bugs fixed; three diff clusters classified as non-bugs (cold-image caveat / diff --git a/fixtures/parity-corpus/fetch_realworld_corpus.py b/fixtures/parity-corpus/fetch_realworld_corpus.py deleted file mode 100644 index 9034b130..00000000 --- a/fixtures/parity-corpus/fetch_realworld_corpus.py +++ /dev/null @@ -1,465 +0,0 @@ -#!/usr/bin/env python3 -"""Fetch a pinned real-world devcontainer corpus into /tmp/realworld-corpus. - -This materializes reproducible workspace snapshots from public GitHub repos -without vendoring any third-party content into the repository. Each entry is a -workspace root. - -The Tier-1 parity drivers were ported from Python to Rust nextest binaries and -deleted (018-harden-parity-harness): the runners are now -`crates/deacon/tests/parity_corpus_tier1.rs` / -`parity_corpus_merged.rs`, which discover cases under the in-repo corpus root -`fixtures/parity-corpus/` (see `registry.json`) and run under -`cargo nextest run --profile parity` / `make test-parity`. This helper remains an -exploratory aid: to exercise a fetched snapshot, copy it as a case directory -under the corpus root (raising the corpus `min_cases` if you want the floor to -rise) and run `make test-parity`. - -The script uses pinned commit SHAs and GitHub's contents API via `gh api`. -""" - -from __future__ import annotations - -import argparse -import base64 -import json -import shutil -import subprocess -from dataclasses import asdict, dataclass -from functools import lru_cache -from pathlib import Path -from typing import Iterable - - -DEFAULT_DEST = Path("/tmp/realworld-corpus") - - -@dataclass(frozen=True) -class CorpusEntry: - name: str - repo: str - ref: str - workspace_root: str = "" - include_paths: tuple[str, ...] = () - config_path: str = "" - notes: str = "" - - -ENTRIES: tuple[CorpusEntry, ...] = ( - CorpusEntry( - name="images-javascript-node", - repo="devcontainers/images", - ref="31b61b521d55926d62c748b659f24ae71774c0e3", - workspace_root="src/javascript-node", - notes="Dockerfile + feature-heavy image recipe with scripts.", - ), - CorpusEntry( - name="images-python", - repo="devcontainers/images", - ref="31b61b521d55926d62c748b659f24ae71774c0e3", - workspace_root="src/python", - notes="Dockerfile build with multiple official features.", - ), - CorpusEntry( - name="images-go", - repo="devcontainers/images", - ref="31b61b521d55926d62c748b659f24ae71774c0e3", - workspace_root="src/go", - notes="Dockerfile build plus Go/Node/common-utils features.", - ), - CorpusEntry( - name="images-rust", - repo="devcontainers/images", - ref="31b61b521d55926d62c748b659f24ae71774c0e3", - workspace_root="src/rust", - notes="Dockerfile build with Rust feature and lifecycle customizations.", - ), - CorpusEntry( - name="images-java", - repo="devcontainers/images", - ref="31b61b521d55926d62c748b659f24ae71774c0e3", - workspace_root="src/java", - notes="Dockerfile build with Java/Node features.", - ), - CorpusEntry( - name="images-php", - repo="devcontainers/images", - ref="31b61b521d55926d62c748b659f24ae71774c0e3", - workspace_root="src/php", - notes="Dockerfile build with a checked-in local feature.", - ), - CorpusEntry( - name="templates-javascript-node-postgres", - repo="devcontainers/templates", - ref="95f7406a57fc5f0798964a5853c5ac04added322", - workspace_root="src/javascript-node-postgres", - notes="Compose-based template workspace with app + Postgres services.", - ), - CorpusEntry( - name="templates-go-postgres", - repo="devcontainers/templates", - ref="95f7406a57fc5f0798964a5853c5ac04added322", - workspace_root="src/go-postgres", - notes="Compose-based template workspace for Go + Postgres.", - ), - CorpusEntry( - name="try-node", - repo="microsoft/vscode-remote-try-node", - ref="45f5e33e47f4b113804ea808b7ce4c90a6823867", - include_paths=(".devcontainer", "package.json", "package-lock.json", "server.js"), - notes="Small image-based Node workspace used by the reference ecosystem.", - ), - CorpusEntry( - name="try-python", - repo="microsoft/vscode-remote-try-python", - ref="e351212b72f76fb557c6f31956eb44756300b8b4", - include_paths=(".devcontainer", "app.py", "requirements.txt", "static"), - notes="Small image-based Python workspace with app files.", - ), - CorpusEntry( - name="try-go", - repo="microsoft/vscode-remote-try-go", - ref="f4575309350c5ca2f1495c28a13b4b07088e8cea", - include_paths=(".devcontainer", "go.mod", "hello", "server.go"), - notes="Small image-based Go workspace with module files.", - ), - CorpusEntry( - name="try-rust", - repo="microsoft/vscode-remote-try-rust", - ref="d3cb2a9843af67d20491c9fde829b2c77230847b", - include_paths=(".devcontainer", "Cargo.toml", "Cargo.lock", "src"), - notes="Small image-based Rust workspace with crate sources.", - ), - CorpusEntry( - name="try-java", - repo="microsoft/vscode-remote-try-java", - ref="4638a925031f05cca946b8ce6ab640c433c93585", - include_paths=(".devcontainer", "pom.xml", "src"), - notes="Small image-based Java workspace with Maven sources.", - ), - CorpusEntry( - name="try-dotnetcore", - repo="microsoft/vscode-remote-try-dotnetcore", - ref="ca7ad4d9216a1bcc1469e4ca3545a66ff3e771a0", - include_paths=( - ".devcontainer", - "Program.cs", - "vscode-remote-try-dotnet.csproj", - "appsettings.json", - "appsettings.Development.json", - "appsettings.HttpsDevelopment.json", - ), - notes="Small image-based .NET workspace.", - ), - CorpusEntry( - name="try-cpp", - repo="microsoft/vscode-remote-try-cpp", - ref="22af031095a03864b8c2032b6498ff6d21e46c36", - include_paths=(".devcontainer",), - notes="Dockerfile-based C++ sample; .devcontainer contains required support files.", - ), - CorpusEntry( - name="try-php", - repo="microsoft/vscode-remote-try-php", - ref="9c4c759e95499bb57be2a35e2e0c55f292036908", - include_paths=(".devcontainer", "index.php"), - notes="Small image-based PHP workspace.", - ), - CorpusEntry( - name="oss-ruff", - repo="astral-sh/ruff", - ref="82b550741e8766224f23ce6d71fea262b866966b", - include_paths=(".devcontainer",), - notes="Real OSS Rust/Python workspace with volume mounts and a post-create script.", - ), - CorpusEntry( - name="oss-gh-cli", - repo="cli/cli", - ref="57b9b207d900114092f30d78020c193b40621dfa", - include_paths=(".devcontainer",), - notes="Real OSS Go workspace with sshd feature and explicit remoteUser.", - ), - CorpusEntry( - name="oss-vscode", - repo="microsoft/vscode", - ref="af752dba42ade664df7d6e01f4f459e0c1718512", - include_paths=(".devcontainer",), - notes="Dockerfile-based workspace with desktop-lite and rust features.", - ), - CorpusEntry( - name="oss-typescript", - repo="microsoft/TypeScript", - ref="7964e22f2b85f16e520f0e902c7fd7b6f0c15416", - include_paths=(".devcontainer",), - notes="Image-based workspace with feature metadata and rich VS Code customizations.", - ), - CorpusEntry( - name="oss-fluentui", - repo="microsoft/fluentui", - ref="de337cf86b501d2aa7d9b12c472489c7c88d6b24", - include_paths=(".devcontainer",), - notes="Dockerfile-based workspace with build args and feature metadata.", - ), - CorpusEntry( - name="oss-promptflow", - repo="microsoft/promptflow", - ref="3928a727b406e66d64ff42621534bb58e0ca18ce", - include_paths=(".devcontainer",), - notes="Dockerfile-based workspace with remoteEnv, runArgs, and Azure CLI feature.", - ), - CorpusEntry( - name="oss-autogen", - repo="microsoft/autogen", - ref="027ecf0a379bcc1d09956d46d12d44a3ad9cee14", - include_paths=(".devcontainer",), - notes="Compose-based real repo with many features and explicit workspaceFolder.", - ), - CorpusEntry( - name="oss-fhir-server", - repo="microsoft/fhir-server", - ref="7769ecc5eb170920f384f309afa6e852e7fdee78", - include_paths=(".devcontainer",), - notes="Compose-based real repo with custom workspaceFolder and helper scripts.", - ), - CorpusEntry( - name="oss-monaco-editor", - repo="microsoft/monaco-editor", - ref="7374dcb41a787a63d5885a5be5e6bbc2e6bc338c", - include_paths=(".devcontainer",), - notes="Simple image-based workspace with lightweight customizations.", - ), - CorpusEntry( - name="oss-web-dev-for-beginners", - repo="microsoft/Web-Dev-For-Beginners", - ref="5f220217d35499881cfff61a5b4c2dab033ab228", - include_paths=(".devcontainer",), - notes="Universal-image workspace with a feature and editor customizations.", - ), - CorpusEntry( - name="oss-generative-ai-for-beginners", - repo="microsoft/generative-ai-for-beginners", - ref="61a1240c5de4109ceac54142934411365c67c759", - include_paths=(".devcontainer",), - notes="Universal-image workspace with hostRequirements and post-create script.", - ), - CorpusEntry( - name="oss-ml-for-beginners", - repo="microsoft/ML-For-Beginners", - ref="24028ae995117b45fabb883cf42114e721ed65b5", - include_paths=(".devcontainer",), - notes="Dockerfile-based workspace with context '..' and init runArgs.", - ), - CorpusEntry( - name="oss-code-with-engineering-playbook", - repo="microsoft/code-with-engineering-playbook", - ref="016770e43d8a75be87b98c000c049f07c4a6e6f8", - include_paths=(".devcontainer",), - notes="Dockerfile-based workspace with parent build context.", - ), - CorpusEntry( - name="oss-procdump-linux", - repo="microsoft/ProcDump-for-Linux", - ref="f2717626a2960f8e0d0aebd9efe2e95bfe6c43b1", - include_paths=(".devcontainer",), - notes="Dockerfile-based workspace selecting a non-default Dockerfile variant.", - ), - CorpusEntry( - name="oss-sample-app-aoai-chatgpt", - repo="microsoft/sample-app-aoai-chatGPT", - ref="54f0af2c09bfe71f93da4acfb06422e11f984d71", - include_paths=(".devcontainer",), - notes="Image-based workspace with multiple OCI features including latest azd.", - ), - CorpusEntry( - name="oss-agentic-cookbook", - repo="microsoft/AgenticCookBook", - ref="32c6b754cff666962b6cd4679a2bdd9183fbe28e", - include_paths=(".devcontainer",), - config_path=".devcontainer/.devcontainer.json", - notes="Nonstandard config path (.devcontainer/.devcontainer.json) with hostRequirements.", - ), - CorpusEntry( - name="oss-presidio-anonymizer", - repo="microsoft/presidio", - ref="83ab7eb85609c49d9b0b17c44b5c025575966876", - include_paths=(".devcontainer/presidio-anonymizer", "presidio-anonymizer/Dockerfile.dev"), - config_path=".devcontainer/presidio-anonymizer/devcontainer.json", - notes="Nested config path with workspaceMount and cross-directory Dockerfile/context references.", - ), -) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--dest", type=Path, default=DEFAULT_DEST, help="Corpus output directory.") - parser.add_argument( - "--name", - dest="names", - action="append", - default=[], - help="Fetch only the named entry (repeatable).", - ) - parser.add_argument( - "--clean", - action="store_true", - help="Remove the destination directory before fetching.", - ) - parser.add_argument( - "--list", - action="store_true", - help="List the pinned corpus entries and exit.", - ) - return parser.parse_args() - - -def run_gh_json(endpoint: str) -> object: - result = subprocess.run( - ["gh", "api", endpoint], - check=True, - capture_output=True, - text=True, - ) - return json.loads(result.stdout) - - -@lru_cache(maxsize=None) -def get_commit_tree_sha(repo: str, ref: str) -> str: - commit = run_gh_json(f"repos/{repo}/commits/{ref}") - return commit["commit"]["tree"]["sha"] - - -@lru_cache(maxsize=None) -def get_tree_modes(repo: str, ref: str) -> dict[str, str]: - tree_sha = get_commit_tree_sha(repo, ref) - tree = run_gh_json(f"repos/{repo}/git/trees/{tree_sha}?recursive=1") - return {item["path"]: item.get("mode", "") for item in tree.get("tree", [])} - - -@lru_cache(maxsize=None) -def get_content_metadata(repo: str, ref: str, remote_path: str) -> object: - endpoint = f"repos/{repo}/contents" - if remote_path: - endpoint = f"{endpoint}/{remote_path}" - return run_gh_json(f"{endpoint}?ref={ref}") - - -def get_file_bytes(repo: str, ref: str, remote_path: str) -> bytes: - metadata = get_content_metadata(repo, ref, remote_path) - if not isinstance(metadata, dict) or metadata.get("type") != "file": - raise RuntimeError(f"Expected file at {repo}:{remote_path}@{ref}") - - content = metadata.get("content") - if not isinstance(content, str): - raise RuntimeError(f"Missing inline file content for {repo}:{remote_path}@{ref}") - return base64.b64decode(content) - - -def write_file(repo: str, ref: str, remote_path: str, destination: Path) -> None: - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(get_file_bytes(repo, ref, remote_path)) - - mode = get_tree_modes(repo, ref).get(remote_path, "") - if mode == "100755": - destination.chmod(0o755) - - -def fetch_tree(repo: str, ref: str, remote_dir: str, destination: Path) -> None: - metadata = get_content_metadata(repo, ref, remote_dir) - if not isinstance(metadata, list): - raise RuntimeError(f"Expected directory at {repo}:{remote_dir}@{ref}") - - destination.mkdir(parents=True, exist_ok=True) - for child in metadata: - child_type = child["type"] - child_name = child["name"] - child_remote_path = child["path"] - child_destination = destination / child_name - if child_type == "dir": - fetch_tree(repo, ref, child_remote_path, child_destination) - elif child_type == "file": - write_file(repo, ref, child_remote_path, child_destination) - else: - raise RuntimeError( - f"Unsupported GitHub contents type {child_type!r} at {repo}:{child_remote_path}@{ref}" - ) - - -def fetch_path(repo: str, ref: str, remote_path: str, destination: Path) -> None: - metadata = get_content_metadata(repo, ref, remote_path) - if isinstance(metadata, list): - fetch_tree(repo, ref, remote_path, destination) - return - if metadata.get("type") != "file": - raise RuntimeError(f"Unsupported path type at {repo}:{remote_path}@{ref}: {metadata}") - write_file(repo, ref, remote_path, destination) - - -def selected_entries(names: Iterable[str]) -> list[CorpusEntry]: - if not names: - return list(ENTRIES) - - by_name = {entry.name: entry for entry in ENTRIES} - missing = sorted(set(names) - set(by_name)) - if missing: - raise SystemExit(f"Unknown corpus entries: {', '.join(missing)}") - return [by_name[name] for name in names] - - -def write_manifest(entries: list[CorpusEntry], dest: Path) -> None: - manifest_path = dest / "_manifest.json" - payload = { - "entries": [asdict(entry) for entry in entries], - "generated_by": "fixtures/parity-corpus/fetch_realworld_corpus.py", - } - manifest_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def list_entries(entries: list[CorpusEntry]) -> None: - for entry in entries: - payload = { - "name": entry.name, - "repo": entry.repo, - "ref": entry.ref, - "workspace_root": entry.workspace_root, - "include_paths": list(entry.include_paths), - "config_path": entry.config_path, - "notes": entry.notes, - } - print(json.dumps(payload, sort_keys=True)) - - -def fetch_entry(entry: CorpusEntry, dest: Path) -> None: - target = dest / entry.name - if target.exists(): - shutil.rmtree(target) - target.mkdir(parents=True, exist_ok=True) - - if entry.workspace_root: - fetch_tree(entry.repo, entry.ref, entry.workspace_root, target) - - for include_path in entry.include_paths: - fetch_path(entry.repo, entry.ref, include_path, target / include_path) - - -def main() -> None: - args = parse_args() - entries = selected_entries(args.names) - - if args.list: - list_entries(entries) - return - - if args.clean and args.dest.exists(): - shutil.rmtree(args.dest) - args.dest.mkdir(parents=True, exist_ok=True) - - for entry in entries: - print(f"==> fetching {entry.name} from {entry.repo}@{entry.ref}") - fetch_entry(entry, args.dest) - - write_manifest(entries, args.dest) - print(f"\nFetched {len(entries)} workspace snapshots into {args.dest}") - - -if __name__ == "__main__": - main() diff --git a/fixtures/parity-corpus/registry.json b/fixtures/parity-corpus/registry.json index 749dec61..40c002b6 100644 --- a/fixtures/parity-corpus/registry.json +++ b/fixtures/parity-corpus/registry.json @@ -40,5 +40,31 @@ "consistency_remote_env_flags", "consistency_env_probe_flag" ], - "corpora": [] + "corpora": [], + "discovery_binaries": [ + { + "name": "discovery_campaign", + "role": "live", + "tests_dir": "crates/deacon/tests", + "docker_required": true + }, + { + "name": "discovery_metamorphic", + "role": "live", + "tests_dir": "crates/deacon/tests", + "docker_required": false + }, + { + "name": "discovery_hermetic", + "role": "guard", + "tests_dir": "crates/deacon/tests", + "docker_required": false + }, + { + "name": "discovery_cli", + "role": "guard", + "tests_dir": "crates/conformance/tests", + "docker_required": false + } + ] } diff --git a/specs/025-exploratory-parity-discovery/checklists/requirements.md b/specs/025-exploratory-parity-discovery/checklists/requirements.md new file mode 100644 index 00000000..db73b804 --- /dev/null +++ b/specs/025-exploratory-parity-discovery/checklists/requirements.md @@ -0,0 +1,88 @@ +# Specification Quality Checklist: Exploratory Parity Discovery + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2026-07-27 +**Feature**: [spec.md](../spec.md) + +## Content Quality + +- [x] No implementation details (languages, frameworks, APIs) +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic (no implementation details) +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded +- [x] Dependencies and assumptions identified + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No implementation details leak into specification + +## Validation Notes + +**Iteration 1 findings and resolutions:** + +1. *Acceptance scenarios missing a **When** clause* — US2 scenarios 4 and 6 stated a + condition and an outcome without an explicit trigger. Rewritten with explicit + **Given/When/Then** structure. +2. *Requirements section lacked the template's `### Functional Requirements` heading* — + the FR groups were promoted under a single `### Functional Requirements` heading with + `####` subgroups, preserving template section order. +3. *Classification vocabulary risked reading as identifiers* — the six classification values + are stated in prose form in the requirements and entities so the spec stays free of + implementation-level token names; the closed set is unambiguous either way. + +**Deliberate judgement calls (not defects):** + +- The spec names existing project artifacts conceptually ("the deterministic record", "the + pinned reference", "the normalization definition", "observable channels") rather than by + file path or crate. This matches the house style of specs 019–024, which describe the + conformance record as a domain concept. No language, framework, or API is named. +- Every mandatory acceptance test named in the feature request has at least one covering + acceptance scenario: seed reproduction (US1-1, SC-001), semantic generation (US1-2/3, + SC-002/003), shrinking (US2-1..4, SC-004), metamorphic failures (US6-4/6, SC-011), + classification (US4-1, SC-007), deduplication (US4-2/3, SC-006), review-only promotion + (US5-1..3, SC-008/009), pinned real-world provenance (US7-1/2, SC-012), and + hermetic/network lane isolation (US3-1..4, SC-013/014). +- Zero [NEEDS CLARIFICATION] markers: every gap in the request had a defensible default, + each recorded explicitly in the **Assumptions** section (10 entries) so a reviewer can + overturn a default rather than reverse-engineer it. The two highest-impact defaults are + Assumption 1 (tiered comparison surface — bounds campaign cost) and Assumption 4 + (version-controlled findings queue — the only way cross-run deduplication and triage + state can exist). + +## Clarification Session — 2026-07-27 + +Five questions asked and answered; all recommendations accepted. Re-validated after +integration: 0 `[NEEDS CLARIFICATION]` markers, 5 clarification bullets, 69 functional +requirements, 19 success criteria, heading hierarchy intact. + +| # | Decision | Where it landed | +|---|----------|-----------------| +| 1 | Findings queue lives outside the registry; never loaded, never an input to certification | FR-034a, US5-8, SC-018, Key Entities, Assumption 4 | +| 2 | Signature = channel + path + difference kind + value-shape class; concrete values excluded | FR-030a, Key Entities | +| 3 | Mutation seeds are committed fixtures only; corpus is a comparison input, never a seed | FR-008a, FR-054a, US7-6, Assumption 9 | +| 4 | Per-campaign admission cap with a reported suppressed count; never fails the campaign | FR-034b, FR-061, US4-7, SC-019 | +| 5 | Pipeline proven by an injected difference; a real promotion is reported, not required | FR-042a, US5-7, SC-016 | + +Two of these overturned defaults the initial spec carried. SC-016 previously required a real +discovered-and-promoted behavior — an acceptance criterion no amount of effort can guarantee, +since it depends on the implementations actually disagreeing where the generator reaches. And +the findings queue was described as "version-controlled" without stating that it sits *outside* +the record, which left open the reading that would have let an unreviewed stochastic finding +reach the release gate. + +## Notes + +- Items marked incomplete require spec updates before `/speckit.clarify` or `/speckit.plan`. diff --git a/specs/025-exploratory-parity-discovery/contracts/discovery-cli.md b/specs/025-exploratory-parity-discovery/contracts/discovery-cli.md new file mode 100644 index 00000000..1fca7a4b --- /dev/null +++ b/specs/025-exploratory-parity-discovery/contracts/discovery-cli.md @@ -0,0 +1,173 @@ +# Contract: Discovery Command Surface + +**Feature**: `025-exploratory-parity-discovery` + +Every command here is **development-only**. None is a `deacon` subcommand, and +`parity_registry_check` asserts on every PR that `deacon --help` gains nothing from any of them +(FR-059). + +## The exit-status rule (applies to all commands below) + +> A discovery command's exit status reflects **whether it ran**, never **what it found**. + +This is the `coverage report` discipline extended to the whole surface (FR-058). A campaign that +finds 40 differences exits `0`; a campaign that cannot verify the oracle exits non-zero. The one +exception is stated explicitly per command and is always a *machinery* failure, never a finding. + +Rationale: any command whose status depends on its findings becomes a gate the moment someone +wires it into CI, and a stochastic gate makes green non-reproducible. Keeping status +machinery-only is what lets the discovery lane be added to CI safely. + +--- + +## Hermetic commands — `cargo run -p deacon-conformance -- discovery ` + +No network, no Docker, no oracle. Safe in any lane, though FR-055 still keeps them out of the PR +lane's *selection*. + +### `discovery check` + +Validates the discovery data root. The hermetic gate that blocks a PR. + +| | | +|---|---| +| **Reads** | `conformance/discovery/{findings,campaigns,corpus}.json`, `conformance/registry/` (to resolve `promotedTo` and pins) | +| **Writes** | nothing — read-only by construction | +| **stdout** | violation report (text) or a single JSON document under `--format json` | +| **Exit `0`** | no D1–D5 violations | +| **Exit `1`** | one or more violations, all reported in a single run | + +Reports **all** violations in one pass rather than stopping at the first, matching `validate`. + +### `discovery report` + +Renders the queue and the campaign history. + +| | | +|---|---| +| **Writes** | `target/discovery/queue.{json,md}` — git-ignored, byte-stable, no timestamps or absolute paths | +| **Exit `0`** | artifacts written | +| **Exit `1`** | could not write | + +**Never gates.** Exit status reflects only whether the files were produced. A queue holding fifty +untriaged findings still exits `0`. + +Content: findings by state and classification, the **counted untriaged bucket** (FR-029), the +no-longer-reproducing set with the campaign that last observed each (FR-033), promoted findings +with their cases, and per-campaign suppression counts (FR-034b). + +### `discovery triage --classification [--notes ]` + +Records a reviewer's classification. The **only** writer of `classification`. + +| | | +|---|---| +| **Writes** | `conformance/discovery/findings.json` (atomic) | +| **Exit `0`** | classification recorded | +| **Exit `1`** | unknown finding, invalid classification, or a finding already in a terminal state | + +Interactive-equivalent, not automated: no campaign invokes it. FR-028's "exactly one +classification" is enforced here and re-checked by `discovery check` (D2), so a hand edit that +bypasses the command still fails. + +### `discovery split ` + +Splits a signature-merged finding whose witnesses turn out to have different causes (FR-032). +Children carry `splitFrom`; the deduplication rule must never re-merge them. + +### `discovery scaffold ` + +Emits to **stdout** a skeleton behavior + case + fixture layout for promoting a finding, with +`UNREVIEWED` sentinels the registry loader rejects. + +| | | +|---|---| +| **Writes** | **nothing** — stdout only | +| **Exit `0`** | skeleton emitted | +| **Exit `1`** | finding is non-promotable (`normalizer-defect` / `fixture-defect`, FR-035) or unknown | + +Stdout-only is the same discipline as `inventory scaffold` and `clause scaffold`: **generation +never writes a hand-authored file**. Promotion is a human editing the registry with this output +as a starting point, which is what makes FR-036 hold — there is no code path from a finding to a +registry write. + +--- + +## Live commands — `cargo run -p parity-harness --bin ` + +Require the verified pinned oracle; some additionally require Docker or network. All fail loudly +on a missing or mismatched prerequisite (FR-003) — never a silent skip. + +### `discovery-campaign --seed --tier [--budget-seconds ] [--lane ]` + +Runs one campaign. + +| | | +|---|---| +| **Requires** | verified pinned oracle (all tiers except `metamorphic`); Docker (`container-differential`); network (`corpus`) | +| **Writes** | `conformance/discovery/{findings,campaigns}.json` (atomic); `target/discovery/candidates//` | +| **Never writes** | anything under `conformance/registry/`, `conformance/snapshots/`, or `conformance/obligations/` (FR-036) | +| **stdout** | single JSON campaign outcome | +| **stderr** | progress and diagnostics via `tracing` | +| **Exit `0`** | the campaign ran to completion or to budget exhaustion — **regardless of findings** | +| **Exit `1`** | prerequisite failure, normalization failure, or an unwritable data root | + +`--seed` is **required**, not defaulted. A defaulted seed would let a campaign run without its +reproducibility input being a conscious choice, and FR-001 depends on the seed being recorded +rather than inferred. + +Tiers (research D10): `metamorphic` needs nothing external; `config-differential` is the +**nightly** scheduled tier; `corpus` is the **weekly** scheduled tier in the network-backed lane; +`container-differential` is invoked-only. + +`corpus` has a scheduled cadence of its own rather than being invocation-only, because its whole +purpose is to be an ecological canary — and a canary that sings only when asked is not one. Weekly +rather than nightly because the corpus changes only when someone re-pins it, so nightly runs would +mostly re-confirm the previous night at network cost. + +### `discovery-proof` + +The FR-042a pipeline proof. Injects a known difference at the sealed evidence-source boundary and +requires it to traverse generation → comparison → minimization → candidate → classification → +promotable. + +| | | +|---|---| +| **Writes** | `target/discovery/proof.json` | +| **Exit `0`** | every injected difference traversed the full pipeline | +| **Exit `1`** | any injected difference failed to surface, **or** any injection was inapplicable | + +This is the one command whose status depends on an outcome — and it is not a finding-dependent +status. It asserts a property of the machinery, so a non-zero exit means the pipeline is broken, +which is exactly the thing that should fail a lane. + +An injection that never landed exits `1` as `InjectionInapplicable` rather than being counted as +"nothing found" — a mis-authored proof record must never masquerade as a working pipeline +(inherited from `inject.rs`, research D7). + +--- + +## Make targets + +| Target | Runs | +|---|---| +| `make test-discovery` | `cargo nextest run --profile discovery`, then `discovery report` | +| `make test-discovery-proof` | `discovery-proof` | +| `make test-discovery-check` | `discovery check` (hermetic; also runs in the fast lane) | + +## Lane wiring (FR-055 – FR-057) + +`[profile.discovery]`'s `default-filter` is an **explicit `binary(=…)` allow-list**, never a +`discovery_*` glob. The glob would capture the hermetic guard `discovery_hermetic` and silently +remove it from the fast lane — the exact mistake the parity profile already documents having made +with `parity_harness_faults` and `parity_registry_check`. + +Every live discovery binary is excluded from the `default-filter` of all six existing profiles +(`default`, `dev-fast`, `full`, `ci`, `mvp-integration`, `parity`). Exclusion from `parity` is +deliberate: the two lanes answer different questions and a campaign would exceed the parity lane's +budget. + +`parity_registry_check` is extended to enforce registry ↔ `tests/*.rs` ↔ `.config/nextest.toml` +agreement for the discovery lane, so a binary cannot be added without its wiring. That check +already fails loudly on drift; FR-057 describes its shape exactly, so extending it is strictly +better than a parallel checker. diff --git a/specs/025-exploratory-parity-discovery/contracts/findings-queue.md b/specs/025-exploratory-parity-discovery/contracts/findings-queue.md new file mode 100644 index 00000000..0603464f --- /dev/null +++ b/specs/025-exploratory-parity-discovery/contracts/findings-queue.md @@ -0,0 +1,143 @@ +# Contract: Findings Queue + +**Feature**: `025-exploratory-parity-discovery` +**Location**: `conformance/discovery/findings.json` — a sibling of `registry/`, **not** inside it + +## The unreachability guarantee + +The registry loader (`crates/conformance/src/load.rs`) enumerates *named* subdirectories under +`conformance/registry/` — `cases/`, `behaviors/`, `sources/`, `waivers/`, `classifications/`, +`clause-classifications/`, `obligation-dispositions/`. There is no wildcard walk at the registry +root, so a sibling of `registry/` has no code path that reaches it. `certify` consumes the loaded +record only. + +This makes the guarantee **structural**, not conventional: an unreviewed finding cannot influence +a release gate because there is no function that could carry it there. The distinction matters +because the failure mode is silent — a finding quietly joining the denominator — which is the +class of mistake 024 D1 documented when a scenario dimension was nearly added to `dimensions.json`. + +**One reference crosses the boundary, and it points outward**: `Finding.promotedTo → case-`. +Nothing in the registry points back. Following references from the registry can never arrive at a +finding. + +## Record schema + +```json +{ + "schemaVersion": 1, + "records": [ + { + "id": "fnd-", + "signature": { + "id": "sig-", + "channel": "chan-structured-output", + "path": "configuration.remoteUser", + "kind": "value", + "valueShapeClass": "type-changed" + }, + "witnesses": [ + { + "id": "wit-", + "campaignId": "cmp-", + "candidateId": "cnd-", + "minimalInput": { }, + "isMinimal": true, + "reductionSteps": ["drop-optional-key", "un-apply-mutation"], + "observedValues": { "deacon": null, "reference": null }, + "mutationOperators": ["mop-wrong-type"] + } + ], + "classification": "deacon-regression", + "state": "triaged", + "firstObserved": "cmp-", + "lastObserved": "cmp-", + "promotedTo": null, + "splitFrom": null, + "notes": "" + } + ] +} +``` + +Unknown fields are rejected at load, matching every other record kind here. + +## Invariants + +| # | Invariant | Enforced by | +|---|---|---| +| Q1 | `id` is derived from `signature.id` — two findings with the same signature are the same record | id derivation (duplicates are unrepresentable) | +| Q2 | `witnesses` is non-empty and declaration-ordered by first observation | D1 | +| Q3 | `signature.channel` resolves in `channels.json` | D1 | +| Q4 | exactly one `classification` once `state` is `triaged` or later; `null` only while `untriaged` | D2 | +| Q5 | `state == "promoted"` ⟺ `promotedTo` is non-null **and** resolves to a real case | D3 | +| Q6 | `classification ∈ {normalizer-defect, fixture-defect}` ⇒ `state != "promoted"` | D2 + the promotion path | +| Q7 | a finding with a non-null `splitFrom` is never re-merged into its ancestor | deduplication skips split lineages | +| Q8 | every `firstObserved` / `lastObserved` resolves in `campaigns.json` | D1 | +| Q9 | every `pinnedInputSet` element on the referenced campaign names a revision in `revisions.json` | D5 | +| Q10 | a finding in state `split` has ≥2 children naming it in `splitFrom`, carries no `classification` of its own, and is never a merge target | D1 + D2 | + +**Q10 spelled out**, because the parent's fate after a split is easy to leave undefined: the +parent becomes an inert ancestor. It keeps its witnesses as historical record, stops accepting +new ones, and surrenders classification to its children — a split exists precisely because one +classification could not describe them all. A parent that kept a classification would assert the +judgement the split rejected. + +## Deduplication (FR-030 – FR-032) + +**Merge rule**: a new divergence whose signature equals an existing finding's signature appends a +witness to that finding. It does not create a record. + +**Non-merge rules**, both load-bearing: + +- **Distinct signatures stay distinct even when they map to the same behavior** (FR-031). They may + be *reported* grouped under that behavior, but grouping is a view, not a merge — merging would + destroy the ability to tell whether a fix addressed one cause or all of them. +- **A split lineage is never re-merged** (FR-032). Without this, a reviewer's judgement that two + witnesses have different causes is silently reverted by the next campaign, and the split becomes + unrepeatable work. + +**Admission cap** (FR-034b): a campaign admits at most `budget.admissionCap` newly distinct +signatures. Excess signatures are counted in `signaturesSuppressed` and reported. Exceeding the +cap **never** fails the campaign — that would make discovery gate on its own output. + +Suppression is always visible. A campaign that repeatedly hits its cap is itself a signal that +something systemic is diverging, and a silent truncation would read as "we found 25 things" +instead of "we found many more than we can review". + +## Reproduction lifecycle (FR-033) + +A finding that stops reproducing moves to `no-longer-reproducing`, retaining the campaign that +last observed it. It is **not** deleted. + +Deleting it would destroy the ability to distinguish two very different situations: a fix landed, +or the generator stopped reaching that input. The first is success; the second is a coverage +regression in the discovery machinery itself. Only the retained record makes them separable. + +A later campaign that reproduces it moves it back to `triaged`, keeping its classification — +re-triaging a finding a reviewer already judged would be wasted work. + +## Pin invalidation (Assumption 8) + +Findings are claims about a specific pinned pair of implementations. On a pin change — a +re-vendored schema surface, a new oracle version, a `NORMALIZER_VERSION` bump — a finding's +`pinnedInputSet` no longer matches the current one. + +Such findings are **re-evaluated, not carried forward**. `discovery report` lists them as +pin-stale; the next campaign under the new pins either reproduces them (they return to `triaged` +with a new witness) or does not (they move to `no-longer-reproducing`). Carrying them forward +unchanged would assert that a difference observed against oracle *v0.80* still holds against +*v0.81* — a claim nothing verified. + +## Violation classes (`conformance discovery check`) + +| Class | Statement | +|---|---| +| **D1** | malformed queue record; empty `witnesses`; a signature naming an undeclared channel; an unresolvable campaign reference | +| **D2** | a finding with zero or more than one classification while in state `triaged` or later; a non-promotable classification in state `promoted` | +| **D3** | a `promoted` finding whose `promotedTo` does not resolve to a real case | +| **D4** | *(corpus)* a non-immutable reference, or a digest recorded then removed | +| **D5** | a finding or campaign whose `pinnedInputSet` names a revision absent from `revisions.json` | + +Numbered separately from the registry's V-series because they are emitted by a different command +over a different data root. Folding them into V-numbering would imply the registry validator can +see the queue — which is precisely what this contract exists to prevent. diff --git a/specs/025-exploratory-parity-discovery/contracts/metamorphic-catalogue.md b/specs/025-exploratory-parity-discovery/contracts/metamorphic-catalogue.md new file mode 100644 index 00000000..939f7efa --- /dev/null +++ b/specs/025-exploratory-parity-discovery/contracts/metamorphic-catalogue.md @@ -0,0 +1,121 @@ +# Contract: Metamorphic Relation Catalogue + +**Feature**: `025-exploratory-parity-discovery` +**Location**: `conformance/registry/metamorphic.json` — **inside** the registry + +## Why relations live in the registry and findings do not + +A metamorphic relation is an **assertion the project makes**: "reordering these keys must not +change the result, and here is the clause that says so." It is hand-authored, reviewed, stable, +and references `clu-` / `bhv-` ids that only the registry loader resolves. That is the same kind +of object as an applicability rule. + +A finding is a **candidate for an assertion** — machine-produced, unreviewed, possibly wrong. + +The split follows from what each thing *is*, not from where it is convenient to put it. + +## Record schema + +```json +{ + "schemaVersion": 1, + "records": [ + { + "id": "mrl-key-order-invariance", + "transformation": "permute the key order within an unordered JSON object", + "effect": "invariance", + "ground": "clu-a1b2c3d4", + "channels": ["chan-structured-output"], + "rationale": "Object member order carries no meaning in JSON, and the configuration schema declares no ordered object. A result that changes under key permutation is reading order it must not read." + } + ] +} +``` + +## The two effects + +| `effect` | Assertion | Catches | +|---|---|---| +| `invariance` | the transformation MUST NOT change the normalized result | a tool reading meaning that is not there | +| `sensitivity` | the transformation MUST change the normalized result | a tool ignoring meaning that *is* there | + +**Sensitivity relations are the ones the differential cannot replace.** If deacon and the +reference both wrongly ignore declaration order, the differential comparison is clean and the +defect is invisible to it — both sides agree, and agreeing is what the differential checks. A +sensitivity relation asserts the result *must* change, so consistent-wrongness fails it. + +This is why FR-043 mandates both kinds rather than treating sensitivity as an optional extra. + +## Mandated families (FR-044) + +Every row must have at least one record, or **V32**. + +| Id | Effect | Transformation | Ground kind | +|---|---|---|---| +| `mrl-formatting-invariance` | invariance | reindent, rewrap, alter insignificant whitespace | clause | +| `mrl-comment-invariance` | invariance | insert JSONC comments and trailing commas | clause | +| `mrl-key-order-invariance` | invariance | permute keys within an unordered map | clause | +| `mrl-path-relocation` | invariance | relocate the workspace to a different absolute path | behavior | +| `mrl-lifecycle-equivalence` | invariance | switch between equivalent lifecycle command forms | clause | +| `mrl-extends-flattening` | invariance | replace an `extends` chain with its hand-flattened equal | behavior | +| `mrl-declaration-order-sensitivity` | **sensitivity** | permute a declaration-ordered collection | clause | + +## The ground requirement (FR-045) + +Every relation MUST name a `ground` that resolves to a normative clause (`clu-`) or a recorded +behavior (`bhv-`). A relation with no ground, or with one that does not resolve, is **V31**. + +This mirrors the `ground` that 024 already requires on applicability rules, and for the same +reason: without it, a relation records an author's intuition about what *ought* to be irrelevant. +An ungrounded invariance relation that happens to be wrong does not fail — it *passes*, silently, +while asserting something the spec never said. A grounded one can be checked by reading the clause. + +`mrl-path-relocation` and `mrl-extends-flattening` are grounded on behaviors rather than clauses +because both concern resolution mechanics the prose describes operationally rather than in a +single normative sentence. + +## Path relocation and the tokenizer (FR-046) + +`mrl-path-relocation` compares **modulo the declared path tokenization** and reports any residual +difference the tokenization does not account for. + +That residual is the interesting output. A leaked absolute path that the tokenizer misses shows up +here as a relation failure, which means this relation is simultaneously a check on deacon and a +check on the normalizer. A residual difference should therefore be triaged carefully: it is as +likely to be a `normalizer-defect` as a `deacon-regression`, and misfiling it as the latter sends +someone to fix code that is correct. + +## Evaluation (FR-048) + +Relations are evaluated against **deacon alone** — no oracle, no Docker, no network. + +This makes the metamorphic tier the only part of discovery that runs with none of the three +prerequisites. Two consequences worth acting on: + +- A contributor with no devcontainer CLI installed can develop and test this story locally. +- It is the cheapest complete vertical slice through generation → comparison → signature → + candidate, so building it first exercises the entire hermetic spine before any live oracle + provisioning exists (research D12). + +It does **not** license running it in the PR lane. FR-055 is absolute, and its reason is +stochasticity, not resource cost. + +## Failure output (FR-047) + +A relation failure produces a candidate naming the relation, the transformation applied, both +inputs, and both normalized outputs — the same reviewable-candidate shape as a differential +finding, so both enter one triage pipeline. + +The signature for a metamorphic finding uses the relation's channel plus the divergence between +the two *deacon* outputs. Its `kind` and `valueShapeClass` are computed identically, so a +metamorphic finding and a differential finding at the same path deduplicate against each other — +which is correct: they are the same defect observed two ways. + +## Violation classes (`conformance validate`) + +| Class | Statement | +|---|---| +| **V31** | unresolvable or missing `ground`; unknown `effect`; duplicate `transformation`; a `channels` entry not in `channels.json` | +| **V32** | a mandated relation family (FR-044) has no record | + +Both block a PR via the existing hermetic `registry_valid` test. diff --git a/specs/025-exploratory-parity-discovery/data-model.md b/specs/025-exploratory-parity-discovery/data-model.md new file mode 100644 index 00000000..5f766b80 --- /dev/null +++ b/specs/025-exploratory-parity-discovery/data-model.md @@ -0,0 +1,432 @@ +# Phase 1 Data Model: Exploratory Parity Discovery + +**Feature**: `025-exploratory-parity-discovery` +**Date**: 2026-07-27 + +All records are strict JSON. Unknown fields are rejected at load (matching every other record +kind in this repository). All writes are atomic — serialize to a unique temp file, then +`fs::rename` into place. + +Two data roots, deliberately separate (research D6, D11): + +| Root | Ownership | Reachable from `certify`? | +|---|---|---| +| `conformance/discovery/` | machine-produced + hand-triaged | **No** — sibling of `registry/`, no loader path reaches it | +| `conformance/registry/metamorphic.json` | hand-authored | Yes — it is an assertion the project makes | + +--- + +## 1. Identity and hashing + +All ids are substance-anchored using the existing `hash8` helper (SHA-256 truncated to 8 hex +chars), the same primitive behind `clu-` and `cst-` ids. Substance-anchoring means a record that +is reordered, re-annotated, or moved keeps its id; only a change to the thing itself changes it. + +| Prefix | Entity | Hashed substance | +|---|---|---| +| `sig-` | Normalized signature | `channel ‖ path ‖ kind ‖ valueShapeClass` | +| `fnd-` | Finding | its `signature` (1:1 with signature; the finding *is* the signature's record) | +| `wit-` | Witness | `campaignId ‖ candidateId` | +| `cmp-` | Campaign | `seed ‖ canonical(pinnedInputSet) ‖ lane ‖ profile` | +| `cnd-` | Candidate input | `canonical(document) ‖ canonical(operations)` | +| `mop-` | Mutation operator | its declared name (stable, hand-assigned, not hashed) | +| `mrl-` | Metamorphic relation | its declared name (stable, hand-assigned, not hashed) | +| `cor-` | Corpus entry | `repository ‖ commit ‖ path` | + +**Why `fnd-` is derived from the signature and not independently assigned**: FR-030 makes the +signature the deduplication key, so two findings with the same signature *are* one finding. If +ids were independently assigned, the invariant would have to be maintained by the merge logic +and could be violated by a bad merge. Deriving the id makes duplicate findings unrepresentable. + +--- + +## 2. Normalized signature + +The deduplication key (spec clarification Q2, research D3). Computed from +`parity_harness::normalize::diff`'s existing `ConfigDivergence` output — never re-derived. + +``` +Signature { + id: "sig-" + channel: string // one of the 11 declared channels + path: string // ConfigDivergence::path, verbatim + kind: "ref-only" | "deacon-only" | "value" // DiffKind::as_str() + valueShapeClass: "present-absent" | "type-changed" | "ordering-changed" | "value-changed" +} +``` + +### Value-shape classification + +The only new derivation. A pure function of the divergence: + +| `DiffKind` | Condition | `valueShapeClass` | +|---|---|---| +| `RefOnly` / `DeaconOnly` | always | `present-absent` | +| `Value` | the two JSON types differ | `type-changed` | +| `Value` | both arrays, and one is a permutation of the other | `ordering-changed` | +| `Value` | otherwise | `value-changed` | + +`ordering-changed` is classified before `value-changed` and is a distinct class rather than a +subcase, because declaration-order defects are a known recurring family in this codebase +(`BTreeMap` where the spec requires declaration order). Folding them into `value-changed` would +merge an order defect with an unrelated value defect at the same path. + +**Concrete observed values never enter the signature.** They are retained on the witness, where +they are evidence rather than identity. + +**Validation**: a signature whose `channel` is not in `channels.json` is **D1**. + +--- + +## 3. Finding — `conformance/discovery/findings.json` + +One record per distinct signature. Machine-created, hand-triaged. + +``` +Finding { + id: "fnd-" // derived from signature.id + signature: Signature + witnesses: [Witness] // >= 1, declaration-ordered by first observation + classification: Classification | null // null == untriaged (the visible bucket, FR-029) + state: FindingState + firstObserved: "cmp-" // campaign that first admitted it + lastObserved: "cmp-" // most recent campaign that reproduced it + promotedTo: "case-" | null // set only in state `promoted` + splitFrom: "fnd-" | null // provenance when a reviewer splits a merged finding + notes: string // reviewer prose; excluded from every hash +} +``` + +### Classification (FR-028) — closed set, exactly one + +| Value | Meaning | Promotable? | +|---|---|---| +| `deacon-regression` | deacon is wrong; the reference and/or spec is right | Yes — becomes a behavior + a fix | +| `reference-quirk` | the reference diverges from the spec; deacon is right | Yes — behavior + waiver | +| `spec-ambiguity` | the spec does not decide the question | Yes — behavior with `spec: unspecified` | +| `unsupported-behavior` | deacon lacks the capability entirely | Yes — becomes a `gap-` record | +| `normalizer-defect` | the comparison machinery manufactured or hid the difference | **No** (FR-035) | +| `fixture-defect` | the generated input was invalid in a way the harness cannot express | **No** (FR-035) | + +The last two are non-promotable because they describe a defect in the discovery machinery, not a +behavior of either implementation. Resolving them changes the normalizer or the generator. + +### State machine + +``` + admitted by a campaign + │ + ▼ + ┌─────────────┐ + │ untriaged │ classification == null + └──────┬──────┘ + reviewer assigns a classification + │ + ▼ + ┌─────────────┐ + ┌─────────────│ triaged │─────────────┐ + │ └──────┬──────┘ │ + │ │ │ + reviewer splits reviewer promotes difference stops + │ │ reproducing + ▼ ▼ ▼ + ┌───────────┐ ┌───────────┐ ┌──────────────────────┐ + │ split │ │ promoted │ │ no-longer-reproducing│ + └───────────┘ └───────────┘ └──────────────────────┘ + (children terminal; terminal-ish; a later + carry splitFrom) promotedTo campaign may revive it + is set back to `triaged` +``` + +Rules the transitions encode: + +- **`untriaged` is visible, never implicit.** FR-029 requires a counted bucket, so "not yet + looked at" can never read as "nothing found". +- **`no-longer-reproducing` is a state, not a deletion** (FR-033). The disappearance is + information: it may mean a fix landed, or it may mean the generator stopped reaching the input. + Deleting the record destroys the ability to tell those apart. +- **A `split` finding's children carry `splitFrom`** and the deduplication rule must not re-merge + them (FR-032). Enforcement: signature-equality merging skips any finding with a non-null + `splitFrom` chain reaching the same ancestor. +- **`promoted` requires `promotedTo` to resolve** to a real case (**D3**), so the queue can never + claim coverage that does not exist. +- **`normalizer-defect` / `fixture-defect` cannot reach `promoted`** (**D2** guards the + classification arity; the promotion path rejects these two by construction). + +### Witness + +``` +Witness { + id: "wit-" + campaignId: "cmp-" + candidateId: "cnd-" + minimalInput: // the reduced fixture + isMinimal: boolean // false => budget exhausted (FR-022) + reductionSteps: [string] // ordered catalogue step names applied + observedValues: { deacon: , reference: } + mutationOperators: ["mop-"] // FR-009 attribution +} +``` + +Witnesses are retained per finding (FR-032) so a merge can be reviewed and reversed. The concrete +observed values live here — evidence, not identity. + +--- + +## 4. Campaign — `conformance/discovery/campaigns.json` + +Provenance for every run. Append-only; a campaign record is never rewritten. + +``` +Campaign { + id: "cmp-" + seed: string // hex, the recorded seed (FR-001) + lane: "scheduled" | "invoked" + tier: "metamorphic" | "config-differential" | "container-differential" | "corpus" + pinnedInputSet: PinnedInputSet + budget: Budget + outcome: CampaignOutcome +} + +PinnedInputSet { // all SEVEN elements required (FR-002) + schemaPin: string // conformance/schemas/ + prosePin: string // conformance/spec/ + oracleVersion: string // exact, verified (FR-003) + normalizerVersion: string // NORMALIZER_VERSION + grammarVersion: string // constraints.json revision + fingerprint (research D1) + mutationCatalogVersion: string // the mutation OPERATOR set + generatorVersion: string // PRNG algorithm identity + reduction-catalogue ORDER +} + +Budget { + wallClockSeconds: integer // 1800 for scheduled (research D10) + perCandidateSeconds: integer // 60 hermetic / 300 container-backed + shrinkStepsPerFinding: integer + admissionCap: integer // 25 (research D10) +} + +CampaignOutcome { + candidatesGenerated: integer + candidatesExecuted: integer + candidatesDiscardedUnsafe: integer // FR-011 + parseStageFailures: integer // numerator for the SC-002 ratio + budgetExhausted: boolean // FR-005 + spaceCoveredFraction: number // reported when budgetExhausted (FR-005) + mutationApplications: { "": integer } // FR-010, all 11 keys always present + signaturesObserved: integer + signaturesAdmitted: integer + signaturesSuppressed: integer // FR-034b — never silent +} +``` + +**`mutationApplications` always carries all eleven category keys**, including zeroes. A category +absent from the map is indistinguishable from a category that was never applied; FR-010 requires +zero to be reported as an explicit generation deficiency, which needs the key present. + +**`grammarVersion` binds the campaign to the constraint inventory** (research D1). A re-vendored +schema pin regenerates the inventory, which changes this string, which correctly invalidates +every finding bound to the old value (Assumption 8) with no separate bookkeeping. + +**`generatorVersion` covers the two things that determine output but are neither a grammar nor a +mutation**: the pseudorandom stream's algorithm identity (research D2) and the reduction +catalogue's *order* (§ 6). Both are reproducibility-critical — FR-001 depends on the stream, +FR-020 on the order — and folding either into `mutationCatalogVersion` would name it for +something it is not, so a deliberate change to reduction order would look like a change to the +mutation operators. + +**Validation**: any `pinnedInputSet` element naming a revision absent from `revisions.json` is +**D5**. + +--- + +## 5. Mutation operator catalogue (in code, not data) + +Eleven categories mandated by FR-008. The catalogue lives in `mutate.rs` rather than as a data +file because each operator is executable logic, and `mutationCatalogVersion` pins its identity. + +| Category | Operator sketch | Grammar input (research D1) | +|---|---|---| +| `unknown-field` | insert a key absent from the schema at a pointer | `additional-properties`, `property-existence` | +| `wrong-type` | replace a value with one of a different JSON type | `type` | +| `null-value` | replace a value with `null` | `type`, `value-shape` | +| `empty-value` | empty a collection or string in place | `array-shape`, `type` | +| `conflicting-source` | add a second config source (image + Dockerfile + compose) | `union-alternative` | +| `invalid-feature-id` | corrupt a Feature identifier's registry/path/tag shape | `property-existence` under `features` | +| `extends-cycle` | introduce a cycle into an `extends` chain | `property-existence` | +| `substitution-edge` | nest, self-reference, or leave unterminated a `${…}` token | `type` (string-valued fields) | +| `lifecycle-shape` | switch between the permitted string/array/object forms | `union-alternative` | +| `compose-combination` | vary service count, `runServices`, override-file ordering | `union-alternative`, `array-shape` | +| `ordering-change` | permute a declaration-ordered collection | `array-shape` | + +Each application records its `mop-` on the witness (FR-009), which is what lets a candidate +name the operators that produced it and what lets shrinking un-apply one operator as a reduction +step (research D5). + +--- + +## 6. Reduction step catalogue (in code, ordered) + +Ordered because greedy reduction is order-sensitive and FR-020 requires the same finding and seed +to yield the identical minimal input. The order is part of **`generatorVersion`** — not +`mutationCatalogVersion`, which names the mutation operator set and would misdescribe it. + +1. `drop-optional-key` — remove a key the grammar does not mark `required` +2. `un-apply-mutation` — reverse one recorded mutation operator +3. `empty-collection` — replace a non-empty array/object with an empty one +4. `collapse-extends-level` — inline one `extends` parent and remove the link +5. `drop-compose-service` — remove one service not referenced by `service`/`runServices` +6. `minimize-scalar` — replace a scalar with the schema-minimal value of its own type +7. `drop-feature` — remove one entry from `features` + +Every step keeps the intermediate schema-plausible, so almost every probe is informative +(research D5). `isMinimal` is true only when all seven have been applied once with no step +preserving the signature — which makes FR-021 a finite, checkable claim. + +--- + +## 7. Metamorphic relation — `conformance/registry/metamorphic.json` + +Hand-authored, **inside** the registry, because a relation is an assertion the project makes and +references `clu-`/`bhv-` ids only the registry loader resolves (research D11). + +``` +MetamorphicRelation { + id: "mrl-" + transformation: string // what is applied to the input + effect: "invariance" | "sensitivity" + ground: "clu-" | "bhv-" // required (FR-045) + channels: ["chan-…"] // channels the relation asserts over + rationale: string +} +``` + +### Mandated relation families (FR-044) + +| `mrl-` | Effect | Transformation | +|---|---|---| +| `mrl-formatting-invariance` | invariance | reindent, rewrap, change insignificant whitespace | +| `mrl-comment-invariance` | invariance | insert JSONC comments and trailing commas | +| `mrl-key-order-invariance` | invariance | permute keys within an unordered map | +| `mrl-path-relocation` | invariance | relocate the workspace to a different absolute path | +| `mrl-lifecycle-equivalence` | invariance | switch between equivalent lifecycle command forms | +| `mrl-extends-flattening` | invariance | replace an `extends` chain with its hand-flattened equal | +| `mrl-declaration-order-sensitivity` | **sensitivity** | permute a declaration-ordered collection | + +The last is the only sensitivity relation in the mandated set and it is the one that catches what +the differential cannot: if deacon and the reference *both* wrongly ignore declaration order, the +differential is clean and the defect is invisible. A sensitivity relation asserts the result +**must** change, so consistent-wrongness fails it. + +`mrl-path-relocation` compares modulo the declared path tokenization (FR-046) and reports any +residual the tokenization does not account for — which makes it a live check on the tokenizer as +well as on deacon. + +**Validation**: unresolvable `ground`, unknown `effect`, or a duplicate transformation is **V31**; +a mandated family with no record is **V32**. + +--- + +## 8. Corpus entry — `conformance/discovery/corpus.json` + +The 33 pinned entries, moved out of the Python fetcher into Rust-owned strict JSON so the +immutable-reference check runs hermetically (research D8). + +``` +CorpusEntry { + id: "cor-" + name: string + repository: string // "owner/repo" + commit: string // 40-hex, immutable — a branch or tag is D4 + path: string // workspace root within the repository + contentDigest: string | null // null until first materialization; verified thereafter + notes: string +} +``` + +**`commit` must be a 40-hex object name.** A branch name, a tag, `HEAD`, or `latest` is **D4**, +rejected at load — hermetically, on every PR, without network access. This is the point of moving +the manifest into Rust: a validation that only runs when the network is up is a validation that +does not run. + +**`contentDigest` is null exactly once**, at first materialization. Every later fetch verifies it +(FR-051); a mismatch fails that entry loudly rather than comparing against unexpected content. A +non-null digest that goes missing on re-authoring is **D4**. + +Corpus content is never vendored (FR-053) — this file records provenance, not bytes. + +--- + +## 9. Reviewable candidate (generated, `target/discovery/candidates//`) + +Not version-controlled. Assembled per FR-024 from records above, self-contained per FR-027. + +``` +target/discovery/candidates// +├── fixture/ # the minimal fixture, as a materializable workspace tree +├── context.json # operations + argv; the campaign and seed; pinnedInputSet +├── raw.json # both sides' evidence, unnormalized ← separate from normalized +├── normalized.json # both sides' normalized evidence + the diff +├── provenance.json # what the deacon side was compared against; mutation operators applied +└── mapping.json # suggested behavior mapping, or an explicit no-match +``` + +`raw.json` and `normalized.json` are separate files, mirroring the committed-snapshot layout — raw +and normalized evidence must never be conflated (FR-014, the FR-016 precedent from 022). + +`provenance.json`'s `reference` block names the comparison shape in its `kind`, because there are +two and conflating them would be a lie in the one file whose job is to say what the evidence is: +`verified-oracle` (every campaign tier — the pinned reference, with its version and the fact that +it was verified) and `injected-self-comparison` (the FR-042a pipeline proof — deacon against its +own unperturbed run, with a known difference planted at the sealed evidence-source boundary, naming +the injection). The second is evidence that the *pipeline* works, never that the two +implementations disagree, and it says so rather than leaving a reviewer to infer it. + +`mapping.json` carries either a resolvable `bhv-` id or `{"match": "none"}`. It never invents an +id (FR-025): a suggestion that fabricates a behavior identity would make the reviewer's job +verifying a plausible-looking id rather than deciding one. + +--- + +## 10. Relationships + +``` +Campaign ─1:N─> Candidate ─produces─> Divergence ─classified─> Signature + │ 1:1 + ▼ + Witness ─N:1─> Finding ──promotedTo──> Case (registry) + │ ▲ + └── references Campaign + Candidate │ + │ +MetamorphicRelation ──ground──> Clause | Behavior ─────covered by──┘ +CorpusEntry ─materializes─> Candidate (network lane only; never a mutation seed, FR-008a) +``` + +The one arrow that crosses the root boundary is `Finding.promotedTo → Case`, and it points +**out** of the discovery root into the registry. Nothing in the registry points back. That +asymmetry is what makes the queue unreachable from `certify` (research D6): following references +from the registry can never arrive at a finding. + +--- + +## 11. Validation summary + +Registry-side, emitted by `conformance validate`, blocking a PR via `registry_valid`: + +| Class | Statement | +|---|---| +| **V31** | metamorphic relation integrity: unresolvable `ground`, unknown `effect`, duplicate transformation | +| **V32** | a mandated relation family (FR-044) has no relation record | + +Discovery-side, emitted by `conformance discovery check`, blocking a PR via a new hermetic test: + +| Class | Statement | +|---|---| +| **D1** | malformed queue record, or a signature naming an undeclared channel | +| **D2** | a finding with zero or more than one classification while in state `triaged` or later | +| **D3** | a `promoted` finding whose `promotedTo` does not resolve to a real case | +| **D4** | a corpus entry with a non-immutable reference, or a digest that was recorded and then removed | +| **D5** | a finding or campaign whose `pinnedInputSet` names a revision absent from `revisions.json` | + +D-classes are numbered separately from the V-series on purpose: they are emitted by a different +command over a different data root. Folding them into V-numbering would imply the registry +validator can see the queue, which research D6 says it must not. diff --git a/specs/025-exploratory-parity-discovery/plan.md b/specs/025-exploratory-parity-discovery/plan.md new file mode 100644 index 00000000..13f7812e --- /dev/null +++ b/specs/025-exploratory-parity-discovery/plan.md @@ -0,0 +1,161 @@ +# Implementation Plan: Exploratory Parity Discovery + +**Branch**: `025-exploratory-parity-discovery` | **Date**: 2026-07-27 | **Spec**: [spec.md](./spec.md) +**Input**: Feature specification from `/specs/025-exploratory-parity-discovery/spec.md` + +## Summary + +Add a discovery pipeline that finds parity differences the curated record never anticipated: +generate valid and near-valid configurations from the **already-committed constraint inventory**, +mutate known-valid fixtures with an attributable operator catalogue, compare deacon against the +verified pinned oracle, reduce each difference structurally while preserving its normalized +signature, and emit a reviewable candidate. Findings accumulate in a persistent queue that lives +*outside* the conformance registry and is structurally unreachable from `certify`; nothing enters +the deterministic record except by human review. + +The technical approach is deliberately additive rather than parallel. The grammar is the existing +`conformance/inventory/constraints.json` (609 units, fingerprint-verified against the vendored +pinned schemas), the signature derives from `normalize::diff`'s existing `ConfigDivergence` +output, and the pipeline proof reuses `inject.rs`'s sealed `EvidenceSource` boundary. Discovery +introduces no second normalization path, no second schema view, and no new workspace crate. + +## Technical Context + +**Language/Version**: Rust, Edition 2024, MSRV 1.95 (`unsafe_code = "deny"` workspace-wide) +**Primary Dependencies**: existing workspace deps only — `serde`/`serde_json` (strict-JSON records), `indexmap` (declaration order), `sha2` (`hash8` signature/fixture ids), `tokio` (bounded async exec), `thiserror` (domain errors), `tracing`, `tempfile` (dev-dep, isolated workspaces). **No new crates** — including no RNG crate (research D2). +**Storage**: strict-JSON, version-controlled. New root `conformance/discovery/` (queue, campaigns, corpus manifest) — a sibling of `registry/`, deliberately outside it. New registry file `conformance/registry/metamorphic.json` (`mrl-` relation records). Generated artifacts under `target/discovery/` (git-ignored, byte-stable). All writes atomic (temp file + `fs::rename`). +**Testing**: `cargo-nextest`. Hermetic tests (generator, mutation, shrinker, signature, queue validation, relation validation) run in `default`/`dev-fast`. Live campaigns run **only** under a new `[profile.discovery]` with an explicit `binary(=…)` allow-list. +**Target Platform**: Linux/macOS developer machines and CI. Hermetic differential tier needs Node 20 + the pinned oracle; container-backed tier additionally needs Docker; corpus canary additionally needs network. The metamorphic tier needs none of the three (research D12). +**Project Type**: development-only tooling inside an existing Rust workspace — libraries plus dev-only bins. Never part of the shipped `deacon` consumer surface (FR-059, asserted by `parity_registry_check`). +**Performance Goals**: scheduled campaign ≤ 30 min wall clock; ≤ 60 s per hermetic candidate, ≤ 5 min per container-backed candidate; ≥ 90% of candidates reach past document parsing (SC-002); median shrink ≥ 80% input-size reduction (SC-004). +**Constraints**: byte-stable outputs (no timestamps, no absolute paths); zero network and zero discovery-program selection in any PR lane (FR-055); no write path from any discovery program into the registry or snapshots (FR-036); exactly one normalization definition (FR-015); a seed must reproduce across dependency bumps (FR-001 → research D2). +**Scale/Scope**: grammar = 609 constraint units (469 non-annotation); mutation seeds = ~90 committed fixtures under `conformance/fixtures/`; corpus = 33 pinned entries; 11 observable channels; admission cap 25 new distinct signatures per campaign (research D10). + +## Constitution Check + +*GATE: Must pass before Phase 0 research. Re-checked after Phase 1 design.* + +| Principle | Verdict | Basis | +|---|---|---| +| **I. Spec-Parity as Source of Truth** | **PASS** | Strengthens it. The grammar is derived from the pinned spec surface, so generation explores what the spec permits rather than what a maintainer imagined. Findings are bound to the pin under which they were observed; a pin bump invalidates rather than carries them (Assumption 8). | +| **II. Consumer-Only Scope** | **PASS** | Dev-only tooling, identical posture to `deacon-conformance` and `parity-harness`. FR-059 forbids any consumer surface; `parity_registry_check` already asserts `deacon --help` gains nothing and is extended to cover the new bins (research D9). | +| **III. Keep the Build Green** | **PASS** | Hermetic tests run in `dev-fast`/`default`; every live discovery binary is excluded from all six existing profile `default-filter`s. A discovery failure cannot fail a PR (FR-058). | +| **IV. No Silent Fallbacks — Fail Fast** | **PASS** | Missing or mismatched pins fail loudly (FR-003); an injection that never landed is `InjectionInapplicable`, never "found nothing" (FR-042a); admission-cap suppression is always reported (FR-034b); a corpus digest mismatch fails that entry (FR-051). No `#[ignore]`, no silent skip. | +| **V. Idiomatic, Safe Rust** | **PASS with one tracked item** | Edition 2024, no `unsafe`, `thiserror` in the libraries, async only for bounded oracle exec. The in-repo PRNG (research D2) reimplements crate functionality — justified in Complexity Tracking; pure wrapping integer arithmetic, no `unsafe`. | +| **VI. Observability & Output Contracts** | **PASS** | Reports are deterministic and byte-stable; the new bins keep the single-JSON-document-on-stdout / diagnostics-on-stderr contract. Ordered structures (`Vec`/`IndexMap`) throughout — the `ordering-changed` signature class exists precisely because order defects are real here. | +| **VII. Testing Completeness** | **PASS** | Every mandated acceptance test in the spec maps to a test. New profile + all-profile exclusions wired per Principle VII's nextest rule; `parity_registry_check` extended so the wiring cannot drift. | +| **VIII. Subcommand Consistency & Shared Abstractions** | **PASS** | Reuses `normalize` (single definition, FR-015), `exec`, `oracle`, `prereq`, `observe`, `inject`, `hash8`, and the inventory loader. The plan explicitly forbids a second normalization or schema-extraction path (research D1, D3). | +| **IX. Executable & Self-Verifying Examples** | **N/A** | No `examples/` surface: this is dev tooling with no user-facing scenario. `quickstart.md` carries the runnable walkthrough instead, in line with 019–024. | + +**Gate result: PASS.** One item tracked in Complexity Tracking; no unjustified violation. + +### Post-Phase-1 re-check + +Re-evaluated after `data-model.md` and `contracts/` were written. No verdict changed. Two +design outcomes strengthen earlier PASSes rather than qualifying them: + +- **Principle IV**: `contracts/discovery-cli.md` gives every discovery command an exit-status + contract where the status reflects *whether the command ran*, never *what it found* — + extending the `coverage report` discipline to the whole surface, so no discovery output can + become a gate by accident. +- **Principle VIII**: the shrinker takes its reproduction predicate as a parameter (research + D4/D5), so reduction strategy stays hermetic and unit-testable while the live campaign + supplies the real predicate. Oracle access remains confined to the crate that already owns it. + +## Project Structure + +### Documentation (this feature) + +```text +specs/025-exploratory-parity-discovery/ +├── plan.md # This file +├── spec.md # Feature specification (clarified 2026-07-27) +├── research.md # Phase 0 output — 12 decisions +├── data-model.md # Phase 1 output — record shapes, validation, state +├── quickstart.md # Phase 1 output — runnable walkthroughs +├── contracts/ # Phase 1 output +│ ├── discovery-cli.md # dev-only command surface + exit-status contracts +│ ├── findings-queue.md # queue record contract + D1–D5 violations +│ └── metamorphic-catalogue.md # mrl- record contract + V31/V32 +├── checklists/ +│ └── requirements.md # Spec quality checklist (passed) +└── tasks.md # Phase 2 output (/speckit.tasks — NOT created here) +``` + +### Source Code (repository root) + +```text +crates/conformance/src/ # HERMETIC half (no oracle, no Docker, no network) +├── discovery/ +│ ├── mod.rs # public surface for the hermetic half +│ ├── grammar.rs # constraint inventory -> generation grammar (D1) +│ ├── rng.rs # in-repo deterministic PRNG (D2) +│ ├── generate.rs # constrained candidate generation +│ ├── mutate.rs # the 11-category mutation operator catalogue +│ ├── shrink.rs # structural reduction; predicate is a parameter (D5) +│ ├── signature.rs # normalized signature + value-shape class (D3) +│ ├── queue.rs # findings-queue model, loader, D1-D5 validation (D6) +│ ├── metamorphic.rs # mrl- relation model + V31/V32 (D11) +│ ├── corpus.rs # corpus manifest model + immutable-ref validation (D8) +│ └── report.rs # byte-stable campaign + queue reports +├── validate.rs # EXTEND: V31, V32 +├── load.rs # EXTEND: load metamorphic.json (registry side only) +└── bin/conformance.rs # EXTEND: `discovery` command group + +crates/parity-harness/src/ # LIVE half (oracle / Docker / network) +├── discovery/ +│ ├── mod.rs +│ ├── campaign.rs # driver: budget, seed, admission cap, tiers +│ ├── differential.rs # deacon vs oracle over a candidate +│ ├── metamorphic_run.rs # deacon-only relation evaluation (D12) +│ ├── minimize.rs # supplies the live reproduction predicate to shrink.rs +│ ├── candidate.rs # assemble the reviewable candidate +│ ├── corpus_fetch.rs # network-lane fetch + digest verification (D8) +│ └── pipeline_proof.rs # injected-difference proof via sealed EvidenceSource (D7) +└── bin/ + ├── discovery-campaign.rs # run a campaign (seed + budget required) + └── discovery-proof.rs # the FR-042a pipeline proof + +crates/deacon/tests/ +├── discovery_hermetic.rs # hermetic guards; runs in default/dev-fast +├── discovery_campaign.rs # LIVE; [profile.discovery] allow-list only +├── discovery_metamorphic.rs # deacon-only, no external prereq; discovery profile +│ # (excluded from PR lanes for stochasticity, not cost) +└── parity_registry_check.rs # EXTEND: discovery lane wiring + no consumer surface + +conformance/ +├── discovery/ # NEW ROOT — outside registry/, unreachable from certify +│ ├── findings.json # the persistent findings queue +│ ├── campaigns.json # campaign provenance (seed + pinned input set) +│ └── corpus.json # 33 pinned entries + content digests +└── registry/ + └── metamorphic.json # NEW — hand-authored mrl- relation records + +.config/nextest.toml # NEW [profile.discovery]; exclusions in all 6 profiles +.github/workflows/discovery.yml # NEW — scheduled + workflow_dispatch lanes +Makefile # NEW targets: test-discovery, test-discovery-proof +``` + +**Structure Decision**: No new workspace crate. The feature splits along the **hermetic/live** +line established by 022 — pure data-to-data logic (grammar, generation, mutation, reduction +strategy, signature, queue, relations, reports) in `deacon-conformance`; everything touching the +oracle, Docker, or the network in `parity-harness`. This keeps the generator, shrinker, and +signature unit-testable in the fast lane with no external dependency, which is what makes the +hermeticity claim of FR-055 cheap to hold rather than a thing to be careful about. Two data roots +are introduced rather than one: `conformance/discovery/` (machine-produced candidates, outside +the registry) and one registry file `metamorphic.json` (hand-authored assertions, inside it) — +the split is the subject of research D11. + +## Complexity Tracking + +| Violation | Why Needed | Simpler Alternative Rejected Because | +|-----------|------------|-------------------------------------| +| In-repo PRNG instead of the `rand` crate (research D2) | FR-001 requires a recorded seed to reproduce an identical candidate sequence, and FR-034 requires findings to persist indefinitely — so a seed recorded today must still reproduce after arbitrary dependency updates. | `rand` does not offer value-stream stability across versions; its stream is documented as an implementation detail. Pinning it to freeze behavior fights Principle V's dependency-hygiene rule and still breaks under a forced advisory bump. Making the stream a property of committed code turns a silent corpus-wide invalidation into a reviewable pin change, exactly as `NORMALIZER_VERSION` does. ~40 lines, no `unsafe`, tested against published vectors. | +| A second data root under `conformance/` (research D6) | The clarified requirement is that no finding — reviewed or not — can influence `certify`. `load.rs` enumerates named subdirectories under `conformance/registry/`, so a sibling of `registry/` is structurally unreachable rather than merely conventionally separate. | Placing the queue inside `conformance/registry/` means either the loader rejects it, or someone wires it in and unreviewed, machine-produced findings join the certification denominator — the silent failure mode 024 D1 documented. A `target/`-only queue cannot satisfy FR-030's cross-campaign deduplication or FR-034's persistence. | + +## Phase 2 note + +`/speckit.plan` stops here. `tasks.md` is produced by `/speckit.tasks`. Sequencing guidance for +that step, from research D12: the **metamorphic tier is the cheapest complete vertical slice** — +it exercises generation → comparison → signature → candidate with no oracle, no Docker, and no +network, so building it first proves the hermetic spine before any live provisioning exists. diff --git a/specs/025-exploratory-parity-discovery/quickstart.md b/specs/025-exploratory-parity-discovery/quickstart.md new file mode 100644 index 00000000..1bef6722 --- /dev/null +++ b/specs/025-exploratory-parity-discovery/quickstart.md @@ -0,0 +1,302 @@ +# Quickstart: Exploratory Parity Discovery + +**Feature**: `025-exploratory-parity-discovery` + +Five workflows: run a campaign, triage what it found, promote a finding, add a metamorphic +relation, and re-pin the real-world corpus. Everything here is **development-only** — none of it +is a `deacon` subcommand. + +--- + +## Prerequisites by tier + +| Tier | Oracle + Node | Docker | Network | +|---|---|---|---| +| `metamorphic` | — | — | — | +| `config-differential` | required | — | — | +| `container-differential` | required | required | — | +| `corpus` | required | — | required | + +Provision the pinned oracle: + +```bash +npm install -g @devcontainers/cli@$(jq -r .version fixtures/parity-corpus/oracle.json) +``` + +A missing or wrong-version oracle **fails the campaign loudly**. There is no skip. + +--- + +## 1. Run a campaign + +```bash +# The cheapest useful run — no oracle, no Docker, no network. +cargo run -p parity-harness --bin discovery-campaign -- \ + --seed 0x5eed1234 --tier metamorphic + +# The scheduled tier. +cargo run -p parity-harness --bin discovery-campaign -- \ + --seed 0x5eed1234 --tier config-differential --budget-seconds 1800 + +# Or via the profile, which runs every registered discovery binary. +make test-discovery +``` + +`--seed` is required and never defaulted. Record it — it is the reproducibility input, and a +finding you cannot re-derive is a finding nobody can act on. + +**Reading the outcome.** The campaign prints a single JSON document to stdout: + +```json +{ + "id": "cmp-9f3a2b71", + "seed": "0x5eed1234", + "outcome": { + "candidatesGenerated": 4820, + "parseStageFailures": 191, + "signaturesObserved": 7, + "signaturesAdmitted": 7, + "signaturesSuppressed": 0, + "mutationApplications": { "unknown-field": 512, "wrong-type": 498, "...": 0 } + } +} +``` + +Three numbers to check before anything else: + +- **`parseStageFailures / candidatesGenerated` must be under 10%** (SC-002). Above that, the + generator is producing garbage and the campaign explored the parser rather than the tool. +- **A zero in `mutationApplications`** is a named generation deficiency (FR-010), not a + non-event. Some category never successfully applied. +- **`signaturesSuppressed > 0`** means the admission cap was hit. The campaign still exited `0` — + suppression is reported, never silent, and never fails the run. + +**A campaign exits `0` whether it finds nothing or forty things.** Non-zero means the machinery +failed: an unverifiable oracle, a normalization failure, an unwritable data root. + +--- + +## 2. Triage the queue + +```bash +cargo run -p deacon-conformance -- discovery report +$EDITOR target/discovery/queue.md +``` + +The report separates states that are easy to conflate: + +| Bucket | Means | +|---|---| +| `untriaged` | nobody has looked yet — **counted**, so it never reads as "nothing found" | +| `triaged` | classified, awaiting a decision | +| `no-longer-reproducing` | stopped reproducing; names the campaign that last saw it | +| `promoted` | now carried by a real case, named | +| `pin-stale` | observed under pins that no longer match; awaiting re-evaluation | + +Classify one: + +```bash +cargo run -p deacon-conformance -- discovery triage fnd-3c9e11a4 \ + --classification deacon-regression \ + --notes "extends child overrides parent remoteUser; reference keeps the parent" +``` + +**Two classifications are dead ends by design.** `normalizer-defect` and `fixture-defect` describe +a defect in the discovery machinery, not a behavior of either implementation, and cannot be +promoted (FR-035). Fix the normalizer or the generator instead. + +**When to reach for `split`**: a single signature merged witnesses that turn out to have different +causes. Splitting is permanent — the deduplication rule never re-merges a split lineage, so a +reviewer's judgement is not silently reverted by the next campaign. + +```bash +cargo run -p deacon-conformance -- discovery split fnd-3c9e11a4 +``` + +--- + +## 3. Promote a finding + +Promotion is **entirely manual**. No command writes into the registry; `scaffold` prints to stdout +and stops. + +```bash +cargo run -p deacon-conformance -- discovery scaffold fnd-3c9e11a4 +``` + +That emits a skeleton behavior, case, and fixture layout with `UNREVIEWED` sentinels the loader +rejects. Then, by hand: + +1. **Author the behavior** in `conformance/registry/behaviors/.json` with all three axes. + A finding does not tell you the disposition — it tells you what differs. Deciding whether + deacon is wrong, the reference is wrong, or the spec is silent is the review. +2. **Copy the minimal fixture** from `target/discovery/candidates/fnd-3c9e11a4/fixture/` into + `conformance/fixtures/fx-/`. +3. **Author the case** in `conformance/registry/cases/.json` with a **full** + `scenarioContext` — every scenario dimension assigned, or none (V26). +4. **Flip the `odp-cmb-*` dispositions** the new case covers off `gap`, in the **same commit**. + Skipping this leaves the coverage report claiming a hole that is filled — the trap 024 + documented. +5. Validate and record the promotion: + +```bash +cargo run -p deacon-conformance -- validate +cargo run -p deacon-conformance -- coverage check +cargo run -p deacon-conformance -- discovery check +``` + +Step 5's third command is what closes the loop: it fails **D3** if the finding claims +`promotedTo` a case that does not exist. + +**Tolerating instead of fixing**: author a scoped `wvr-` waiver with a rationale and an `expires`, +then reference it from a scoped `allowedDifferences` entry on the case. Never a blanket scope — a +waiver whose difference stops reproducing must fail as stale, and a blanket one cannot (FR-041). + +--- + +## 4. Add a metamorphic relation + +Relations live **in** the registry, because they are assertions the project makes. + +```jsonc +// conformance/registry/metamorphic.json — the committed `mrl-comment-invariance` record +{ + "id": "mrl-comment-invariance", + "transformation": "insert JSONC line comments and trailing commas at every legal position", + "effect": "invariance", + "ground": "bhv-readconfig-malformed-jsonc-rejected", + "channels": ["chan-structured-output"], + "rationale": "The grounding behavior fixes the parse dialect as JSON with Comments and records that only a hard syntax error is rejected. A comment and a trailing comma are well-formed in that dialect and denote no member, so neither can alter the value the document describes." +} +``` + +**A `ground` is either a `bhv-` behavior or a `clu-` clause**, and both are in use — but a +`clu-` id is a *substance-anchored slug*, not a short hash: the committed +`mrl-key-order-invariance` grounds on +`clu-json-reference-this-property-allows-you-to-override-the-feature-desc-bebdddcd`. Copy the +real id from `conformance/inventory/clauses.json`; an invented one is V31. + +Then: + +```bash +cargo run -p deacon-conformance -- validate # V31/V32 +cargo run -p parity-harness --bin discovery-campaign -- --seed 0x1 --tier metamorphic +``` + +**`ground` is not optional and not decorative.** An ungrounded invariance relation that is wrong +does not fail — it *passes*, asserting something the spec never said. V31 blocks that. + +**Consider a sensitivity relation.** Invariance relations catch a tool reading meaning that is not +there. Sensitivity relations catch a tool *ignoring* meaning that is — and they are the only thing +that catches deacon and the reference being consistently wrong together, which the differential +cannot see because both sides agree. + +--- + +## 5. Re-pin the real-world corpus + +```jsonc +// conformance/discovery/corpus.json — the committed `images-python` entry +{ + "id": "cor-eb074204", + "name": "images-python", + "repository": "devcontainers/images", + "commit": "31b61b521d55926d62c748b659f24ae71774c0e3", + "path": "src/python", + "contentDigest": null, + "notes": "Dockerfile build with multiple official features." +} +``` + +```bash +cargo run -p deacon-conformance -- discovery check # D4, hermetic — no network needed +cargo run -p parity-harness --bin discovery-campaign -- --seed 0x2 --tier corpus +``` + +- **`id` is derived from the entry's own substance, never hand-authored.** A made-up id is + **D4** ("a corpus id that does not derive from its own substance"). Add the entry with a + placeholder, run `discovery check`, and take the id the violation names. +- **`commit` must be a 40-hex object name.** A branch, tag, `HEAD`, or `latest` is **D4**, + rejected hermetically on every PR. This is why the manifest is Rust-owned data rather than + Python: a validation that only runs when the network is up is a validation that does not run. +- **`contentDigest: null` exactly once.** First materialization records it; every later fetch + verifies it. A mismatch fails that entry rather than comparing against unexpected content. +- **Content is never vendored** (FR-053). The manifest records provenance; the bytes stay upstream. + +--- + +## Where things live + +| Path | What | Version-controlled? | +|---|---|---| +| `conformance/discovery/findings.json` | the queue | yes — outside the registry | +| `conformance/discovery/campaigns.json` | campaign provenance | yes | +| `conformance/discovery/corpus.json` | 33 pinned entries | yes | +| `conformance/registry/metamorphic.json` | relation catalogue | yes — inside the registry | +| `target/discovery/queue.{json,md}` | rendered report | no | +| `target/discovery/candidates//` | reviewable candidates | no | + +**The queue is unreachable from `certify` by construction**, not by convention: the registry +loader enumerates named subdirectories under `conformance/registry/`, and `conformance/discovery/` +is a sibling of that directory, not a member. No function walks from one to the other. + +--- + +## Troubleshooting + +**"Campaign exits 1 immediately."** A prerequisite failed. Check the oracle's exact version +against `fixtures/parity-corpus/oracle.json` — the harness verifies exactly, not +greater-than-or-equal, and a near-miss version fails. + +**"Same seed, different candidates."** Something in the pinned input set moved. Compare the +campaign's recorded `pinnedInputSet` against the current one; a re-vendored schema pin changes +`grammarVersion`, which legitimately changes the stream. This is invalidation working, not a bug. + +**"A finding I fixed keeps coming back."** Check its state. If it is `no-longer-reproducing` it is +being *reported*, not re-found — the record is retained deliberately, so that "a fix landed" stays +distinguishable from "the generator stopped reaching that input". + +**"`discovery check` fails D5 after a pin bump."** Expected. Findings are claims about a specific +pinned pair of implementations; on a pin change they are re-evaluated, not carried forward. Run a +campaign under the new pins and let each finding reproduce or lapse. + +**"The queue has 200 untriaged findings."** Check `signaturesSuppressed` across recent campaigns. +Repeatedly hitting the admission cap usually means one generator change surfaced a systemic +divergence rather than many independent ones — triage a few and look for a shared cause before +working through the list. + +**"I ran both CLIs on the candidate's fixture and the outputs look identical."** You are +comparing **raw** output; the signature is defined on **normalized** output. This is the one +step of the SC-017 reproduction that is not self-evident from the candidate, so do it +explicitly: + +```bash +# 1. The fixture IS the reproduction input. Copy it somewhere fresh. +cp -r target/discovery/candidates//fixture/. /tmp/repro/ + +# 2. Verify your oracle equals the candidate's `pinnedInputSet.oracleVersion` — exactly. +devcontainer --version + +# 3. Run the argv recorded in context.json on both sides. +devcontainer read-configuration --workspace-folder /tmp/repro > ref.json +deacon read-configuration --workspace-folder /tmp/repro > deacon.json + +# 4. Compare the way the pipeline does. `raw.json` and `normalized.json` in the candidate +# are committed side by side precisely so you can see what the rules changed. +diff <(jq -S . ref.json) <(jq -S . deacon.json) +``` + +If step 4 shows no difference at the signature's `path` while `normalized.json` does, the +difference is produced by a **named normalization rule**, not by the two CLIs disagreeing. +The usual one is `drop_absent_optional`, which applies to **deacon's `configuration` block +only**: deacon serializes every modeled optional unconditionally, so the rule elides its +empty values — but it cannot distinguish *"the author wrote `"secrets": {}`"* from *"deacon +emitted an unset optional"*. The reference's `configuration` is an echo of the authored +document, so its authored empty survives. A fixture that authors an empty value for any key +in `ABSENT_OPTIONAL_KEYS` therefore yields a `ref-only` signature at that path with +**identical raw output on both sides**. + +That is a finding about the observer, not about deacon's behavior — triage it +`normalizer-defect`, which is a deliberate dead end (it cannot be promoted, FR-035). The +underlying fix is on deacon: `skip_serializing_if` so absent optionals are omitted, which +retires the rule entirely (`specs/023-migrate-parity-to-conformance/tasks.md#T111`). diff --git a/specs/025-exploratory-parity-discovery/research.md b/specs/025-exploratory-parity-discovery/research.md new file mode 100644 index 00000000..7992b983 --- /dev/null +++ b/specs/025-exploratory-parity-discovery/research.md @@ -0,0 +1,468 @@ +# Phase 0 Research: Exploratory Parity Discovery + +**Feature**: `025-exploratory-parity-discovery` +**Date**: 2026-07-27 + +Every decision below was checked against the code, not inferred. Where a decision rejects a +conventional choice, the rejection reason is a property this feature actually needs, not a +preference. + +--- + +## Decision 1 — The generator's grammar is the committed constraint inventory, not the schemas + +**Decision**: Generation draws its grammar from `conformance/inventory/constraints.json` +(609 units at the current pin), not from re-parsing `conformance/schemas//*.json`. + +**Rationale**: The inventory is already exactly what a constrained generator needs, and it is +already governed. Its non-annotation units carry the generative content: + +| Kind | Count | Generative use | +|---|---|---| +| `type` | 187 | the value domain to draw from, and the wrong-type mutation's target set | +| `property-existence` | 117 | which keys may appear at a pointer | +| `array-shape` | 41 | element type, tuple-vs-list, min/max | +| `additional-properties` | 38 | whether an unknown-field mutation is legal or near-valid there | +| `required` | 20 | which keys a *valid* instance must carry — the difference between valid and near-valid | +| `union-alternative` | 18 | the branch set for `oneOf`/`anyOf` shapes (Compose vs Dockerfile vs image) | +| `enum` / `const` | 14 | exact legal values, and the near-miss set one edit away | +| `value-shape` / `default` | 18 | scalar constraints and omission semantics | + +Three properties come free by using it: + +1. **The grammar pin is already a recorded revision.** `constraints.json` carries a `revision` + field that V14 validates against the registry schema pin. FR-002 requires the grammar + version in the pinned input set; it is already there and already guarded. +2. **A schema pin bump automatically surfaces as a generation-input change.** Re-vendoring + regenerates the inventory, `inventory diff` enumerates the delta, and every finding bound to + the old revision is correctly invalidated (FR-002, Assumption 8) with no separate bookkeeping. +3. **No second extraction path.** FR-015 forbids a second normalization definition; the same + argument applies one level up to schema interpretation. Two views of the pinned schema + surface that could disagree is the identical defect class. + +**Alternatives considered**: +- *Re-parse the vendored schemas directly.* Rejected: duplicates the extraction logic that + `inventory generate` owns, creates an ungoverned second view of the same bytes, and would + drift silently because nothing cross-checks the two. +- *Hand-author a generation grammar.* Rejected outright — it would re-import the very + maintainer imagination this feature exists to escape. A hand-written grammar generates the + shapes its author thought of, which is what curated fixtures already do. + +**Consequence for the pinned input set**: `constraints.json`'s `revision` and content +fingerprint are two of the six FR-002 elements. + +--- + +## Decision 2 — The pseudorandom stream is in-repo, not from an external crate + +**Decision**: Implement the campaign PRNG inside the discovery module — SplitMix64 seed +expansion feeding a xoshiro256\*\* stream, ~40 lines of pure integer arithmetic, unit-tested +against published reference vectors. No new dependency. + +**Rationale**: FR-001 requires that a recorded seed reproduce an identical candidate sequence, +and FR-034 requires findings to persist across campaigns — which means a seed recorded today +must still reproduce after arbitrary dependency updates. `rand` explicitly does **not** offer +value-stream stability across versions; its own documentation treats the stream as an +implementation detail. Depending on it would make every finding's reproducibility hostage to a +`cargo update`, and a security advisory could force a bump that silently invalidates the entire +recorded corpus with no signal. + +Making the stream a property of *our* committed code inverts that: the algorithm identity becomes +an explicit component of `generatorVersion` — the **seventh** element of the pinned input set +(FR-002, data-model § 4), which also carries the reduction-catalogue order (D5). A deliberate +change to either is then a recorded, reviewable pin change, exactly like `NORMALIZER_VERSION` and +for the identical reason. + +This also matches the workspace's stated posture (023 D6: existing dependencies only) and +carries no `unsafe` (Principle V): xoshiro256\*\* is wrapping integer arithmetic and shifts. + +**Alternatives considered**: +- *`rand` + `rand_chacha`, version-pinned.* Rejected: pinning a dependency specifically to + freeze behavior fights Principle V's dependency-hygiene rule ("keep dependencies current"), + and still breaks under a forced advisory bump. It trades a maintenance obligation for a + guarantee it cannot actually deliver. +- *Hash-based derivation (seed + counter → SHA-256 → bytes) using the existing `sha2`.* + Rejected as the primary stream on cost grounds — a shrink pass makes many thousands of draws + and hashing each is wasteful — but this is the fallback if the PRNG's test vectors ever prove + awkward to maintain. Noted, not adopted. + +**Complexity note**: this is the one place the plan reimplements something a crate provides. +It is tracked in the plan's Complexity Tracking table with this justification. + +--- + +## Decision 3 — The normalized signature derives from the existing `ConfigDivergence` + +**Decision**: Compute the signature from `normalize::diff`'s existing output rather than +introducing a comparison path. The clarified composition (channel + observable path + +difference kind + value-shape class) maps onto what is already there: + +``` +ConfigDivergence { kind: DiffKind, path: String, deacon: Option, reference: Option } +DiffKind = RefOnly | DeaconOnly | Value +``` + +- **channel** — supplied by the caller (`chan-structured-output`, `chan-exit-code`, …); the + observers already partition evidence this way. +- **observable path** — `ConfigDivergence::path` verbatim. +- **difference kind** — `DiffKind::as_str()`: `ref-only` / `deacon-only` / `value`. +- **value-shape class** — the one new derivation, a pure function of the two `Option`s. + `RefOnly`/`DeaconOnly` classify as `present-absent`. `Value` classifies as `type-changed` + when the two JSON types differ, `ordering-changed` when both are arrays that are permutations + of each other, and `value-changed` otherwise. + +The signature id is `hash8` over the tuple — the same helper that produces `clu-` and `cst-` +ids, so signature ids are substance-anchored in the same sense: they survive a reordering of +the finding record and change only when the difference itself changes. + +**Rationale**: FR-015 forbids a second normalization path, and a signature computed from +independently re-diffed values would be exactly that — a second opinion on what differs, able to +disagree with the one the comparison used. Deriving from the comparison's own output makes +disagreement structurally impossible. + +The value-shape class is the level at which "same defect" is true. Structure alone +(channel+path+kind) merges a missing `remoteUser` with a wrongly-typed `remoteUser`; including +concrete values splits one defect across every generated value. `ordering-changed` earns its +place separately because declaration-order defects are a known real class in this codebase +(`BTreeMap`-vs-`IndexMap` violations) and collapsing them into `value-changed` would hide a +family the project has already been bitten by. + +**Alternatives considered**: +- *Hash the whole normalized diff.* Rejected: every generated value produces a distinct hash, so + deduplication does nothing and the queue grows with campaign volume — the failure mode FR-030 + exists to prevent. +- *Channel + path only.* Rejected: merges genuinely distinct defects at the same path, and a + merged finding cannot be split back into its causes because the distinguishing information + was never recorded. + +--- + +## Decision 4 — Code splits across the two existing crates along the hermetic/live line + +**Decision**: No new workspace crate. Hermetic logic goes in `deacon-conformance`; live +execution goes in `parity-harness`. + +| Concern | Crate | Why | +|---|---|---| +| grammar loading, generation, mutation catalogue | `deacon-conformance` | pure data → data; needs the inventory loader that already lives there | +| shrink strategy (which reductions, in what order) | `deacon-conformance` | pure; the *predicate* is injected by the caller | +| signature computation | `deacon-conformance` | pure function over diff output | +| findings-queue model, loader, validation | `deacon-conformance` | strict-JSON records, same shape as every other record kind | +| report rendering | `deacon-conformance` | byte-stable, no I/O beyond the write | +| campaign driver, oracle invocation, evidence capture | `parity-harness` | already owns `exec`/`oracle`/`prereq`/`observe`/`normalize` | +| corpus fetch | `parity-harness` | the only network-touching code | +| injected-difference proof | `parity-harness` | reuses `inject.rs`'s sealed boundary | + +**Rationale**: This is the 022 precedent applied unchanged — "the hermetic data/validation/ +staleness logic lives in `deacon-conformance`; the live execution/observation/record logic in +`parity-harness`." Following it means the generator, shrinker, and signature are unit-testable +in the fast lane with no oracle, no Docker, and no network, which is what makes FR-055's +hermeticity claim cheap to hold rather than a thing to be careful about. + +The shrinker deserves a specific note: it takes the reproduction predicate as a parameter rather +than calling the oracle itself. That keeps the reduction *strategy* hermetic and unit-testable +against a synthetic predicate, while the live campaign supplies the real one. Without this split +the shrinker could only be tested by running a campaign. + +**Alternatives considered**: +- *A fifth crate, `deacon-discovery`.* Rejected: nothing needs isolating, and it would need to + depend on both existing crates to reach `normalize` and the inventory loader — reproducing + the dependency shape the current split already has, with an extra compilation unit and a + third place to look for parity vocabulary. +- *All of it in `parity-harness`.* Rejected: the queue records are registry-adjacent data whose + loader and validator belong with the other loaders, and putting them in the live crate would + make queue validation require Docker to compile-and-test in practice. + +--- + +## Decision 5 — Shrinking is structural delta-debugging over parsed JSON, not text ddmin + +**Decision**: Reduce the parsed configuration document with an ordered catalogue of structural +steps: remove an optional key, empty a collection, collapse one `extends` level, replace a +scalar with the schema-minimal value of its own type, drop a Compose service, un-apply one +mutation operator. Never reduce at the byte or line level. + +**Rationale**: Text-level ddmin on JSON produces syntactically broken intermediates. Each one +fails at the document-parse stage, which (a) cannot reproduce a signature that lives past +parsing, so the reduction is wasted, and (b) costs a full oracle invocation to discover — and +the oracle invocation is the expensive step of the entire feature. A campaign whose shrinker +spends most of its budget on malformed intermediates is the same pathology SC-002 guards against +at generation time, relocated to minimization. + +Structural steps keep every intermediate schema-plausible, so nearly every probe is informative. +They also make FR-021 checkable: "minimal with respect to the declared catalogue" is a finite, +enumerable claim — apply each of the N steps once, confirm none preserves the signature — rather +than an unfalsifiable assertion about all possible smaller inputs. + +The catalogue is ordered and the order is part of `generatorVersion`, because FR-020 requires +the same finding and seed to yield the identical minimal input, and greedy reduction is +order-sensitive. + +**Alternatives considered**: +- *`proptest`/`quickcheck` integrated shrinking.* Rejected on three counts: their shrinkers are + coupled to their generators, so adopting the shrinker means adopting their generation model + instead of the constraint inventory (killing Decision 1); their shrink order is not stable + across versions (the Decision 2 problem again); and the shrink predicate here is an expensive + external process, which their designs assume is cheap. +- *Reduce toward a fixed minimal document rather than by steps.* Rejected: it discards the + mutation provenance that FR-009 requires the candidate to carry. + +--- + +## Decision 6 — The findings queue sits at `conformance/discovery/`, structurally out of reach + +**Decision**: `conformance/discovery/` — a sibling of `registry/`, alongside the existing +`inventory/`, `migration/`, `obligations/`, `snapshots/`, `spec/`, `schemas/`. + +**Rationale**: Verified against `crates/conformance/src/load.rs`: the registry loader enumerates +*named* subdirectories under `conformance/registry/` (`cases/`, `behaviors/`, `sources/`, +`waivers/`, `classifications/`, `clause-classifications/`, `obligation-dispositions/`). It has no +wildcard directory walk at the registry root, so a sibling of `registry/` cannot be picked up — +not by convention, but because there is no code path that would reach it. `certify` consumes the +loaded record only. + +This makes the clarified guarantee — an unreviewed finding can never influence a release gate — +a property of the directory layout rather than a rule someone must remember. That distinction +matters here specifically: the failure mode is silent (a finding quietly joins the denominator), +which is precisely the class of mistake 024 D1 documented when a scenario dimension was nearly +added to `dimensions.json`. + +**Alternatives considered**: +- *Inside `conformance/registry/`.* Rejected: either the loader rejects the unknown collection + (noisy but survivable) or someone wires it in and unreviewed findings reach `certify`. +- *Git-ignored under `target/`.* Rejected: FR-034 needs cross-campaign persistence and FR-030's + deduplication needs to see prior campaigns' signatures. A queue that evaporates re-reports + every known finding on every nightly run. + +--- + +## Decision 7 — The pipeline proof reuses the sealed `EvidenceSource` injection boundary + +**Decision**: The FR-042a injected-difference proof injects through +`parity_harness::inject::perturb_source`, the existing sealed-trait entry point. + +**Rationale**: `inject.rs` already establishes the property this proof needs and cannot easily +re-establish: the entry points are generic over a **sealed** `EvidenceSource` trait that no +observer output can implement, so injecting into an observer's *return* value does not compile. +A proof that could inject downstream of the comparison would demonstrate nothing about whether +the pipeline works — it would be asserting on data it planted past the part under test. Reusing +the boundary inherits that guarantee instead of re-arguing it. + +`InjectionInapplicable` is inherited too, and matters as much: a perturbation that never landed +must fail loudly rather than be counted as "the pipeline found nothing", which is the exact +distinction FR-042a draws. + +**What is new**: the assertion target. `coverage-regressions` asserts a *channel verdict* flips +clean → failing. This proof asserts a *pipeline traversal*: the difference surfaces, minimizes to +a stable signature, produces a complete candidate, classifies, and is promotable. That needs its +own verdict type; the injection primitive underneath is shared unchanged. + +--- + +## Decision 8 — The corpus manifest becomes Rust-owned strict JSON; fetching stays in the network lane + +**Decision**: Move the 33 pinned entries from `fetch_realworld_corpus.py`'s `ENTRIES` tuple into +a strict-JSON manifest the Rust side loads and validates. Fetching remains network-lane-only. + +**Rationale**: FR-050 (reject any non-immutable reference) must be checkable **hermetically** — +it is a property of the manifest, not of a fetch, and a validation that only runs when the +network is up is a validation that does not run on most PRs. That requires the manifest to be +loadable by hermetic Rust code. The entries are already commit-pinned and already inventoried as +`realworld::*` baseline units under `res-realworld-corpus-not-vendored`, so this is a +representation change, not a new coverage claim. + +**What is genuinely new**: the per-entry content digest (FR-049/FR-051). The current manifest has +no digest, so first materialization records one and every later run verifies it. This closes a +real hole — GitHub's contents API at a pinned SHA is expected to be stable, but "expected" is not +"verified", and an unverified fetch means comparing against content nobody checked. + +**Deliberately left to `/speckit.tasks`**: whether the Python fetcher is retired once the Rust +fetch lands, or kept as an exploratory aid. Both are defensible; it does not affect the design. + +--- + +## Decision 9 — One new nextest profile, with an explicit allow-list + +**Decision**: A `discovery` profile whose `default-filter` is an explicit `binary(=…)` +allow-list, plus exclusion of every discovery binary from the `default-filter` of `default`, +`dev-fast`, `full`, `ci`, `mvp-integration`, and `parity`. + +**Rationale**: This is the 018 lesson taken verbatim. The parity profile's filter is an explicit +allow-list precisely because a `parity_*` glob wrongly captured the hermetic guards +`parity_harness_faults` and `parity_registry_check`. A `discovery_*` glob would make the same +mistake with the hermetic discovery guards, and the symptom — a hermetic guard silently not +running in the fast lane — is invisible until it matters. + +Exclusion from `parity` as well as the PR profiles is deliberate: discovery and live parity +certification answer different questions and have different budgets, and a discovery campaign +inside the parity lane would push it past its window. + +**Enforcement**: extend `parity_registry_check` to cover the discovery lane — registry ↔ +`tests/*.rs` ↔ `.config/nextest.toml` agreement, and the assertion that `deacon --help` gains +nothing. That check already exists and already fails on drift; FR-057 is asking for exactly its +shape, so extending it is strictly better than adding a parallel checker. + +--- + +## Decision 10 — Two scheduled cadences; the container tier is invoked-only + +**Decision**: The 30-minute **nightly** campaign spends its entire budget on the +configuration-resolution tier. The corpus canary runs **weekly** in the network-backed lane. The +container-backed tier is explicitly-invoked only. Admission cap: 25 newly distinct signatures per +campaign. + +**On the corpus cadence** (corrected during `/speckit.analyze`): an earlier draft made the corpus +invoked-only alongside the container tier. That contradicted its stated purpose — US7 calls it an +*ecological canary*, and a canary that runs only when someone remembers to invoke it cannot warn +anyone. It gets a weekly schedule of its own. Weekly rather than nightly because the corpus +changes only when someone re-pins it: nightly runs would mostly re-confirm the previous night at +network cost, and the signal being watched for (the ecosystem drifting away from what deacon +handles) moves on the order of weeks, not hours. + +**Rationale**: This resolves the item `/speckit.clarify` deferred to planning. At the clarified +per-candidate ceilings — 60s hermetic, 5 minutes container-backed — a single container-backed +candidate can consume a sixth of the scheduled window, so six of them would consume it entirely. +Sharing one budget between tiers lets the slow tier starve the fast one, and the fast tier is +where nearly all the exploration happens (in practice `read-configuration` against the oracle +returns in about a second, so a 30-minute hermetic campaign reaches thousands of candidates +rather than the ~30 the worst-case ceiling implies). + +The cap of 25 is set from reviewer throughput, not from machine capacity: a nightly run that +admits more than a couple of dozen genuinely new signatures has produced a backlog nobody clears +before the next run, and per FR-034b the excess is *reported*, not discarded silently — so a +campaign that keeps hitting the cap is itself a visible signal that something systemic is +diverging. + +**Alternatives considered**: +- *One budget shared across tiers, weighted.* Rejected: the weighting would need tuning against + the machine, which makes campaign volume environment-dependent and undermines the + reproducibility FR-001 asserts. +- *No admission cap, rely on deduplication.* Rejected: deduplication collapses *repeats* of one + defect; it does nothing about one generator change legitimately surfacing hundreds of distinct + signatures at once, which is the case that destroys the queue. + +--- + +## Decision 11 — Metamorphic relations are registry data; findings are not + +**Decision**: The relation catalogue lives at `conformance/registry/metamorphic.json` as +hand-authored `mrl-` records, validated by the existing `validate` command (new classes V31–V32). +The findings queue lives outside the registry (Decision 6) and is validated by a separate +hermetic `discovery check` command with its own D-class violations. + +**Rationale**: The split is principled rather than cosmetic. A metamorphic relation is an +**assertion the project makes** — "reordering these keys must not change the result, and here is +the clause that says so." That is the same kind of object as an applicability rule or a behavior: +hand-authored, reviewed, stable, and referencing `clu-`/`bhv-` ids that only the registry loader +can resolve. FR-045's ground requirement is structurally identical to the `ground` that 024 +already requires on `rule-` records, so it gets the same validation treatment. + +A finding is a **candidate for an assertion** — machine-produced, unreviewed, possibly wrong. It +must not be able to reach `certify`, and Decision 6 makes that structural. + +**Violation classes**: + +| Class | Where | Statement | +|---|---|---| +| **V31** | `validate` | metamorphic relation integrity: unresolvable `ground`, unknown `effect`, duplicate transformation id | +| **V32** | `validate` | a mandated relation family (FR-044) has no relation record | +| **D1** | `discovery check` | malformed queue record | +| **D2** | `discovery check` | a finding with zero or more than one classification | +| **D3** | `discovery check` | a promoted finding naming a case that does not resolve | +| **D4** | `discovery check` | a corpus entry with a non-immutable reference or a missing digest | +| **D5** | `discovery check` | a finding whose pinned input set names a revision not in `revisions.json` | + +D-classes are numbered separately from V-classes on purpose: they are emitted by a different +command over a different data root, and folding them into the V-series would imply the registry +validator can see the queue, which Decision 6 says it must not. + +--- + +## Decision 12 — Metamorphic evaluation needs no oracle, and that is load-bearing + +**Observation, recorded because it shapes sequencing**: FR-048 requires relations to be evaluable +against deacon alone. This makes the metamorphic tier the only part of discovery that runs with +**neither** Node/oracle **nor** Docker **nor** network. + +That does not license running it in the PR lane — FR-055 is absolute and the reason is +stochasticity, not resource cost. But it does mean US6 can be built and exercised before any +oracle provisioning exists, and it means a contributor with no devcontainer CLI installed can +still develop and test that story locally. Sequencing should take advantage of this: the +metamorphic tier is the cheapest complete vertical slice through generation → comparison → +signature → candidate, and building it first exercises the whole hermetic spine before the live +differential is wired up. + +--- + +## Resolved unknowns + +Every `NEEDS CLARIFICATION` from Technical Context is resolved above: + +| Unknown | Resolved by | +|---|---| +| Grammar source for constrained generation | D1 — the committed constraint inventory | +| Deterministic randomness with cross-version stability | D2 — in-repo xoshiro256\*\* | +| Signature computation without a second comparison path | D3 — derive from `ConfigDivergence` | +| Where the code lives | D4 — hermetic/live split across the two existing crates | +| Shrink algorithm | D5 — structural delta-debugging with a declared ordered catalogue | +| Queue location and unreachability from `certify` | D6 — `conformance/discovery/`, verified against `load.rs` | +| How the pipeline proof avoids cheating | D7 — reuse the sealed `EvidenceSource` boundary | +| Corpus manifest ownership and digest verification | D8 — Rust-owned strict JSON, fetch stays network-lane | +| Lane wiring and its enforcement | D9 — explicit allow-list profile, enforced by `parity_registry_check` | +| Budget apportionment between tiers (deferred by clarify) | D10 — scheduled campaign is hermetic-tier only; cap 25 | +| Where relations vs findings are validated | D11 — V31/V32 in `validate`; D1–D5 in `discovery check` | + +--- + +## Measured thresholds (T114) — SC-002 and SC-004 + +A threshold nobody measured is a guess. These are the **observed** values from three real +`config-differential` campaigns against the verified pinned oracle (`@devcontainers/cli` +0.87.0) on `prof-linux-amd64-docker-0870`, run 2026-07-27. Each campaign started from an +empty queue so its findings were minimized on admission rather than deferring to a record +an earlier campaign had already made. + +| Campaign | Seed | Candidates | Parse-stage failures | **SC-002** (≤10%) | Minimized samples | **SC-004** median (≥0.80) | +|---|---|---|---|---|---|---| +| 1 | `0xa1b2c3d4` | 200 | 0 | **0.00%** ✅ | 2 | 0.6219 ⚠️ (n=2) | +| 2 | `0x5eed0002` | 40 | 1 | **2.50%** ✅ | 4 | **0.8516** ✅ | +| 3 | `0x0badc0de` | 25 | 0 | **0.00%** ✅ | 7 | **0.8889** ✅ | + +**SC-002 holds with a wide margin.** The worst observed document-syntax failure rate is +**2.50%**, against a 10% ceiling — the generator is exploring the tool, not the parser. The +campaign additionally reports `trivialFailureFraction` against a declared +`trivialFailureCeiling` of 0.1 and did not breach it in any run. + +**SC-004 holds on the pooled sample.** Pooling the 13 independent minimizations across all +three campaigns: + +``` +n = 13 median = 0.8438 ✅ mean = 0.7921 min = 0.4000 max = 0.9286 +sorted: 0.400 0.400 0.763 0.824 0.839 0.842 0.844 0.861 0.861 0.889 0.918 0.929 0.929 +``` + +The pooled median of **0.8438** clears the 0.80 floor. Campaign 1's apparent 0.6219 is an +**n=2 artifact**, not a regression — see the measurement note below. + +**Measurement note — why n is much smaller than the finding count.** Only 13 of the 66 +emitted candidates carry a live reduction. A candidate re-emitted for a signature the queue +already carries deliberately does **not** re-minimize: `campaign.rs` routes it through +`Reduction::not_attempted` with the reason *"the findings queue already carries this +signature, and the reduced input recorded when it was admitted still stands"*, and the +already-reduced input lives on the admitting **witness** (`minimalInput` + +`reductionSteps`), not on the re-emitted candidate. That is a deliberate cost gate (FR-022) +and it states its reason rather than claiming minimality — but it means **the campaign +report does not surface an SC-004 aggregate**, so the metric has to be recomputed from +candidate provenance as done here. Campaign 1 generated 200 candidates over the same +signature set, so nearly every emission was a re-emission and only 2 live reductions +survived; campaigns 2 and 3 were deliberately run with smaller `--candidates` counts to +raise the number of first-observations and thus the sample size. + +The two 0.400 outliers are both small inputs (5 → 3 nodes): a document already near-minimal +has little to remove, so a low *fraction* there is not a shrinker deficiency. SC-004 +specifies a **median** precisely to tolerate them. + +**Follow-up worth considering** (not required by SC-004, which the pooled data satisfies): +have the campaign outcome carry a reduction aggregate (count and median over the campaign's +own live minimizations) so the threshold is readable from the report instead of recomputed +from `target/discovery/candidates/*/provenance.json`. diff --git a/specs/025-exploratory-parity-discovery/spec.md b/specs/025-exploratory-parity-discovery/spec.md new file mode 100644 index 00000000..2862263a --- /dev/null +++ b/specs/025-exploratory-parity-discovery/spec.md @@ -0,0 +1,730 @@ +# Feature Specification: Exploratory Parity Discovery + +**Feature Branch**: `025-exploratory-parity-discovery` +**Created**: 2026-07-27 +**Status**: Draft +**Input**: User description: "Create a feature specification for discovering parity behaviors that curated deterministic cases did not anticipate, using constrained generation, shrinking, metamorphic assertions, and pinned real-world workspaces." + +## Why This Feature Exists + +The conformance record now answers three questions well and one question not at all. + +It knows **what it claims** (behaviors with three-axis dispositions), **whether each claim +is backed by evidence** (cases, waivers, gaps), and — since the coverage model — **which +declared scenario combinations remain uncovered**. Every one of those answers is computed +over a denominator that a human wrote down. + +That is the limit. A behavior nobody imagined is not uncovered; it is **absent from the +denominator**, so every report is green about it. The coverage model made the *known* hole +countable. It did nothing about the *unknown* one, and by construction it cannot: enumerating +combinations of dimensions a person chose can only ever redistribute that person's +imagination. + +Two forces make that unknown hole real rather than theoretical: + +- **The input space is adversarial and open.** Configurations arrive from templates, + generators, editors, and hand-editing. They carry unknown keys, nulls, empty collections, + wrong types, conflicting sources, cyclic `extends`, exotic substitution nesting, and + lifecycle shapes in every representation the schema permits. Curated fixtures encode the + shapes a maintainer thought of on the day they wrote them. +- **Expected output is often impractical to state.** For a merged configuration document + over an `extends` chain with substitution, nobody can write the expected bytes by hand. + Today that is handled by comparing against the pinned reference — which works only for + inputs somebody thought to compare. + +This feature adds the complementary discipline: **generate inputs nobody curated, compare +them, and turn each surviving difference into a reviewable candidate.** It is deliberately a +*discovery* mechanism, not a *gate*. Everything it finds enters the existing deterministic +record only through human review, with a stable behavior identity and a disposition — the +same door every other claim uses. Discovery that could silently widen a tolerance or rewrite +a reference snapshot would not add knowledge; it would launder unknowns into green. + +## Clarifications + +### Session 2026-07-27 + +- Q: Where does the findings queue live, and does it participate in certification? → A: In its **own namespace outside the registry** — a sibling of the registry rather than a member of it. The registry loader never reads it and `certify` never takes it as input. Placing it inside the registry would repeat the scenario-dimension mistake of 024: a new record kind added to a loaded collection silently changes the certification denominator, which here would let an unreviewed, stochastically-discovered finding block a release. +- Q: What composes the normalized signature that serves as the deduplication key? → A: **Channel + observable path + difference kind + a normalized value-shape class** (type-changed, present/absent, ordering-changed), with concrete values excluded. Structure alone merges genuinely different defects at the same path; including concrete values makes every generated value its own finding and defeats deduplication entirely. +- Q: What is the seed corpus for mutation in the hermetic generation lane? → A: **Committed conformance fixtures only.** The real-world corpus is a separate canary input consumed exclusively by the network-backed lane and is never a mutation seed — seeding from it would make generation itself network-dependent and non-reproducible from the recorded seed and pinned input set alone. +- Q: What bounds findings-queue growth when a campaign surfaces many differences? → A: A **per-campaign admission cap** on newly admitted distinct signatures, with the suppressed count reported explicitly. An unbounded queue lets one generator defect destroy reviewability; a cap that *fails* the campaign would make discovery gate on its own output. Suppression is always visible — never a silent truncation. +- Q: How is the end-to-end pipeline proven? → A: By a **deliberately injected known difference** that must traverse generation, comparison, minimization, candidate emission, classification, and review-only promotion — the same evidence-source injection discipline that proves observable channels can fail. A real discovered-and-promoted behavior is reported when it occurs but is not an acceptance condition, because it depends on the two implementations actually disagreeing where the generator reaches. + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Find a difference nobody curated (Priority: P1) + +A maintainer wants to know whether deacon and the pinned reference disagree on inputs no one +has written a case for. They start a discovery campaign with a recorded seed against a pinned +input set. The campaign generates valid and near-valid configurations from the pinned schema +surface and grammar rules, applies controlled mutations to known-valid configurations, runs +both implementations over each candidate, and reports every normalized difference it +observes. + +**Why this priority**: This is the entire premise. Without generated inputs reaching past the +parser and a differential comparison over them, none of the other stories has an input. +Delivered alone, it already answers a question the project cannot answer today: *does +anything differ outside what we curated?* + +**Independent Test**: Run a campaign with a fixed seed against the pinned input set on a +machine with the verified reference available, and confirm it produces a finding set, that the +candidates exercise stages beyond document parsing, and that re-running the same seed produces +the same candidates and the same findings. + +**Acceptance Scenarios**: + +1. **Given** a recorded seed and a pinned input set, **When** a campaign is run twice, + **Then** both runs generate the identical ordered sequence of candidate inputs and report + the identical finding set. +2. **Given** a campaign run, **When** its generated candidates are inspected, **Then** the + proportion that fail at the document-syntax stage is below the declared trivial-failure + ceiling, and the remainder reach configuration resolution or a later stage. +3. **Given** a mutation catalogue covering unknown fields, wrong types, nulls, empty values, + conflicting configuration sources, invalid Feature identifiers, `extends` cycles, + substitution edge cases, lifecycle shapes, Compose combinations, and ordering changes, + **When** a campaign completes, **Then** each mutation category was applied at least once + and its application count is reported. +4. **Given** the reference implementation is missing or is not the pinned version, **When** a + differential campaign is started, **Then** it fails loudly naming the cause and reports no + findings, rather than skipping silently or comparing against an unverified reference. +5. **Given** a campaign budget, **When** the budget is exhausted, **Then** the campaign stops, + records how much of the planned space it covered, and reports the findings gathered so far + as a complete, self-describing partial run. + +--- + +### User Story 2 - Minimize the difference and hand over a reviewable candidate (Priority: P1) + +A raw differential failure on a generated configuration is nearly unusable: the input is large +and mostly irrelevant, and the difference may be one field deep in a merged document. The +maintainer needs the system to reduce the input while the difference still reproduces, then +emit a single reviewable candidate containing the minimal fixture, the invocation context, the +raw evidence from both sides, the normalized difference, the reference provenance, and a +suggested mapping to a behavior. + +**Why this priority**: A finding that cannot be reduced and explained will not be triaged, so +discovery volume converts to reviewer fatigue rather than to knowledge. Minimization is what +makes a finding cheap enough to act on and stable enough to deduplicate. + +**Independent Test**: Take a known difference on a large generated input, run minimization, and +confirm the result is smaller, still reproduces the same normalized difference signature, is +minimal with respect to the declared reduction steps, and is packaged as a complete candidate. + +**Acceptance Scenarios**: + +1. **Given** a finding on a large generated input, **When** minimization runs, **Then** the + reduced input still produces the same normalized difference signature as the original. +2. **Given** a minimized input, **When** any single further reduction step from the declared + catalogue is applied, **Then** the difference no longer reproduces — the result is minimal + with respect to that catalogue. +3. **Given** a minimization run, **When** it is repeated from the same finding and seed, + **Then** it yields the identical minimal input. +4. **Given** a minimization budget is exhausted before a minimum is reached, **When** the result + is emitted, **Then** the best reduction found is emitted and explicitly marked as + not-minimal, rather than being silently presented as minimal or discarded. +5. **Given** a completed candidate, **When** it is inspected, **Then** it contains the minimal + fixture, the operations and arguments used, the raw evidence from both sides preserved + separately from the normalized form, the normalized difference, the reference provenance, + and a suggested behavior mapping (an existing behavior identity or an explicit "no existing + behavior matches"). +6. **Given** a reduction step that changes the difference into a *different* signature, **When** + minimization evaluates that step, **Then** it rejects the step for the finding under + reduction and reports the new signature as a separate candidate finding rather than losing + it. + +--- + +### User Story 3 - Keep the deterministic lane hermetic (Priority: P1) + +The pull-request lane's value is that it is fast, hermetic, and truthful. Discovery is the +opposite: it is slow, stochastic across seeds, and — for the real-world corpus — network-backed. +A maintainer must be able to trust that a green pull-request run means what it meant before +this feature existed, and that no discovery activity can reach into it. + +**Why this priority**: This is the highest-severity failure mode of the whole feature. A +discovery campaign leaking into the deterministic lane would make pull-request results +non-reproducible and network-dependent — destroying a property that took three prior features +to establish. It is also the smallest story, so there is no reason to defer it. + +**Independent Test**: Run the deterministic lane with the network unavailable and confirm it +passes, selects no discovery program, and performs no fetch; then confirm the discovery lanes +are reachable only on a schedule or by explicit invocation. + +**Acceptance Scenarios**: + +1. **Given** the hermetic deterministic lane, **When** it runs with no network access, + **Then** it passes and no discovery or corpus-fetch activity is selected. +2. **Given** the lane definitions, **When** they are checked structurally, **Then** every + discovery program is selected by a scheduled or explicitly-invoked lane and by no + pull-request lane, and a mismatch fails that structural check. +3. **Given** a discovery campaign reports findings, **When** any pull-request lane runs, + **Then** its result is unaffected — discovery outcomes never determine a pull-request + verdict. +4. **Given** a discovery campaign fails or times out, **When** the outcome is reported, **Then** + the failure is visible in the discovery lane and does not block a release or a pull request. +5. **Given** a discovery lane, **When** it is invoked explicitly, **Then** it accepts a seed and + a budget and records both in its output. + +--- + +### User Story 4 - Classify and deduplicate what was found (Priority: P2) + +Campaigns produce repeats: the same underlying difference surfaces from many generated inputs, +and the same difference reappears on every scheduled run until it is resolved. The maintainer +needs each finding placed into one of a fixed set of causes, and needs repeats collapsed so the +triage queue reflects distinct problems rather than campaign volume. + +**Why this priority**: Without classification and deduplication the queue grows without bound +and the nightly report becomes noise that nobody reads — at which point the discovery machinery +is worse than absent, because it appears to be watching. + +**Independent Test**: Run two campaigns with different seeds over inputs known to trigger the +same underlying difference, and confirm the queue holds one finding with two witnesses, carrying +one classification. + +**Acceptance Scenarios**: + +1. **Given** a finding, **When** it is triaged, **Then** it carries exactly one classification + from the closed set: deacon regression, reference quirk, specification ambiguity, + unsupported behavior, normalizer defect, or fixture defect. +2. **Given** two findings from different campaigns whose normalized signatures are equal, + **When** they enter the queue, **Then** they are recorded as one finding with two witnesses, + not two findings. +3. **Given** two findings with different normalized signatures that map to the same behavior + identity, **When** they enter the queue, **Then** they remain distinct findings and are + reported as grouped under that behavior. +4. **Given** a finding classified as a normalizer defect or a fixture defect, **When** + promotion is attempted, **Then** it is rejected — those classifications describe a defect in + the discovery or comparison machinery, not a behavior of either implementation, and must be + fixed rather than recorded. +5. **Given** a finding whose difference stops reproducing on a later campaign, **When** the + queue is refreshed, **Then** it is reported as no-longer-reproducing rather than silently + dropped. +6. **Given** an untriaged finding, **When** the queue is reported, **Then** it appears in an + explicit unclassified bucket whose count is visible, so that "not yet looked at" is never + indistinguishable from "nothing found". +7. **Given** a campaign that observes more distinct signatures than its admission cap, **When** + it completes, **Then** it admits at most the cap, reports the suppressed count, and still + succeeds — the cap bounds reviewer load without turning discovery into a gate on its own + output. + +--- + +### User Story 5 - Promote a finding only through review (Priority: P2) + +A finding worth keeping must become an ordinary deterministic case, indistinguishable from a +hand-authored one: linked to a stable behavior identity, dispositioned against the standard and +the reference, covered by an executable case with a committed minimal fixture. A finding worth +tolerating must become an explicit, scoped, expiring waiver. Neither may happen automatically. + +**Why this priority**: This is the guardrail that makes the rest safe to run. Automatic +promotion would let a stochastic process author the record it is supposed to be tested against, +and automatic tolerance would let a difference disappear by being observed. + +**Independent Test**: Attempt to have the discovery machinery write into the deterministic +record, and confirm it cannot; then promote a finding by hand and confirm the result validates +as an ordinary case. + +**Acceptance Scenarios**: + +1. **Given** any discovery run, **When** it completes, **Then** it has written nothing into the + deterministic record — no behavior, no case, no waiver, no allowed difference, and no + reference snapshot — and a structural check enforces that no discovery program has such a + write path. +2. **Given** a reviewed finding, **When** it is promoted, **Then** the change carries a stable + behavior identity (an existing behavior or a newly authored one with all three axes), a + disposition, an executable case, and the minimal fixture, and the full validation of the + record passes on that change. +3. **Given** a promotion that omits a behavior identity or a disposition, **When** validation + runs, **Then** it fails naming what is missing. +4. **Given** a promoted case, **When** it is executed by the ordinary deterministic runner, + **Then** it runs like any other case with no discovery-specific machinery involved. +5. **Given** a promoted finding, **When** the queue is reported, **Then** the finding is marked + promoted and names the case that now carries it, so the same difference is not rediscovered + and re-triaged forever. +6. **Given** a finding a reviewer decides to tolerate, **When** it is recorded, **Then** it + becomes a scoped waiver with a rationale and an expiry that self-invalidates when the + difference stops reproducing — never a blanket allowed difference. +7. **Given** a known difference injected at the evidence source, **When** the pipeline runs, + **Then** it surfaces, minimizes, produces a candidate, is classified, and is promotable; and + **When** an injected difference fails to surface, **Then** the run fails loudly as a pipeline + defect rather than reporting a clean campaign. +8. **Given** a queue holding unreviewed findings, **When** the certification gate runs, **Then** + its result is identical to a run with an empty queue — the queue is not among its inputs. + +--- + +### User Story 6 - Assert what cannot be written down (Priority: P2) + +For many inputs there is no practical way to state the expected output — a merged document +over an `extends` chain with substitution is the standard example. The maintainer needs +assertions of the form "this transformation of the input must (or must not) change the +output", each justified by the specification rather than by intuition. + +**Why this priority**: Metamorphic assertions are what let discovery run where the pinned +reference is unavailable, ambiguous, or itself suspect — and they catch a class of defect the +differential cannot see at all, namely deacon and the reference being *consistently* wrong +together about invariance. + +**Independent Test**: Apply each declared transformation to a corpus of known-valid +configurations and confirm the declared relation holds; then deliberately break one relation +and confirm the failure is reported and names the relation and the transformation. + +**Acceptance Scenarios**: + +1. **Given** a known-valid configuration and a transformation declared *irrelevant*, **When** + the transformation is applied, **Then** the normalized result is unchanged — covering at + minimum insignificant formatting, comments and trailing commas, and key ordering within + unordered maps. +2. **Given** a workspace relocated to a different absolute path, **When** the same operation is + run in both locations, **Then** the normalized results are equal modulo the declared path + tokenization, and any residual difference is reported. +3. **Given** two representations the specification declares equivalent — including the + permitted lifecycle command shapes — **When** each is resolved, **Then** the normalized + results are equal. +4. **Given** a transformation declared *significant* (for example reordering where declaration + order is normative), **When** it is applied, **Then** the result MUST change, and a failure + to change is reported as a finding. +5. **Given** any declared relation, **When** the relation set is validated, **Then** each + relation names its specification ground — a normative clause or a recorded behavior — and a + relation with no ground fails validation. +6. **Given** a metamorphic failure, **When** the candidate is emitted, **Then** it names the + relation, the transformation applied, both inputs, and both normalized outputs. + +--- + +### User Story 7 - Watch the real ecosystem (Priority: P3) + +Generated inputs explore the space the schema permits; real repositories exercise the space +people actually write, which is differently shaped. The maintainer needs a corpus of pinned +real-world workspaces run as an ecological canary, with provenance recorded so that a finding +can always be traced to an exact upstream state. + +**Why this priority**: Highest ecological validity, lowest velocity — the corpus changes only +when someone re-pins it, so it finds fewer new things over time than generation does. It is +also the only story that requires network access, which is why it is last and isolated. + +**Independent Test**: Run the corpus canary in the network-backed lane and confirm every entry +resolves to an immutable commit, that provenance is recorded for each, and that an entry naming +a mutable reference is rejected. + +**Acceptance Scenarios**: + +1. **Given** the corpus manifest, **When** it is validated, **Then** every entry names a + repository and an immutable commit identifier, and an entry naming a branch, a tag, or a + floating reference is rejected fail-loud. +2. **Given** a corpus run, **When** it completes, **Then** each entry's recorded provenance + identifies the repository, the commit, the path within it, and a content digest of the + materialized workspace. +3. **Given** a fetched entry whose content digest disagrees with the recorded one, **When** the + run proceeds, **Then** it fails loudly for that entry naming the mismatch, rather than + comparing against unexpected content. +4. **Given** a corpus entry that is unreachable, **When** the run completes, **Then** the entry + is reported unreachable and distinguished from an entry that ran and produced no finding. +5. **Given** a corpus finding, **When** it is emitted, **Then** it enters the same + minimization, classification, deduplication, and review-only promotion pipeline as a + generated finding, and its candidate names its upstream provenance. +6. **Given** the generation lane, **When** its inputs are inspected, **Then** no corpus entry + appears among its mutation seeds — the corpus is a comparison input in the network-backed + lane only, so generation stays reproducible without network access. + +--- + +### Edge Cases + +- **A generated input is invalid in a way the harness cannot express.** The candidate is a + fixture defect, not a behavior difference; it is classified as such, is not promotable, and + drives a generator fix. +- **Both implementations reject the input, with different messages.** Whether this is a + difference at all depends on the declared comparison scope. Message text is not compared; + outcome and diagnostic classification are. A difference in outcome is a finding; a difference + only in wording is not. +- **The difference is caused by the comparison machinery.** A missing or over-broad + normalization rule can manufacture a difference or hide one. Such a finding is classified as + a normalizer defect and is never promoted as a behavior; resolving it changes the + normalization definition, which invalidates affected recorded evidence. +- **Minimization crosses into a different difference.** Reduction must preserve the signature; + a step that changes it is rejected for the current finding and the new signature is captured + as its own finding. +- **Minimization cannot reduce at all.** The original input is emitted, marked not-minimal, with + the reason. +- **The same normalized signature arises from genuinely different causes.** Signature-based + deduplication would merge them. The merge is reversible: witnesses are retained per finding, + and a reviewer can split a finding, which must not resurrect the merged duplicate. +- **A finding stops reproducing between discovery and review.** It is reported as + no-longer-reproducing with the run that last observed it, and is not silently deleted — the + disappearance is itself information. +- **The pinned reference or schema pin advances.** Every recorded finding is bound to the pin it + was found under. On a pin change, findings are re-evaluated against the new pin rather than + carried forward as still-true. +- **A campaign generates an input that is destructive or unbounded to execute** (for example a + configuration that would start an unpinned image or run an unbounded command). Execution is + bounded and constrained; a candidate that cannot be executed safely within the declared + constraints is discarded and counted, not run. +- **The corpus upstream deletes or force-pushes over a pinned commit.** The entry becomes + unreachable and is reported as such; provenance already recorded is not rewritten. +- **A scheduled campaign finds nothing.** The run reports zero findings *and* the volume it + covered, so "nothing found" is distinguishable from "nothing ran". + +## Requirements *(mandatory)* + +### Functional Requirements + +#### Reproducibility and pinning + +- **FR-001**: Every discovery run MUST record a seed, and re-running with that seed and the + same pinned input set MUST reproduce the identical ordered sequence of generated candidates. +- **FR-002**: A run's **pinned input set** MUST be recorded and MUST include, at minimum: the + schema surface pin, the normative prose pin, the reference implementation version, the + normalization definition version, the generator grammar version, the mutation catalogue + version, and the **generator version** — the identity of the pseudorandom stream and the order + of the reduction catalogue, both of which determine output and neither of which is a grammar or + a mutation. +- **FR-003**: A run MUST fail loudly if any element of its pinned input set is missing, or is + present at a version other than the pinned one. It MUST NOT proceed against an unverified + reference and MUST NOT skip silently. +- **FR-004**: All discovery outputs MUST be byte-stable given the same seed and pinned input + set: no timestamps, absolute paths, or machine-specific values in compared content. +- **FR-005**: A run MUST record its budget and, on exhaustion, MUST report the portion of the + planned space covered rather than presenting a truncated run as complete. + +#### Constrained generation + +- **FR-006**: The generator MUST produce configurations derived from the pinned schema surface + and the grammar rules relevant to configuration resolution, spanning valid and near-valid + inputs. +- **FR-007**: Generation MUST respect semantic constraints sufficiently that candidates reach + meaningful resolution stages. The proportion of a campaign's candidates that fail at the + document-syntax stage MUST stay below a declared ceiling, and that proportion MUST be + reported for every run. +- **FR-008**: The generator MUST support **controlled mutation** of known-valid configurations, + with a declared catalogue covering at minimum: unknown fields, wrong types, nulls, empty + values, conflicting configuration sources, invalid Feature identifiers, `extends` cycles, + substitution edge cases, lifecycle representation shapes, Compose combinations, and ordering + changes. +- **FR-008a**: The seed corpus for mutation MUST be the committed deterministic fixtures only. + The real-world corpus MUST NOT be a mutation seed source, so that generation remains fully + reproducible from the recorded seed and pinned input set without network access. +- **FR-009**: Every mutation operator MUST be individually identifiable in a candidate's + provenance, so a finding names which operators produced it. +- **FR-010**: A run MUST report per-category application counts, and MUST report a category + that was never successfully applied as an explicit generation deficiency rather than + omitting it. +- **FR-011**: Generated candidates MUST be executed under bounded resources with a per-candidate + timeout. A candidate that cannot be executed within the declared safety and resource + constraints MUST be discarded and counted, never executed. +- **FR-012**: Generation MUST NOT emit candidates that reference unpinned image inputs when the + candidate is destined for a container-backed comparison. + +#### Differential comparison + +- **FR-013**: A differential comparison MUST run both deacon and the verified pinned reference + over the same candidate and compare the declared observable channels. +- **FR-014**: Raw evidence from both sides MUST be preserved separately from its normalized + form. +- **FR-015**: Comparison MUST reuse the single existing normalization definition. Discovery MUST + NOT introduce a second, parallel, or discovery-only normalization path. +- **FR-016**: Comparison MUST relate outcomes and structured content, never diagnostic message + wording. +- **FR-017**: A difference already covered by an existing recorded case, waiver, or allowed + difference MUST be reported as already-characterized and MUST NOT enter the triage queue as + new. +- **FR-018**: Discovery MUST NOT consume, extend, or author an entry in the allowed-difference + mechanism to suppress what it finds. That mechanism records reviewed tolerances; a discovery + program writing to it would let a difference disappear by being observed. + +#### Minimization + +- **FR-019**: On a difference, the system MUST attempt to reduce the input while preserving the + finding's **normalized signature**. +- **FR-020**: Minimization MUST be deterministic: the same finding and seed MUST yield the same + reduced input. +- **FR-021**: A reduced input MUST be reported as minimal only when no single step from the + declared reduction catalogue further reduces it while preserving the signature. +- **FR-022**: Minimization MUST be bounded; on budget exhaustion the best reduction found MUST + be emitted and explicitly marked not-minimal with the reason. +- **FR-023**: A reduction step that changes the signature MUST be rejected for the finding under + reduction, and the resulting new signature MUST be captured as a separate candidate finding. + +#### The reviewable candidate + +- **FR-024**: Each candidate MUST contain: the minimal fixture, the operations and arguments + that produced the difference, the raw evidence from both sides, the normalized difference, + the reference provenance, and a suggested behavior mapping. +- **FR-025**: The suggested behavior mapping MUST either name an existing behavior identity or + state explicitly that no existing behavior matches. It MUST NOT invent a behavior identity. +- **FR-026**: A candidate MUST record the full pinned input set under which it was found and the + seed and campaign that found it. +- **FR-027**: A candidate MUST be self-contained: reproducing it MUST require only the candidate + and the pinned input set it names. + +#### Classification and deduplication + +- **FR-028**: Every triaged finding MUST carry exactly one classification from the closed set: + deacon regression, reference quirk, specification ambiguity, unsupported behavior, normalizer + defect, fixture defect. +- **FR-029**: An untriaged finding MUST appear in an explicit unclassified bucket with a visible + count. +- **FR-030**: Findings MUST be deduplicated by normalized signature; equal signatures collapse + to one finding carrying multiple witnesses. +- **FR-030a**: The normalized signature MUST be composed of exactly: the observable channel, the + observable path within that channel, the kind of difference, and a normalized value-shape + class (such as type-changed, present-versus-absent, or ordering-changed). Concrete observed + values MUST NOT contribute to the signature. +- **FR-031**: Distinct signatures MUST remain distinct findings even when they map to the same + behavior identity; they MAY be reported grouped under that behavior. +- **FR-032**: Witnesses MUST be retained per finding so a merge can be reviewed and split, and a + split finding MUST NOT be re-merged by the deduplication rule that originally merged it. +- **FR-033**: A finding whose difference stops reproducing MUST be reported as + no-longer-reproducing, naming the run that last observed it, and MUST NOT be silently dropped. +- **FR-034**: Findings MUST persist across runs so that deduplication and triage state survive a + campaign boundary. +- **FR-034a**: The findings queue MUST live in its own version-controlled namespace, separate + from the deterministic record. The record's loader MUST NOT read it and the certification gate + MUST NOT take it as input, so that no finding — reviewed or not — can block a release. +- **FR-034b**: A campaign MUST admit at most a declared maximum number of newly distinct + signatures to the queue, and MUST report the count of signatures it observed but did not + admit. Exceeding the cap MUST NOT fail the campaign, and suppression MUST NEVER be silent. +- **FR-035**: Normalizer-defect and fixture-defect findings MUST be non-promotable; they + describe a defect in the discovery or comparison machinery and MUST be resolved there. + +#### Review-only promotion + +- **FR-036**: No discovery program MAY write into the deterministic record — behaviors, cases, + waivers, allowed differences, dispositions, or reference snapshots. A structural check MUST + enforce the absence of such a write path. +- **FR-037**: Promotion MUST require a stable behavior identity: an existing behavior, or a newly + authored one carrying all three disposition axes. +- **FR-038**: Promotion MUST require a disposition against both the written standard and the + observed reference; a promotion lacking either MUST fail validation naming what is missing. +- **FR-039**: Promotion MUST commit the minimal fixture and produce a case executable by the + ordinary deterministic runner, with no discovery-specific execution machinery. +- **FR-040**: A promoted case MUST satisfy every existing validation rule that applies to a + hand-authored case, including full scenario-context assignment and the corresponding + coverage-obligation updates in the same change. +- **FR-041**: A finding a reviewer chooses to tolerate MUST become a scoped waiver with a + rationale and an expiry, self-invalidating when the difference stops reproducing. A blanket + or unscoped allowed difference MUST be rejected. +- **FR-042**: A promoted finding MUST be marked promoted in the queue and MUST name the case + that now carries it, so it is not rediscovered and re-triaged. +- **FR-042a**: The pipeline MUST be provable by injecting a known difference at the evidence + source and requiring it to traverse generation, comparison, minimization, candidate emission, + classification, and review-only promotion. An injected difference that fails to surface MUST + fail loudly as a pipeline defect, distinct from a campaign that legitimately found nothing. + +#### Metamorphic assertions + +- **FR-043**: The system MUST support metamorphic relations of two kinds: **invariance** (a + declared-irrelevant transformation MUST NOT change the normalized result) and **sensitivity** + (a declared-significant transformation MUST change it). +- **FR-044**: The declared relation set MUST include at minimum: insignificant formatting; + comments and trailing commas; key ordering within unordered maps; controlled workspace path + relocation; equivalent lifecycle command representations; and `extends` flattening against its + hand-flattened equivalent. +- **FR-045**: Every relation MUST name its specification ground — a normative clause or a + recorded behavior. A relation with no ground MUST fail validation. +- **FR-046**: Path relocation MUST compare modulo the declared path tokenization, and MUST + report any residual difference the tokenization does not account for. +- **FR-047**: A metamorphic failure MUST produce a candidate naming the relation, the + transformation, both inputs, and both normalized outputs. +- **FR-048**: Metamorphic relations MUST be evaluable against deacon alone, so they remain + usable where the reference is unavailable, ambiguous, or itself under suspicion. + +#### Real-world corpus + +- **FR-049**: The real-world corpus manifest MUST record, per entry, a repository, an immutable + commit identifier, the path within the repository, and a content digest of the materialized + workspace. +- **FR-050**: An entry naming a branch, a moving tag, or any floating reference MUST be rejected + fail-loud at validation time. +- **FR-051**: A materialized entry whose content digest disagrees with the recorded digest MUST + fail loudly for that entry. +- **FR-052**: An unreachable entry MUST be reported as unreachable, distinguished from an entry + that ran and produced no finding. +- **FR-053**: Corpus content MUST NOT be vendored into this repository; it is fetched on demand + in a network-backed lane, consistent with the existing decision on third-party content. +- **FR-054**: A corpus finding MUST enter the same minimization, candidate, classification, + deduplication, and review-only promotion pipeline as a generated finding, and its candidate + MUST name its upstream provenance. +- **FR-054a**: The corpus MUST be consumed only as a direct comparison input in the + network-backed lane. It MUST NOT feed the generator as a mutation seed (FR-008a). + +#### Lane isolation + +- **FR-055**: The hermetic deterministic pull-request lane MUST NOT perform network access, fetch + the corpus, run a generation campaign, or select any discovery program. +- **FR-056**: Discovery MUST be reachable only from a scheduled lane or an explicitly invoked + lane, and the explicit invocation MUST accept a seed and a budget and record both. The + real-world corpus canary MUST have a scheduled cadence of its own; leaving it invocation-only + would make it a canary nobody hears. +- **FR-057**: A structural check MUST enforce lane selection: every discovery program is selected + by a discovery lane and by no pull-request lane, and a mismatch MUST fail that check. +- **FR-058**: Discovery outcomes MUST NOT determine any pull-request or release verdict. A + discovery program's exit status MUST reflect whether the campaign ran, never what it found. +- **FR-059**: Discovery MUST NOT be a shipped consumer capability; it is development-only tooling + and MUST NOT appear in the consumer command surface. +- **FR-060**: The container-backed portion of discovery MUST be separately selectable from the + configuration-resolution portion, so a campaign can run where containers are unavailable. + +#### Reporting + +- **FR-061**: Every run MUST produce a report stating the seed, the pinned input set, the budget + and how much of it was used, candidate volume, per-mutation-category counts, trivial-failure + proportion, findings by classification, the unclassified count, and the count of distinct + signatures observed but not admitted under the per-campaign cap. +- **FR-062**: A zero-finding run MUST report the volume it covered, so "nothing found" is + distinguishable from "nothing ran". +- **FR-063**: Reports MUST be deterministic and free of timestamps and absolute paths, and MUST + be written to a location that is not part of the version-controlled record. + +### Key Entities + +- **Campaign** — one discovery run. Holds the seed, the pinned input set, the lane, the budget, + the generation profile, and the resulting findings. +- **Pinned input set** — the complete set of versioned inputs that determine a campaign's + output: schema pin, prose pin, reference version, normalization version, grammar version, + mutation catalogue version. +- **Generation profile** — which grammar rules, mutation operators, and target operations a + campaign draws on; the knob that trades breadth against depth within a budget. +- **Mutation operator** — one named, individually attributable transformation of a known-valid + configuration, belonging to one declared category. +- **Candidate input** — a generated or mutated configuration together with the operations and + arguments to run over it, plus its generation provenance. +- **Finding** — one distinct observed difference. Carries a normalized signature, one or more + witnesses, raw and normalized evidence, a classification, a triage state, and the pins under + which it was observed. +- **Normalized signature** — the stable identity of a difference: the observable channel, the + observable path within it, the kind of difference, and a normalized value-shape class, with + concrete observed values excluded. It is the deduplication key and the invariant minimization + must preserve. +- **Witness** — one concrete input that exhibits a finding, with the campaign and seed that + produced it. +- **Reviewable candidate** — the reviewer-facing package for a finding: minimal fixture, + context, raw evidence, normalized difference, reference provenance, suggested behavior + mapping. +- **Metamorphic relation** — a named transformation plus its declared effect (invariance or + sensitivity) plus its specification ground. +- **Corpus entry** — one pinned real-world workspace: repository, immutable commit, path, + content digest. +- **Findings queue** — the persistent, reviewable collection of findings across campaigns, + carrying triage and promotion state. It lives in its own version-controlled namespace beside + the deterministic record, is never loaded by that record, and is never an input to the + certification gate. +- **Promotion** — the human-reviewed change that converts a finding into a behavior identity, + a disposition, and either an executable case or a scoped expiring waiver. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: Re-running a recorded seed against its pinned input set reproduces the identical + candidate sequence and the identical finding set, verified over three consecutive runs, with + zero differences. +- **SC-002**: In every campaign report, at most 10% of generated candidates fail at the + document-syntax stage; the remaining 90% or more reach configuration resolution or a later + stage. +- **SC-003**: Every declared mutation category is applied at least once in a full-budget + campaign, and any category with zero successful applications is reported as a named + generation deficiency. +- **SC-004**: Minimization reduces a finding's input by at least 80% (median, measured in + input size) while preserving its signature, and every finding reported as minimal survives an + independent check that no single further reduction step preserves the signature. +- **SC-005**: 100% of emitted candidates contain all six required parts — minimal fixture, + context, raw evidence, normalized difference, reference provenance, suggested behavior + mapping — and each is independently reproducible from the candidate plus its named pins. +- **SC-006**: Repeating a campaign with an unchanged seed and unchanged pins adds zero new + findings to the queue. +- **SC-007**: 100% of findings in the queue carry exactly one classification or appear in the + visible unclassified bucket; no finding is in neither state and none is in both. +- **SC-008**: Zero discovery-authored records exist in the deterministic record: a structural + check confirms no discovery program can write a behavior, case, waiver, allowed difference, + disposition, or reference snapshot, and it fails if such a path is introduced. +- **SC-009**: 100% of promoted findings pass the full existing record validation on the + promoting change, including behavior identity, both disposition axes, scenario context, and + coverage-obligation updates. +- **SC-010**: 100% of declared metamorphic relations name a specification ground; a relation + without one fails validation. +- **SC-011**: Deliberately breaking each declared metamorphic relation causes exactly that + relation to fail and be named — zero relations are inert. +- **SC-012**: 100% of real-world corpus entries resolve to an immutable commit; zero entries + resolve a branch, moving tag, or floating reference, enforced fail-loud. +- **SC-013**: The hermetic deterministic pull-request lane completes with zero network requests + and selects zero discovery programs, verified with the network unavailable. +- **SC-014**: No discovery outcome — finding, failure, or timeout — changes any pull-request or + release verdict, verified by a discovery failure alongside a passing pull-request run. +- **SC-015**: The scheduled discovery campaign completes within 30 minutes, with a per-candidate + timeout of 60 seconds for configuration-resolution comparison and 5 minutes for + container-backed comparison. +- **SC-016**: A deliberately injected known difference traverses the full pipeline — generation, + comparison, minimization, candidate emission, classification, and review-only promotion — and + an injected difference that fails to surface fails loudly as a pipeline defect. Real + discovered-and-promoted behaviors are reported when they occur but are not an acceptance + condition, because they depend on the implementations actually disagreeing where the + generator reaches. +- **SC-018**: The findings queue is never read by the deterministic record's loader and never + influences the certification gate, verified by a queue holding unreviewed findings alongside a + certification run whose result is unchanged by them. +- **SC-019**: Every campaign that exceeds its admission cap reports a non-zero suppressed count; + zero campaigns truncate the queue silently. +- **SC-017**: A reviewer can go from a report entry to a locally reproducing minimal case using + only the candidate and the pins it names, in under 10 minutes and with no access to the + machine that ran the campaign. + +## Assumptions + +1. **The comparison surface is tiered.** The high-volume tier compares configuration resolution + and merged-configuration results, which are fast and need no containers; a smaller, + separately selectable tier drives container-backed operations. This keeps a campaign + affordable while still reaching Compose combinations and lifecycle shapes. +2. **The differential oracle is the existing pinned reference**, verified at its exact pinned + version, exactly as the current live comparison does. Discovery introduces no second oracle. +3. **Comparison, normalization, and evidence capture are reused, not reimplemented.** Discovery + is a new *input source* and a new *triage pipeline* on top of the existing comparison + machinery; a parallel normalization path would be a defect, not a feature. +4. **The findings queue is version-controlled and hand-triaged, in its own namespace.** + Deduplication across campaigns and "already triaged" state cannot exist without persistence, + and the review requirement means a human edits it. It sits beside the deterministic record + rather than inside it, so neither the record's loader nor the certification gate can see it. + Machine-generated run reports remain outside version control entirely. +5. **Discovery reports are advisory.** They gate nothing. Only the existing certification gate + blocks a release, and it does so on the deterministic record, which discovery cannot write. +6. **Two scheduled cadences, not one.** The hermetic generation campaign runs **nightly**, + alongside the existing nightly live comparison. The real-world corpus canary runs **weekly** + in the network-backed lane — a canary that runs only when someone remembers to invoke it is + not a canary. The deep campaign and the container-backed tier remain explicitly invoked with + a seed and a larger budget. +7. **Classification is a human judgement**, assisted but not decided by the tooling. The tooling + may suggest a classification; it may not assign one unreviewed. +8. **A pin advance invalidates findings rather than migrating them.** Findings are claims about + a specific pinned pair of implementations; on a pin change they are re-evaluated, not + carried forward. +9. **The existing real-world corpus manifest is the starting point** for the pinned canary; its + entries are already commit-pinned and already recorded as a non-vendored coverage source. It + is a comparison input only, never a generation seed. +10. **"Near-valid" means schema-adjacent**: inputs a reasonable author could produce — a wrong + type, a stray key, an empty collection — not arbitrary byte-level corruption. Random binary + noise is out of scope because it only exercises the document parser. + +## Out of Scope + +- **A general-purpose fuzzer** for the container runtime, the registry client, or any network + protocol. The target is configuration and lifecycle behavior parity. +- **Automatic repair.** Discovery reports differences; it never proposes or applies a code change. +- **Automatic promotion, tolerance, or snapshot refresh**, in any form, under any flag. +- **Vendoring third-party workspace content** into this repository. +- **Comparing diagnostic message wording** between implementations. +- **Feature-authoring surfaces**, which remain permanently outside this project's scope. +- **Making any discovery capability part of the shipped consumer command surface.** + +## Dependencies + +- The pinned schema surface and the pinned normative prose, and their existing provenance + checks, as the grammar source for generation. +- The verified pinned reference implementation, for differential comparison. +- The single existing normalization definition and its version, as the basis of the normalized + signature and of every comparison. +- The existing observable-channel set and observers, as the comparison surface. +- The existing record validation, certification gate, and coverage-obligation model, which every + promotion must satisfy. +- The existing scheduled live-comparison lane, alongside which the discovery lanes run. +- Container tooling for the container-backed tier, and network access for the corpus canary — + both confined to non-pull-request lanes. diff --git a/specs/025-exploratory-parity-discovery/tasks.md b/specs/025-exploratory-parity-discovery/tasks.md new file mode 100644 index 00000000..91d551b0 --- /dev/null +++ b/specs/025-exploratory-parity-discovery/tasks.md @@ -0,0 +1,543 @@ +--- + +description: "Task list for 025-exploratory-parity-discovery" +--- + +# Tasks: Exploratory Parity Discovery + +**Input**: Design documents from `/specs/025-exploratory-parity-discovery/` +**Prerequisites**: plan.md, spec.md, research.md, data-model.md, contracts/ + +**Tests**: **MANDATORY.** The feature specification explicitly requires acceptance tests for nine +areas (seed reproduction, semantic generation, shrinking, metamorphic failures, finding +classification, deduplication, review-only promotion, pinned real-world provenance, and lane +isolation), and constitution Principle VII treats spec-mandated tests as acceptance criteria. +Test tasks precede implementation within each story. + +**Organization**: Grouped by user story so each can be implemented, tested, and landed +independently. Per the repository default, **each story is its own small CI-gated PR** with a +Conventional-Commit title (`feat`/`fix`/`chore` — never `test`/`style`). + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: Can run in parallel (different files, no dependencies on incomplete tasks) +- **[Story]**: Maps to a user story from spec.md (US1–US7) + +## Path Conventions + +Rust workspace. Hermetic logic in `crates/conformance/`, live execution in +`crates/parity-harness/`, test binaries in `crates/deacon/tests/`, version-controlled data under +`conformance/`. See plan.md § Project Structure. + +--- + +## Phase 1: Setup (Shared Infrastructure) + +**Purpose**: Module skeletons, data roots, and lane wiring so every later task has a home. + +- [X] T001 Create hermetic module skeleton `crates/conformance/src/discovery/mod.rs` with empty submodules (`grammar`, `rng`, `generate`, `mutate`, `shrink`, `signature`, `queue`, `metamorphic`, `corpus`, `report`) and register `pub mod discovery;` in `crates/conformance/src/lib.rs` +- [X] T002 [P] Create live module skeleton `crates/parity-harness/src/discovery/mod.rs` with empty submodules (`campaign`, `differential`, `metamorphic_run`, `minimize`, `candidate`, `corpus_fetch`, `pipeline_proof`) and register `pub mod discovery;` in `crates/parity-harness/src/lib.rs` +- [X] T003 [P] Create the discovery data root: `conformance/discovery/findings.json`, `conformance/discovery/campaigns.json`, `conformance/discovery/corpus.json`, each `{"schemaVersion": 1, "records": []}` +- [X] T004 [P] Add `DiscoveryError` variants (malformed record, unresolvable reference, unknown channel, stale pin) to the domain error enum in `crates/conformance/src/lib.rs` +- [X] T005 [P] Add discovery variants (`OracleUnverified`, `CandidateTimeout`, `ShrinkBudgetExhausted`, `CorpusDigestMismatch`, `CorpusUnreachable`) to `HarnessError` in `crates/parity-harness/src/lib.rs` +- [X] T006 Add `[profile.discovery]` to `.config/nextest.toml` with an explicit `default-filter = 'binary(=discovery_campaign) | binary(=discovery_metamorphic)'` allow-list — NOT a `discovery_*` glob (research D9) +- [X] T007 Add `binary(=discovery_campaign)` and `binary(=discovery_metamorphic)` exclusions to the `default-filter` of all six existing profiles in `.config/nextest.toml` (`default`, `dev-fast`, `full`, `ci`, `mvp-integration`, `parity`) +- [X] T008 [P] Add `test-discovery`, `test-discovery-proof`, and `test-discovery-check` targets to `Makefile` per contracts/discovery-cli.md § Make targets +- [X] T009 [P] Add `target/discovery/` to `.gitignore` +- [X] T121 Assign nextest **test groups** to every new test binary in `.config/nextest.toml` across ALL profiles (constitution Principle VII): `discovery_campaign` → `docker-shared` for the container tier, `discovery_metamorphic` → `fs-heavy`, and the hermetic guards ungrouped. Add the override rules to `default`, `dev-fast`, `full`, `ci`, `mvp-integration`, `parity`, and `discovery` — a binary excluded from a `default-filter` still needs its group declared where it *does* run + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**Purpose**: The hermetic spine every story depends on — deterministic randomness, the grammar, +the signature, and the queue as a persistence substrate. + +**⚠️ CRITICAL**: No user story work can begin until this phase is complete. + +- [X] T010 Implement the in-repo deterministic PRNG (SplitMix64 seed expansion → xoshiro256\*\* stream) in `crates/conformance/src/discovery/rng.rs`, with the algorithm identity documented as a `generatorVersion` component (research D2) +- [X] T011 [P] Unit-test the PRNG against published xoshiro256\*\* reference vectors in `crates/conformance/src/discovery/rng.rs` — this is what makes the stream a reviewable pin rather than an accident +- [X] T012 Implement grammar loading in `crates/conformance/src/discovery/grammar.rs`: read `conformance/inventory/constraints.json`, index the 469 non-annotation units by pointer and kind, expose lookup by schema pointer (research D1) +- [X] T013 [P] Unit-test grammar loading in `crates/conformance/src/discovery/grammar.rs` asserting per-kind unit counts (187 `type`, 117 `property-existence`, 41 `array-shape`, 20 `required`, 18 `union-alternative`, 14 `enum`+`const`) so a re-vendored inventory surfaces as a test change +- [X] T014 Implement `Signature`, the value-shape classifier, and the `sig-` id derivation in `crates/conformance/src/discovery/signature.rs`, consuming `parity_harness::normalize::ConfigDivergence` — derive only, never re-diff (research D3) +- [X] T015 [P] Unit-test value-shape classification in `crates/conformance/src/discovery/signature.rs` covering `present-absent`, `type-changed`, `ordering-changed` (array permutation detection), and `value-changed` +- [X] T016 Implement `Finding`, `Witness`, `FindingState`, and `Classification` record models in `crates/conformance/src/discovery/queue.rs` per data-model.md § 3 +- [X] T017 [P] Implement `Campaign`, `PinnedInputSet`, `Budget`, and `CampaignOutcome` record models in `crates/conformance/src/discovery/queue.rs` per data-model.md § 4 +- [X] T018 Implement the strict-JSON loader for `conformance/discovery/` in `crates/conformance/src/discovery/queue.rs`, rejecting unknown fields at load +- [X] T019 Implement the atomic writer (unique temp file + `fs::rename`) for the discovery data root in `crates/conformance/src/discovery/queue.rs` +- [X] T020 Implement signature-keyed upsert (insert a new finding, or append a witness to the existing one) in `crates/conformance/src/discovery/queue.rs` — the derived `fnd-` id makes duplicate findings unrepresentable +- [X] T021 Implement violation classes **D1** (malformed record, empty witnesses, undeclared channel, unresolvable campaign reference) and **D5** (pin absent from `revisions.json`) in `crates/conformance/src/discovery/queue.rs` +- [X] T022 Wire the `discovery` command group (`check`, `report`, `triage`, `split`, `scaffold`) into `crates/conformance/src/bin/conformance.rs` with the exit-status contract from contracts/discovery-cli.md +- [X] T023 [P] Create hermetic guard binary `crates/deacon/tests/discovery_hermetic.rs` asserting the discovery data root loads and validates clean +- [X] T024 [P] Register `discovery_hermetic` in the `default` and `dev-fast` nextest profiles in `.config/nextest.toml` — it MUST run in the fast lane (it is a guard, not a campaign) + +**Checkpoint**: The hermetic spine exists. Stories can now proceed. + +--- + +## Phase 3: User Story 1 - Find a difference nobody curated (Priority: P1) 🎯 MVP + +**Goal**: Generate valid and near-valid configurations from the pinned grammar, mutate known-valid +fixtures, run both implementations, and report every normalized difference. + +**Independent Test**: Run a campaign with a fixed seed on a machine with the verified oracle; +confirm it produces findings, that under 10% of candidates fail at document parsing, and that +re-running the seed reproduces both the candidates and the findings. + +### Tests for User Story 1 + +- [X] T025 [P] [US1] Seed-reproduction acceptance test in `crates/deacon/tests/discovery_campaign.rs`: the same seed and pinned input set produce an identical ordered candidate sequence and an identical finding set across two runs (SC-001) +- [X] T026 [P] [US1] Trivial-failure-ceiling test in `crates/deacon/tests/discovery_campaign.rs`: `parseStageFailures / candidatesGenerated` stays below 10% (SC-002) +- [X] T027 [P] [US1] Mutation-category coverage test in `crates/deacon/tests/discovery_campaign.rs`: all eleven categories applied at least once, all eleven keys present in `mutationApplications` including zeroes (SC-003) +- [X] T028 [P] [US1] Oracle fail-loud test in `crates/deacon/tests/discovery_campaign.rs`: a missing or wrong-version oracle fails naming the cause and reports no findings — never a silent skip +- [X] T029 [P] [US1] Budget-exhaustion test in `crates/deacon/tests/discovery_campaign.rs`: an exhausted budget stops the campaign, sets `budgetExhausted`, and reports `spaceCoveredFraction` + +### Implementation for User Story 1 + +- [X] T030 [US1] Implement constrained generation in `crates/conformance/src/discovery/generate.rs`: draw from the grammar so `required` keys are satisfied for valid candidates and violated deliberately for near-valid ones +- [X] T031 [US1] Implement the eleven-category mutation catalogue in `crates/conformance/src/discovery/mutate.rs` per data-model.md § 5, each application recording its `mop-` name +- [X] T032 [P] [US1] Unit tests per mutation operator in `crates/conformance/src/discovery/mutate.rs`, asserting each produces a schema-adjacent (not byte-corrupted) result +- [X] T033 [US1] Implement the campaign driver in `crates/parity-harness/src/discovery/campaign.rs`: seed, tier, budget, per-candidate timeout, and outcome accumulation +- [X] T034 [US1] Implement the differential comparison in `crates/parity-harness/src/discovery/differential.rs`, reusing `exec`, `oracle`, and `prereq` — no new process-execution path +- [X] T035 [US1] Wire `normalize::diff` output into `signature.rs` from `crates/parity-harness/src/discovery/differential.rs`, reusing the single normalization definition (FR-015 — a second path is a defect, not a feature) +- [X] T036 [US1] Implement already-characterized suppression in `crates/parity-harness/src/discovery/differential.rs`: a difference covered by an existing case, waiver, or tolerated difference reports as characterized and never enters the queue as new (FR-017) +- [X] T037 [US1] Implement unsafe-candidate discard-and-count in `crates/parity-harness/src/discovery/campaign.rs` (FR-011) and the unpinned-image guard for container-bound candidates (FR-012) +- [X] T038 [US1] Implement the `discovery-campaign` bin in `crates/parity-harness/src/bin/discovery-campaign.rs` with `--seed` **required** (never defaulted) and `--tier`/`--budget-seconds`/`--lane` per contracts/discovery-cli.md +- [X] T039 [US1] Implement campaign-outcome reporting in `crates/conformance/src/discovery/report.rs`, emitting all eleven `mutationApplications` keys always — an absent key is indistinguishable from a never-applied category (FR-010) +- [X] T040 [US1] Create the live test binary `crates/deacon/tests/discovery_campaign.rs` and verify it is selected by `[profile.discovery]`'s allow-list and excluded from all six other profiles (the allow-list entry itself lands in T006) +- [X] T122 [US1] Implement the outcome-only comparison guard in `crates/parity-harness/src/discovery/differential.rs` — relate exit status and structured content, never diagnostic message wording — with a test asserting two rejections differing only in wording produce no finding (FR-016) +- [X] T123 [US1] Implement zero-finding volume reporting in `crates/conformance/src/discovery/report.rs` and test in `crates/deacon/tests/discovery_campaign.rs` that a campaign finding nothing still reports `candidatesGenerated`/`candidatesExecuted`, so "nothing found" is distinguishable from "nothing ran" (FR-062) + +**Checkpoint**: US1 answers the question the project cannot answer today — *does anything differ +outside what we curated?* + +--- + +## Phase 4: User Story 2 - Minimize the difference and hand over a reviewable candidate (Priority: P1) + +**Goal**: Reduce each finding's input while preserving its signature, then package a complete, +self-contained reviewable candidate. + +**Independent Test**: Take a known difference on a large generated input, minimize it, and confirm +the result is smaller, reproduces the same signature, is minimal against the declared catalogue, +and is packaged with all six parts. + +### Tests for User Story 2 + +- [X] T041 [P] [US2] Signature-preservation test in `crates/conformance/src/discovery/shrink.rs` using a synthetic predicate: the reduced input yields the same signature as the original +- [X] T042 [P] [US2] Minimality test in `crates/conformance/src/discovery/shrink.rs`: applying any single further catalogue step to a minimal result no longer preserves the signature (FR-021) +- [X] T043 [P] [US2] Determinism test in `crates/conformance/src/discovery/shrink.rs`: the same finding and seed yield the identical minimal input (SC-004) +- [X] T044 [P] [US2] Budget-exhaustion test in `crates/conformance/src/discovery/shrink.rs`: the best reduction is emitted with `isMinimal: false` and a reason — never silently presented as minimal +- [X] T045 [P] [US2] Signature-drift test in `crates/conformance/src/discovery/shrink.rs`: a step that changes the signature is rejected for the finding under reduction and the new signature is captured as a separate candidate finding (FR-023) +- [X] T046 [P] [US2] Candidate-completeness test in `crates/deacon/tests/discovery_campaign.rs`: every emitted candidate contains all six parts and is reproducible from the candidate plus its named pins (SC-005) + - The completeness half is a file check; the *reproducibility* half is not. Each emitted + candidate's `fixture/` tree is copied into a fresh workspace and both implementations + are re-run over it, and the signature the candidate claims must still be observed. A + candidate whose six files were all present but whose fixture no longer reproduced would + pass a file-existence check while being worthless — and that is precisely the failure + mode minimization introduces, because minimization is the step that rewrites the fixture. + +### Implementation for User Story 2 + +- [X] T047 [US2] Implement the ordered seven-step structural reduction catalogue in `crates/conformance/src/discovery/shrink.rs` per data-model.md § 6, taking the reproduction predicate as a **parameter** so the strategy stays hermetic (research D4/D5) + - The predicate is the `ReproductionProbe` trait, with **three** answers rather than a + boolean: FR-023 needs the third, because a `bool` would discard the drifted signature at + the moment it was observed. + - Termination is a property, not a hope: every accepted step strictly decreases a + lexicographic `(mutations remaining, node count, non-minimal scalars)` triple. Each + component exists because exactly one step reduces it — and the first must lead, because + `un-apply-mutation` can *grow* the document, so a size-only objective would let the + shrinker oscillate between emptying and un-applying forever. + - `un-apply-mutation` is an **exact** reversal, not a category-directed guess: `mutate.rs` + now records a `Reversal` per application, derived by diffing the operator's pre- and + post-images at the document root. It is a *local* edit rather than the pre-image + document, so un-applying never resurrects what an earlier reduction dropped. + - **No async runtime was added to `deacon-conformance`, deliberately.** `reduce` is async + (the live predicate runs two CLIs), and the first attempt took `tokio` as a + dev-dependency to drive it — which `discovery_hermetic`'s SC-013 guard correctly + rejected: its claim is that the network capability is *absent*, not merely unused, and a + dev-dependency still puts a runtime in this crate's test binary. The tests drive the + future with a five-line `block_on` over `Waker::noop()` instead; the synthetic predicate + never suspends, so there is nothing to schedule. +- [X] T048 [US2] Implement the live reproduction predicate in `crates/parity-harness/src/discovery/minimize.rs`, supplying it to `shrink.rs` + - One probe is one call to `differential::compare` — same execution, same single + normalization, same tolerance index, same signature derivation. Two things are held + constant across every probe: the workspace *shape* (a probe tree that differed from the + candidate tree would be measuring the scaffold) and `deliberately_invalid` (it is a fact + about the candidate's provenance, and recomputing it as mutations are un-applied would + move the tolerance boundary mid-reduction). +- [X] T049 [US2] Implement candidate assembly in `crates/parity-harness/src/discovery/candidate.rs`, writing `target/discovery/candidates//` per data-model.md § 9 +- [X] T050 [US2] Write `raw.json` and `normalized.json` as **separate** files in `crates/parity-harness/src/discovery/candidate.rs` — raw and normalized evidence must never be conflated (FR-014) +- [X] T051 [US2] Implement the suggested behavior mapping in `crates/parity-harness/src/discovery/candidate.rs`, emitting either a resolvable `bhv-` id or an explicit `{"match": "none"}` — never an invented id (FR-025) + - The rule is deliberately narrow: a behavior is suggested only when a committed case + already declares a document-shaped assertion (`jsonSubset`/`jsonEquals`) on the **same + channel at the same observable path**. Whole-stream assertions (`equals`/`contains`/ + `matches`/`nonZero`) name no path and contribute nothing — contributing the empty path + would match every signature on their channel. + - FR-025 is held **structurally**: every proposal is filtered against `registry.behaviors` + before it can be written, so an id that does not resolve cannot escape the function. + Ambiguity (several behaviors at one path) is reported as `match: none` with the + candidates listed, because choosing one would be a coin flip wearing a suggestion's + clothes. +- [X] T052 [US2] Wire minimization into the campaign driver in `crates/parity-harness/src/discovery/campaign.rs` with a per-finding shrink budget + - Added to the existing differential loop; the tier dispatch and `AdmissionQueue` are + unchanged apart from two read-only queries (`is_new`, `holds`). + - Two gates on paying for a reduction, both about not spending two CLI invocations to + learn something already recorded: the signature must be new to this campaign, and the + wall clock must not have run out. **Neither may present the result as minimal** — both + route through `Reduction::not_attempted`, which carries `isMinimal: false` *and* the + reason (FR-022). + - Drifted signatures found during reduction are offered to the same queue as separate + findings (FR-023), and a reviewable candidate is emitted for every finding the queue + actually holds — never for one the admission cap turned away. +- [X] T124 [US2] Test in `crates/deacon/tests/discovery_campaign.rs` that the container-backed tier is selectable independently of the configuration-resolution tier, so a campaign runs where Docker is unavailable (FR-060) + - Asserted as a **pair**, driven through the compiled bin with `DEACON_PARITY_DOCKER` + pointed at a path that cannot exist: `config-differential` still exits `0` and really + compares candidates, and `container-differential` in the *same* environment exits `1` at + its Docker prerequisite. Without the second half the first would be satisfied by a driver + that never probes Docker at all, and "separately selectable" would be vacuous — two tiers + with identical prerequisites are one tier twice. + +**Checkpoint**: Findings are cheap enough to triage and stable enough to deduplicate. + +--- + +## Phase 5: User Story 3 - Keep the deterministic lane hermetic (Priority: P1) + +**Goal**: Guarantee structurally that no discovery activity can reach a pull-request lane. + +**Independent Test**: Run the deterministic lane with the network unavailable; confirm it passes, +selects no discovery program, and performs no fetch. + +### Tests for User Story 3 + +- [X] T053 [P] [US3] No-network test in `crates/deacon/tests/discovery_hermetic.rs`: the hermetic discovery surface completes with zero network requests (SC-013) +- [X] T054 [P] [US3] Profile-selection structural test in `crates/deacon/tests/parity_registry_check.rs`: every discovery binary is selected by `[profile.discovery]` and by no pull-request profile; a mismatch fails (FR-057) +- [X] T055 [P] [US3] Never-gates test in `crates/deacon/tests/discovery_hermetic.rs`: a campaign reporting findings and a campaign reporting none both exit `0` (SC-014) + +### Implementation for User Story 3 + +- [X] T056 [US3] Extend `crates/deacon/tests/parity_registry_check.rs` with discovery-lane wiring assertions: registry ↔ `crates/deacon/tests/*.rs` ↔ `.config/nextest.toml` agreement +- [X] T057 [US3] Extend `crates/deacon/tests/parity_registry_check.rs` asserting `deacon --help` gains no discovery surface (FR-059) +- [X] T058 [US3] Create `.github/workflows/discovery.yml` with a nightly `schedule` lane and a `workflow_dispatch` lane accepting `seed` and `budget` inputs, provisioning the pinned oracle +- [X] T059 [US3] Enforce the exit-status contract in `crates/parity-harness/src/bin/discovery-campaign.rs`: status reflects whether the campaign ran, never what it found (FR-058) + - US1's `main()` already implemented the contract correctly; what was missing was + **evidence**. `crates/deacon/tests/discovery_campaign.rs` now drives the COMPILED bin as + a subprocess and asserts all three statuses: `0` for a `--tier metamorphic --dry-run` + run that reached the whole catalogue (no oracle, no Docker, no network — research D12), + `2` for six malformed invocations, `1` for an unverifiable oracle + (`DEACON_PARITY_DEVCONTAINER` pointed at a path that cannot exist). T055 covers the + library level; only a subprocess can observe `main`'s `Result` → `ExitCode` translation, + which is where the contract is actually enforced. + - `env!("CARGO_BIN_EXE_discovery-campaign")` does not exist here (the bin belongs to + `parity-harness`, the test to `deacon`), so the test reuses `prereq::cargo_bin`, a + generalization of `prereq::deacon_binary`'s build-and-read-the-artifact rule. Guessing a + `target/{debug,release}` path is the 023 T115 defect and is not repeated. +- [X] T060 [US3] Register the discovery binaries in `fixtures/parity-corpus/registry.json` so `parity_registry_check` can enforce their wiring + - Registered: `discovery_campaign`, `discovery_metamorphic` (role `live`), + `discovery_hermetic`, `discovery_cli` (role `guard`). The `parity-harness` **bins** + `discovery-campaign` (T038) and `discovery-proof` (T082) are not test binaries and, + like `parity-report` / `conformance-snapshot` / `equivalence-report`, take no registry + entry and no nextest override. + +**Checkpoint**: A green pull-request run means exactly what it meant before this feature existed. + +--- + +## Phase 6: User Story 4 - Classify and deduplicate what was found (Priority: P2) + +**Goal**: Place every finding in one of six causes and collapse repeats, so the queue reflects +distinct problems rather than campaign volume. + +**Independent Test**: Run two campaigns with different seeds over inputs known to trigger the same +underlying difference; confirm the queue holds one finding with two witnesses and one +classification. + +### Tests for User Story 4 + +- [X] T061 [P] [US4] Exactly-one-classification test in `crates/deacon/tests/discovery_hermetic.rs` (SC-007) — `every_finding_carries_exactly_one_classification_or_is_visibly_unclassified`. The partition has **two** unclassified buckets, not one: `untriaged` and `split` (a split ancestor surrenders its classification to its children, Q10). Both are counted, so "no finding is in neither state" holds with both visible. +- [X] T062 [P] [US4] Signature-merge test in `crates/deacon/tests/discovery_hermetic.rs`: equal signatures from two campaigns collapse to one finding with two witnesses (SC-006) +- [X] T063 [P] [US4] Distinct-signature test in `crates/deacon/tests/discovery_hermetic.rs`: different signatures mapping to the same behavior stay distinct findings, grouped in the report but not merged (FR-031) +- [X] T064 [P] [US4] Non-promotable test in `crates/deacon/tests/discovery_hermetic.rs`: `normalizer-defect` and `fixture-defect` are rejected at promotion (FR-035) +- [X] T065 [P] [US4] No-longer-reproducing test in `crates/deacon/tests/discovery_hermetic.rs`: a finding that stops reproducing is reported with the campaign that last observed it, not deleted (FR-033) +- [X] T066 [P] [US4] Untriaged-bucket test in `crates/deacon/tests/discovery_hermetic.rs`: the count is visible, so "not yet looked at" never reads as "nothing found" (FR-029) +- [X] T067 [P] [US4] Admission-cap test in `crates/deacon/tests/discovery_campaign.rs`: exceeding the cap admits at most the cap, reports a non-zero `signaturesSuppressed`, and still exits `0` (SC-019) — `exceeding_the_admission_cap_suppresses_visibly_and_still_succeeds`, verified live against the pinned oracle. Uses `admission_cap = 1` so the boundary is genuinely reached, and compares against an effectively uncapped run of the same seed: at the default of 25 that comparison run itself suppressed 4 signatures, so the comparison had to be lifted above reviewer throughput to stay sound. + +### Implementation for User Story 4 + +- [X] T068 [US4] Implement the `Classification` closed set and violation class **D2** in `crates/conformance/src/discovery/queue.rs` — the closed set already existed; D2 is new (`check_classifications` + `DiscoveryError::ClassificationArity`). "More than one classification" is unrepresentable rather than unchecked: `Finding` carries `Option` and the strict loader rejects an array, so the arity rule reduces to its only reachable violation — zero where exactly one is required. +- [X] T069 [US4] Implement the finding state machine transitions in `crates/conformance/src/discovery/queue.rs` per data-model.md § 3 — `FindingState::may_transition_to` encodes exactly the diagram's five arrows, plus `Finding::{triage,promote,mark_no_longer_reproducing}` and `TransitionError`. Three plausible-looking arrows are absent on purpose and documented: `untriaged → no-longer-reproducing` (the state requires a classification, so the arrow would manufacture a D2 out of a campaign merely not re-observing something nobody had looked at), `no-longer-reproducing → promoted` (promotion needs current evidence), and `untriaged → split` (a split separates a judgement, so there must be one). +- [X] T070 [US4] Implement split with `splitFrom` lineage in `crates/conformance/src/discovery/queue.rs`, and make the deduplication rule skip split lineages so a reviewer's judgement is never silently reverted (FR-032) — `split_finding` produces one child per witness, each keyed by `Finding::derive_child_id` (`parent ‖ its witness ids`, since a child shares its parent's signature and so cannot use the signature derivation). `upsert_finding` refuses the whole **lineage**, including when the ancestor record is gone and only children remain — without that clause the id lookup misses and a fresh merged record resurrects under the ancestor's id. `check` gained the child-id rule (replacing the blanket exemption) and the Q10 ≥2-children rule. +- [X] T071 [US4] Implement the per-campaign admission cap (default 25, research D10) in `crates/parity-harness/src/discovery/campaign.rs`, always reporting `signaturesSuppressed` — never a silent truncation — the cap and its suppression accounting were already implemented by US1. **One gap fixed**: the cap was measured against `admitted`, which also counts findings the standing queue already carried and this run merely re-witnessed, so a queue grown past the cap would reach it on re-witnesses alone and freeze out every genuinely new signature forever. It now counts only *newly distinct* signatures, per FR-034b's wording. +- [X] T072 [US4] Implement `discovery triage` and `discovery split` in `crates/conformance/src/bin/conformance.rs`, the only writers of `classification` — both now delegate to the queue's state machine rather than re-deciding the lifecycle, and `split` authors the children (previously deferred to this task). Covered end to end by `split_writes_a_lineage_the_checker_accepts_and_the_deduplication_never_re_merges`. +- [X] T073 [US4] Implement `discovery report` in `crates/conformance/src/discovery/report.rs` emitting byte-stable `target/discovery/queue.{json,md}` with the five buckets from quickstart.md § 2 — `untriaged` (counted), `triaged`, `no-longer-reproducing`, `promoted`, `pin-stale` — the buckets existed; this adds the FR-031 grouping view (`BehaviorIndex`/`FindingGroup`, keyed by a *reviewed* behavior where a promoted finding's case resolves, else by observable path) and the queue-level suppression total, which is what tells a reviewer the queue is a **sample** rather than everything. +- [X] T125 [US4] Add a guard test in `crates/deacon/tests/discovery_hermetic.rs` asserting no discovery source file writes to or extends the `allowedDifferences` mechanism — a discovery program authoring a tolerance would let a difference disappear by being observed (FR-018) — `no_discovery_source_writes_to_the_allowed_difference_mechanism`. Scans the hermetic **and** live halves (the live half is the one holding the registry it would have to write to). *Reading* stays permitted — FR-017 requires it — and the guard asserts the read still happens, so a scan that matched nothing cannot pass by checking nothing. + +**Checkpoint**: The nightly report is a triage queue, not noise. + +--- + +## Phase 7: User Story 5 - Promote a finding only through review (Priority: P2) + +**Goal**: Make promotion a human act with a stable behavior identity and a disposition, and make +automatic promotion structurally impossible. + +**Independent Test**: Attempt to have discovery write into the deterministic record and confirm it +cannot; then promote a finding by hand and confirm it validates as an ordinary case. + +### Tests for User Story 5 + +- [X] T074 [P] [US5] No-write-path test in `crates/deacon/tests/discovery_hermetic.rs`: no discovery program can write a behavior, case, waiver, tolerated difference, disposition, or snapshot — modelled on `only_the_refresh_bin_writes_committed_snapshots` (SC-008) +- [X] T075 [P] [US5] Promotion-validates test in `crates/deacon/tests/discovery_hermetic.rs`: a promoted finding's case passes full record validation including `scenarioContext` and obligation updates (SC-009) +- [X] T076 [P] [US5] Missing-identity test in `crates/deacon/tests/discovery_hermetic.rs`: a promotion lacking a behavior identity or a disposition fails validation naming what is missing (FR-038) +- [X] T077 [P] [US5] Certify-isolation test in `crates/deacon/tests/discovery_hermetic.rs`: `certify`'s result with a queue holding unreviewed findings is identical to its result with an empty queue (SC-018) +- [X] T078 [P] [US5] Injected-difference traversal test in `crates/deacon/tests/discovery_hermetic.rs`: an injected difference traverses generation → comparison → minimization → candidate → classification → promotable, and an injection that never lands fails loudly rather than reading as "found nothing" (SC-016) + +### Implementation for User Story 5 + +- [X] T079 [US5] Implement `discovery scaffold` in `crates/conformance/src/bin/conformance.rs` emitting skeleton behavior/case/fixture records to **stdout only**, with `UNREVIEWED` sentinels the loader rejects — generation never writes a hand-authored file +- [X] T080 [US5] Implement violation class **D3** (`promotedTo` must resolve to a real case) in `crates/conformance/src/discovery/queue.rs` +- [X] T081 [US5] Implement the pipeline proof in `crates/parity-harness/src/discovery/pipeline_proof.rs` using `parity_harness::inject::perturb_source`'s sealed `EvidenceSource` boundary, so injecting into an observer's return value does not compile (research D7) +- [X] T082 [US5] Implement the `discovery-proof` bin in `crates/parity-harness/src/bin/discovery-proof.rs`, exiting non-zero on a failed traversal **or** an inapplicable injection +- [X] T083 [US5] Add the no-write-path guard to `crates/deacon/tests/discovery_hermetic.rs` asserting no discovery source file references a registry or snapshot write helper +- [X] T126 [US5] Implement the tolerate path in `crates/conformance/src/bin/conformance.rs`: `discovery scaffold --tolerate` emits a scoped `wvr-` waiver skeleton (rationale + `expires`) plus the scoped `allowedDifferences` entry that references it, to **stdout only**; add a test in `crates/deacon/tests/discovery_hermetic.rs` rejecting a blanket or unscoped scope (FR-041) + +**Checkpoint**: A stochastic process cannot author the record it is tested against. + +--- + +## Phase 8: User Story 6 - Assert what cannot be written down (Priority: P2) + +**Goal**: Metamorphic relations, each grounded in the specification, evaluable against deacon alone. + +**Independent Test**: Apply each declared transformation to known-valid configurations and confirm +the relation holds; deliberately break one and confirm the failure names the relation and the +transformation. + +**Note**: This tier needs no oracle, no Docker, and no network (research D12) — a contributor +without the devcontainer CLI installed can develop and test it locally. + +### Tests for User Story 6 + +- [X] T084 [P] [US6] Invariance tests in `crates/deacon/tests/discovery_metamorphic.rs` for formatting, JSONC comments/trailing commas, and key order within unordered maps (FR-044) +- [X] T085 [P] [US6] Path-relocation test in `crates/deacon/tests/discovery_metamorphic.rs`: results equal modulo the declared tokenization, with any residual reported (FR-046) +- [X] T086 [P] [US6] Lifecycle-equivalence test in `crates/deacon/tests/discovery_metamorphic.rs` across the permitted string/array/object forms +- [X] T087 [P] [US6] Sensitivity test in `crates/deacon/tests/discovery_metamorphic.rs`: permuting a declaration-ordered collection MUST change the result, and a failure to change is reported as a finding (FR-043) +- [X] T088 [P] [US6] Ground-required test in `crates/conformance/src/validate.rs`: a relation with a missing or unresolvable `ground` is **V31** (SC-010) +- [X] T089 [P] [US6] Mandated-family test in `crates/conformance/src/validate.rs`: a family from data-model.md § 7 with no record is **V32** +- [X] T090 [P] [US6] Inert-relation test in `crates/deacon/tests/discovery_metamorphic.rs`: deliberately breaking each relation causes exactly that relation to fail and be named — zero relations are inert (SC-011) + +### Implementation for User Story 6 + +- [X] T091 [US6] Implement the `MetamorphicRelation` model in `crates/conformance/src/discovery/metamorphic.rs` per contracts/metamorphic-catalogue.md +- [X] T092 [US6] Author `conformance/registry/metamorphic.json` with the seven mandated relations, each naming a resolvable `clu-` or `bhv-` ground and a rationale +- [X] T093 [US6] Extend the registry loader in `crates/conformance/src/load.rs` to read `metamorphic.json` +- [X] T094 [US6] Implement violation classes **V31** and **V32** in `crates/conformance/src/validate.rs` +- [X] T095 [US6] Implement deacon-only relation evaluation in `crates/parity-harness/src/discovery/metamorphic_run.rs` +- [X] T096 [US6] Add the `metamorphic` tier to the campaign driver in `crates/parity-harness/src/discovery/campaign.rs`, requiring no external prerequisite + - `campaign::run` dispatches to a dedicated `run_metamorphic` **before** the prerequisite + step, so the tier never consults an oracle it does not use. It evaluates the committed + `registry.metamorphic` catalogue with `Sabotage::None` under the same wall-clock and + per-candidate bounds the differential uses (a relation that exceeds its bound is + discarded **and counted**), and admits each violated relation's residual signatures + through the shared `AdmissionQueue`. + - **The plan IS the catalogue**: `spaceCoveredFraction` is denominated by + `registry.metamorphic.len()`, not `planned_candidates` — there is nothing to generate + for this tier, so honoring the request's figure would report a seven-relation tier as + 3.5% of a plan it never had. `mutationApplications` is all-zero with all eleven keys + present (FR-010) and `parseStageFailures` is 0 (no document-syntax stage: the fixtures + are authored, not drawn). An **empty** catalogue is an error, not a zero-relation run — + V32 already forbids a mandated family with no record, and a run over nothing reports + exactly what a run in which everything held reports. + - **Deliberate scope limit**: `Characterization` (the FR-017 already-characterized + suppression) is NOT wired into this tier, and says so in the code. That index is keyed + to differential-style observable paths from `cases/.json` and the waivers, all of + which describe a deacon-vs-reference difference; a metamorphic residual is a + deacon-vs-deacon difference, so consulting it would either suppress nothing or suppress + by path collision. Left out visibly rather than approximated. + - **Known limitation, reported not silent**: a violated *sensitivity* relation yields no + signature by construction (its failure is the ABSENCE of a difference, and the catalogue + deliberately refuses to key one on the touched site), so it cannot enter the queue. It is + logged at `warn` naming the relation rather than dropped. +- [X] T097 [US6] Create the live test binary `crates/deacon/tests/discovery_metamorphic.rs` and verify it is selected by `[profile.discovery]` and excluded from all six other profiles (the allow-list entry itself lands in T006) +- [X] T127 [US6] Implement metamorphic failure-candidate emission in `crates/parity-harness/src/discovery/metamorphic_run.rs` and test in `crates/deacon/tests/discovery_metamorphic.rs` that the candidate names the relation, the transformation applied, both inputs, and both normalized outputs (FR-047) +- [X] T098 [US6] Add the **V31** and **V32** rows to the violation-class index in `conformance/RULES.md`, keeping `validate.rs`/`RULES.md` lockstep + +**Checkpoint**: The project can now catch deacon and the reference being *consistently* wrong — +which the differential cannot see, because both sides agree. + +--- + +## Phase 9: User Story 7 - Watch the real ecosystem (Priority: P3) + +**Goal**: A pinned real-world workspace corpus as an ecological canary, with provenance recorded +and mutable references rejected. + +**Independent Test**: Run the corpus canary in the network-backed lane; confirm every entry +resolves to an immutable commit, provenance is recorded, and a mutable reference is rejected. + +### Tests for User Story 7 + +- [X] T099 [P] [US7] Immutable-reference test in `crates/deacon/tests/discovery_hermetic.rs`: a branch, tag, `HEAD`, or `latest` is **D4** and rejected hermetically, with no network (SC-012) +- [X] T100 [P] [US7] Provenance test in `crates/deacon/tests/discovery_hermetic.rs`: every entry records repository, commit, path, and content digest +- [X] T101 [P] [US7] Digest-mismatch test in `crates/deacon/tests/discovery_campaign.rs`: a fetched entry whose digest disagrees fails loudly for that entry (FR-051) +- [X] T102 [P] [US7] Unreachable-entry test in `crates/deacon/tests/discovery_campaign.rs`: an unreachable entry is distinguished from one that ran and found nothing (FR-052) +- [X] T103 [P] [US7] Pipeline-parity test in `crates/deacon/tests/discovery_campaign.rs`: a corpus finding enters the same minimization, classification, deduplication, and promotion pipeline, naming its upstream provenance +- [X] T104 [P] [US7] Not-a-seed test in `crates/deacon/tests/discovery_hermetic.rs`: no corpus entry appears among the generator's mutation seeds (FR-008a/FR-054a) + +### Implementation for User Story 7 + +- [X] T105 [US7] Implement the `CorpusEntry` model and violation class **D4** in `crates/conformance/src/discovery/corpus.rs`, requiring a 40-hex commit +- [X] T106 [US7] Port the 33 pinned entries from `fixtures/parity-corpus/fetch_realworld_corpus.py` into `conformance/discovery/corpus.json` with `contentDigest: null` +- [X] T107 [US7] Implement fetch with digest verification in `crates/parity-harness/src/discovery/corpus_fetch.rs`: record the digest on first materialization, verify it thereafter +- [X] T108 [US7] Add the `corpus` tier to the campaign driver in `crates/parity-harness/src/discovery/campaign.rs`, gated to the network-backed lane +- [X] T128 [US7] Add a **weekly** `schedule` trigger for the corpus tier to `.github/workflows/discovery.yml`, separate from the nightly hermetic campaign — an ecological canary that runs only on request cannot warn anyone (FR-056, research D10) +- [X] T109 [US7] Resolve the fate of `fixtures/parity-corpus/fetch_realworld_corpus.py` — retire it now that the manifest is Rust-owned, or keep it as an exploratory aid — and update `fixtures/parity-corpus/README.md` and the `res-realworld-corpus-not-vendored` residual accordingly (research D8 left this open deliberately) + +**Checkpoint**: All seven stories are independently functional. + +--- + +## Phase 10: Polish & Cross-Cutting Concerns + +- [X] T110 [P] Add an "Exploratory Parity Discovery" section to `CLAUDE.md` covering the two data roots, the hermetic/live split, the D-class vs V-class distinction, and the never-gates rule +- [X] T111 [P] Add the discovery-lane row to the gate table in `CLAUDE.md` § "Parity & Conformance", making explicit that discovery is a third lane that gates nothing +- [X] T112 [P] Document the gap-vs-finding distinction in `conformance/RULES.md`: a finding is a *candidate* for an assertion and never blocks; a gap is missing coverage and always blocks +- [X] T113 Run the full quickstart walkthrough in `specs/025-exploratory-parity-discovery/quickstart.md` end to end and correct any drift +- [X] T114 [P] Verify SC-002 and SC-004 thresholds against three real campaigns and record the observed values in `specs/025-exploratory-parity-discovery/research.md` (a threshold nobody measured is a guess) +- [X] T115 [P] Verify SC-017 against a real candidate under `target/discovery/candidates/`: a reviewer on a different machine reproduces it from the candidate plus its named pins alone in under 10 minutes; record the walkthrough in `specs/025-exploratory-parity-discovery/quickstart.md` § Troubleshooting if any step proves unclear +- [X] T116 Confirm the three-covering-case floor holds for every channel the discovery machinery relies on: cross-check `conformance/registry/channels.json` against `conformance/registry/regressions.json` and add any missing `reg-` record (V30) +- [X] T117 Run `cargo fmt --all -- --check` and `cargo clippy --all-targets --all-features -- -D warnings` across the workspace +- [X] T118 Run `make test-nextest` (full gate) and confirm no discovery binary is selected by it +- [X] T119 Run `make test-discovery-check` and `make test-discovery-proof` and confirm both pass +- [X] T120 Confirm `cargo run -p deacon-conformance -- certify` produces an unchanged verdict with a populated findings queue +- [X] T129 Verify SC-015 against the nightly lane: the scheduled campaign completes within 30 minutes, the per-candidate timeout is 60 s hermetic and 5 min container-backed, and a candidate exceeding its timeout is discarded and counted rather than hanging the run + +--- + +## Deferred Work + +**None.** Every research decision resolved to an in-scope task. The one item research D8 left +open — the fate of the Python corpus fetcher — is tracked as **T109** rather than deferred, per +the constitution's requirement that a specification is not complete while deferred tasks remain. + +--- + +## Dependencies & Execution Order + +### Phase Dependencies + +- **Setup (Phase 1)**: no dependencies +- **Foundational (Phase 2)**: depends on Setup — **blocks all user stories** +- **US1, US2, US3 (Phases 3–5, all P1)**: depend on Foundational +- **US4, US5, US6 (Phases 6–8, P2)**: depend on Foundational +- **US7 (Phase 9, P3)**: depends on Foundational +- **Polish (Phase 10)**: depends on all desired stories + +### User Story Dependencies + +| Story | Depends on | Notes | +|---|---|---| +| US1 | Foundational | independent | +| US2 | Foundational | shrink strategy is hermetic and testable with a synthetic predicate, so US2's core lands without US1; T046 and T052 integrate with US1 | +| US3 | Foundational | fully independent — smallest story, highest severity if skipped | +| US4 | Foundational | T067 exercises the cap through a campaign, so it integrates with US1 | +| US5 | Foundational | T078's traversal proof exercises US1+US2+US4; the guards (T074, T083) are independent | +| US6 | Foundational | fully independent, and needs **no oracle, Docker, or network** | +| US7 | Foundational | T101–T103 integrate with US1's pipeline | + +### Within Each User Story + +- Tests are written first and MUST fail before implementation +- Models before services; hermetic logic before its live driver +- Story complete before moving to the next priority + +### Parallel Opportunities + +- Setup: T002–T005, T008, T009 in parallel (T006/T007 both edit `.config/nextest.toml` — serialize) +- Foundational: T011, T013, T015, T017 in parallel; T023/T024 in parallel after T022 +- Every story's test tasks are `[P]` — different assertions in the same or different files, written before implementation +- Across stories: US3 and US6 are fully independent of the others and of each other, so they are the two best candidates for parallel staffing + +--- + +## Parallel Example: User Story 1 + +```bash +# Write all US1 acceptance tests together (they must fail first): +Task: "Seed-reproduction test in crates/deacon/tests/discovery_campaign.rs" +Task: "Trivial-failure-ceiling test in crates/deacon/tests/discovery_campaign.rs" +Task: "Mutation-category coverage test in crates/deacon/tests/discovery_campaign.rs" +Task: "Oracle fail-loud test in crates/deacon/tests/discovery_campaign.rs" +Task: "Budget-exhaustion test in crates/deacon/tests/discovery_campaign.rs" +``` + +## Parallel Example: Foundational + +```bash +# The four hermetic unit-test tasks touch different modules: +Task: "PRNG reference-vector tests in crates/conformance/src/discovery/rng.rs" +Task: "Grammar per-kind count tests in crates/conformance/src/discovery/grammar.rs" +Task: "Value-shape classification tests in crates/conformance/src/discovery/signature.rs" +Task: "Campaign record model in crates/conformance/src/discovery/queue.rs" +``` + +--- + +## Implementation Strategy + +### MVP First (User Story 1) + +1. Phase 1: Setup +2. Phase 2: Foundational — **critical, blocks everything** +3. Phase 3: User Story 1 +4. **STOP and VALIDATE**: run a seeded campaign; confirm reproducibility and the parse-failure ceiling +5. US1 alone answers the question the project cannot answer today + +### The no-oracle path + +A contributor without the pinned devcontainer CLI can still deliver a complete vertical slice: +**Phase 1 → Phase 2 → Phase 8 (US6)**. The metamorphic tier exercises generation → comparison → +signature → candidate with no oracle, no Docker, and no network (research D12), so it proves the +entire hermetic spine before any live provisioning exists. If oracle provisioning is a bottleneck, +build this first and treat US1 as the second increment. + +### Recommended landing order + +`Setup + Foundational` → **US3** (smallest, protects the PR lane before anything stochastic +exists) → **US1** (MVP) → **US2** → **US4** → **US5** → **US6** → **US7**. + +US3 is deliberately pulled ahead of US1 despite both being P1: it costs little, and landing the +lane guards *before* the first campaign binary exists means there is never a window in which a +discovery program could be selected by a pull-request lane. + +### Parallel team strategy + +After Foundational, three tracks run cleanly without contention: + +- **Track A**: US1 → US2 (the differential spine) +- **Track B**: US3 → US5 (lane isolation and the promotion guards) +- **Track C**: US6 → US7 (metamorphic, then corpus) + +Track C needs no oracle until US7. + +--- + +## Notes + +- `[P]` = different files, no dependencies on incomplete tasks +- **`[P]` on test tasks sharing one binary** (T025–T029, T061–T066, T074–T078, T099/T100/T104) + means independent test **functions** with no shared fixture or state: they can be authored + concurrently, but they land in one file, so expect a merge conflict and resolve it as the + **union** of the functions — the same convention already used for `.config/nextest.toml` + `binary(=…)` clauses +- Task IDs T121–T129 were added by `/speckit.analyze` remediation; they sit in their correct + phase, and phase placement (not numeric order) determines execution order +- Each story is its own CI-gated PR with a Conventional-Commit title (`feat`/`fix`/`chore` — + `test` and `style` fail the PR-title check) +- T006 and T007 both edit `.config/nextest.toml`; expect a conflict if two stories add binaries + concurrently, and resolve to the **union** of the `binary(=…)` clauses +- Adding any new observable channel requires a `reg-` regression record (V30) and the + three-covering-case floor — T116 checks this +- Never make `certify` non-blocking or delete a real gap to go green; discovery exists to *find* + differences, not to make them disappear