diff --git a/.github/workflows/e001-canonical-evidence.yml b/.github/workflows/e001-canonical-evidence.yml new file mode 100644 index 0000000..65dc666 --- /dev/null +++ b/.github/workflows/e001-canonical-evidence.yml @@ -0,0 +1,59 @@ +name: E001 canonical evidence + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - .github/workflows/e001-canonical-evidence.yml + - experiments/E001/config/** + - src/superloop_e001/** + - tools/generate_e001_evidence.py + +permissions: + contents: read + +jobs: + evidence: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Generate and repeat the canonical matrix + env: + PYTHONPATH: src + run: >- + python tools/generate_e001_evidence.py + --output build/e001-canonical + --source-commit "$GITHUB_SHA" + + - name: Package deterministic evidence archive + id: package + shell: bash + run: | + archive="E001__CANONICAL_EVIDENCE__${GITHUB_SHA}.tar.gz" + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -C build/e001-canonical -cf - . | gzip -n > "$archive" + sha256sum "$archive" > "$archive.sha256" + sha256sum --check "$archive.sha256" + echo "archive=$archive" >> "$GITHUB_OUTPUT" + + - name: Upload canonical evidence + uses: actions/upload-artifact@v4 + with: + name: e001-canonical-${{ github.sha }} + path: | + ${{ steps.package.outputs.archive }} + ${{ steps.package.outputs.archive }}.sha256 + if-no-files-found: error + retention-days: 90 diff --git a/tests/test_e001_simulator.py b/tests/test_e001_simulator.py index 49fb41c..24fd578 100644 --- a/tests/test_e001_simulator.py +++ b/tests/test_e001_simulator.py @@ -9,6 +9,7 @@ from superloop_e001.model import TRACE_FIELDS from superloop_e001.simulator import matrix_manifest, run_matrix, run_simulation from superloop_e001.workloads import CONFIGURATIONS, SCENARIOS, SEEDS, build_offers +from tools.generate_e001_evidence import generate class E001SimulatorTests(unittest.TestCase): @@ -141,6 +142,22 @@ def test_central_trace_accounts_for_work_blocked_by_failed_neighbor(self) -> Non ] self.assertTrue(blocked_transfer_waits) + def test_evidence_generator_repeats_and_records_provenance(self) -> None: + from tempfile import TemporaryDirectory + + with TemporaryDirectory() as directory: + output = Path(directory) / "evidence" + provenance = generate(output, "test-source-commit") + self.assertEqual(72, provenance["validation"]["run_count"]) + self.assertEqual([], provenance["validation"]["determinism_mismatches"]) + self.assertEqual([], provenance["validation"]["invalid_run_ids"]) + self.assertEqual( + {}, + provenance["validation"]["workload_equivalence_mismatches"], + ) + self.assertEqual(72, len(list((output / "raw").glob("*.jsonl")))) + self.assertEqual(72, len(list((output / "summary").glob("*.json")))) + if __name__ == "__main__": unittest.main() diff --git a/tools/generate_e001_evidence.py b/tools/generate_e001_evidence.py new file mode 100644 index 0000000..8e0a429 --- /dev/null +++ b/tools/generate_e001_evidence.py @@ -0,0 +1,155 @@ +"""Generate a reproducible E001 evidence set from the frozen Stage A matrix.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import platform +from pathlib import Path +from typing import Any + +from superloop_e001.model import canonical_json, digest +from superloop_e001.simulator import RunResult, matrix_manifest, run_matrix + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +CONFIGURATION_PATH = ( + REPOSITORY_ROOT + / "experiments" + / "E001" + / "config" + / "CANONICAL_MATRIX__E001__v0.1__2026-07-13.json" +) +SPECIFICATION_PATH = ( + REPOSITORY_ROOT + / "experiments" + / "E001" + / "EXPERIMENT__E001__THREE_RING_BOUNDED_FLOW__v0.1__2026-07-13.md" +) + + +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _write_json(path: Path, value: dict[str, Any]) -> None: + path.write_text( + json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def _determinism_mismatches( + first: list[RunResult], + second: list[RunResult], +) -> list[str]: + mismatches: list[str] = [] + second_by_run = {result.summary["run_id"]: result for result in second} + for result in first: + run_id = result.summary["run_id"] + repeated = second_by_run.get(run_id) + if repeated is None: + mismatches.append(f"{run_id}:missing_repeat") + continue + if result.trace_jsonl != repeated.trace_jsonl: + mismatches.append(f"{run_id}:trace") + if result.summary != repeated.summary: + mismatches.append(f"{run_id}:summary") + unexpected = sorted(set(second_by_run) - {result.summary["run_id"] for result in first}) + mismatches.extend(f"{run_id}:unexpected_repeat" for run_id in unexpected) + return mismatches + + +def generate(output: Path, source_commit: str) -> dict[str, Any]: + """Run, repeat, validate, and write the frozen E001 matrix.""" + + if output.exists() and any(output.iterdir()): + raise ValueError(f"output directory is not empty: {output}") + raw_dir = output / "raw" + summary_dir = output / "summary" + inputs_dir = output / "inputs" + raw_dir.mkdir(parents=True, exist_ok=True) + summary_dir.mkdir(parents=True, exist_ok=True) + inputs_dir.mkdir(parents=True, exist_ok=True) + + first = run_matrix() + second = run_matrix() + mismatches = _determinism_mismatches(first, second) + manifest = matrix_manifest(first) + manifest["determinism_mismatches"] = mismatches + manifest["source_commit"] = source_commit + + for result in first: + run_id = result.summary["run_id"] + (raw_dir / f"{run_id}.jsonl").write_text( + result.trace_jsonl, + encoding="utf-8", + ) + _write_json(summary_dir / f"{run_id}.json", result.summary) + + configuration_bytes = CONFIGURATION_PATH.read_bytes() + specification_bytes = SPECIFICATION_PATH.read_bytes() + (inputs_dir / CONFIGURATION_PATH.name).write_bytes(configuration_bytes) + _write_json(output / "matrix_manifest.json", manifest) + + provenance = { + "schema_version": "e001.evidence-provenance.v1", + "experiment": "SW.EXPERIMENT.E001", + "source_commit": source_commit, + "specification": { + "path": str(SPECIFICATION_PATH.relative_to(REPOSITORY_ROOT)), + "sha256": _sha256_bytes(specification_bytes), + }, + "configuration": { + "path": str(CONFIGURATION_PATH.relative_to(REPOSITORY_ROOT)), + "sha256": _sha256_bytes(configuration_bytes), + }, + "invocation": ( + "PYTHONPATH=src python tools/generate_e001_evidence.py " + f"--output build/e001-canonical --source-commit {source_commit}" + ), + "runtime": { + "implementation": platform.python_implementation(), + "python_version": platform.python_version(), + }, + "validation": { + "determinism_mismatches": mismatches, + "invalid_run_ids": manifest["invalid_run_ids"], + "run_count": manifest["run_count"], + "workload_equivalence_mismatches": manifest[ + "workload_equivalence_mismatches" + ], + }, + "matrix_manifest_digest": digest(manifest), + } + _write_json(output / "provenance.json", provenance) + + if mismatches: + raise RuntimeError(f"determinism mismatches: {mismatches}") + if manifest["invalid_run_ids"]: + raise RuntimeError(f"invalid runs: {manifest['invalid_run_ids']}") + if manifest["workload_equivalence_mismatches"]: + raise RuntimeError( + "workload equivalence mismatches: " + f"{manifest['workload_equivalence_mismatches']}" + ) + return provenance + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--source-commit", required=True) + return parser + + +def main() -> int: + args = _parser().parse_args() + provenance = generate(args.output, args.source_commit) + print(canonical_json(provenance)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())