From 24351f706290fa611192ac9782e3a4594d5549f1 Mon Sep 17 00:00:00 2001 From: Christian-Manuel Butzke Date: Wed, 1 Jul 2026 17:43:35 +0900 Subject: [PATCH] tests: separate the implementation's unit tests from the conformance suite Give harel-python its own hermetic, offline unit suite in tests/ (default pytest touches no network), and move the language-agnostic conformance harness into a dedicated conformance/ dir with its own fetch conftest. Split CI into two jobs: 'test' (ruff + mypy + unit tests; the required check) and 'conformance' (downloads harel-conformance and runs it black-box; non-blocking). Add a Makefile (make test / make conformance) and document the split in README + CONTRIBUTING. Closes #32. --- .github/workflows/test.yml | 20 ++++++++++++++++++- CONTRIBUTING.md | 22 ++++++++++++++++----- Makefile | 23 ++++++++++++++++++++++ README.md | 12 +++++++++-- conformance/__init__.py | 0 {tests => conformance}/conftest.py | 0 {tests => conformance}/harness.py | 0 {tests => conformance}/test_conformance.py | 0 8 files changed, 69 insertions(+), 8 deletions(-) create mode 100644 Makefile create mode 100644 conformance/__init__.py rename {tests => conformance}/conftest.py (100%) rename {tests => conformance}/harness.py (100%) rename {tests => conformance}/test_conformance.py (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e736541..36def61 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,6 +7,8 @@ on: branches: [main] jobs: + # Unit tests: the implementation's own suite — hermetic, offline, fast. + # This is the blocking gate (`test (ubuntu-24.04)`). test: runs-on: ${{ matrix.os }} strategy: @@ -25,5 +27,21 @@ jobs: run: ruff check . - name: Mypy run: mypy src/harel - - name: Pytest + - name: Unit tests run: pytest -q + + # Conformance: the language-agnostic suite from fruwehq/harel-conformance, run + # black-box against this implementation. Separate job: it downloads an external + # repo, so a failure here means "diverges from the spec suite", not "our code broke". + conformance: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + cache: pip + - name: Install + run: pip install -e '.[dev]' + - name: Conformance suite + run: pytest conformance -q diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f4009b4..a836b70 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,20 +19,32 @@ Python ≥ 3.11. The package is import-named `harel`, distribution-named `harel- ## The gate -Before pushing, run all three and keep them clean: +Before pushing, run all three and keep them clean (`make check` does exactly this): ```sh ruff check . mypy src -pytest +pytest # unit tests only — hermetic, offline ``` CI runs `test (ubuntu-24.04)` on every PR and is **required** — a PR merges only once it -is green. +is green. This job runs the **unit tests only**. -## The conformance suite +## Unit tests vs. conformance — kept separate -The suite is **not** a submodule. `tests/conftest.py` clones +This implementation has its **own unit tests** (`tests/`), which are hermetic and offline +— `pytest` (or `make test`) runs only these, and they never touch the network. That is the +required PR gate. + +**Conformance is separate.** The language-agnostic suite is downloaded and run black-box +against the built CLI/engine, in its own directory (`conformance/`) and its own CI job +(`conformance`, non-blocking by default). Run it locally with: + +```sh +make conformance # == pytest conformance +``` + +The suite is **not** a submodule. `conformance/conftest.py` clones `fruwehq/harel-conformance` at the release tag matching this package's version (falling back to `main` while the tag does not yet exist) into a gitignored `.cache/` directory and reuses it. To force a refresh, delete `.cache/`. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b806791 --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ +.PHONY: test conformance lint typecheck check all + +# Unit tests — the implementation's own suite. Hermetic and offline. +test: + pytest -q + +# Conformance — the language-agnostic suite from fruwehq/harel-conformance, run +# black-box against this implementation. Downloads the suite (pinned to the release +# tag matching this package's version) into .cache/ on first run. +# Offline / against a local checkout: HAREL_CONFORMANCE_DIR=/path/to/harel-conformance make conformance +conformance: + pytest conformance -q + +lint: + ruff check . + +typecheck: + mypy src + +# Everything a PR needs to pass locally (unit gate), plus conformance. +check: lint typecheck test + +all: check conformance diff --git a/README.md b/README.md index da170cd..1a19ba3 100644 --- a/README.md +++ b/README.md @@ -108,14 +108,22 @@ logging.basicConfig(level=logging.DEBUG) # or logging.getLogger("harel").setLe ## Layout - `src/harel/` — the package. -- `tests/` — unit tests and the conformance harness. +- `tests/` — the implementation's own **unit tests** (hermetic, offline). +- `conformance/` — the harness that runs the external **conformance suite** black-box + against this implementation (kept separate from the unit tests). ## Develop ``` python -m venv .venv && . .venv/bin/activate pip install -e '.[dev]' -ruff check . && mypy src/harel && pytest # the suite is fetched into .cache/ on first run + +make check # ruff + mypy + unit tests (hermetic, offline) — the PR gate +make conformance # download & run the language-agnostic conformance suite ``` +Equivalently: `pytest` runs the unit tests only; `pytest conformance` runs the +conformance suite (it fetches `harel-conformance` into `.cache/` on first run — set +`HAREL_CONFORMANCE_DIR` to use a local checkout offline). The two are **separate**: +unit tests never touch the network; conformance is opt-in. ## License MIT — see [LICENSE](LICENSE). diff --git a/conformance/__init__.py b/conformance/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/conformance/conftest.py similarity index 100% rename from tests/conftest.py rename to conformance/conftest.py diff --git a/tests/harness.py b/conformance/harness.py similarity index 100% rename from tests/harness.py rename to conformance/harness.py diff --git a/tests/test_conformance.py b/conformance/test_conformance.py similarity index 100% rename from tests/test_conformance.py rename to conformance/test_conformance.py