diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e736541..36def61 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,6 +7,8 @@ on: branches: [main] jobs: + # Unit tests: the implementation's own suite — hermetic, offline, fast. + # This is the blocking gate (`test (ubuntu-24.04)`). test: runs-on: ${{ matrix.os }} strategy: @@ -25,5 +27,21 @@ jobs: run: ruff check . - name: Mypy run: mypy src/harel - - name: Pytest + - name: Unit tests run: pytest -q + + # Conformance: the language-agnostic suite from fruwehq/harel-conformance, run + # black-box against this implementation. Separate job: it downloads an external + # repo, so a failure here means "diverges from the spec suite", not "our code broke". + conformance: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + cache: pip + - name: Install + run: pip install -e '.[dev]' + - name: Conformance suite + run: pytest conformance -q diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f4009b4..a836b70 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,20 +19,32 @@ Python ≥ 3.11. The package is import-named `harel`, distribution-named `harel- ## The gate -Before pushing, run all three and keep them clean: +Before pushing, run all three and keep them clean (`make check` does exactly this): ```sh ruff check . mypy src -pytest +pytest # unit tests only — hermetic, offline ``` CI runs `test (ubuntu-24.04)` on every PR and is **required** — a PR merges only once it -is green. +is green. This job runs the **unit tests only**. -## The conformance suite +## Unit tests vs. conformance — kept separate -The suite is **not** a submodule. `tests/conftest.py` clones +This implementation has its **own unit tests** (`tests/`), which are hermetic and offline +— `pytest` (or `make test`) runs only these, and they never touch the network. That is the +required PR gate. + +**Conformance is separate.** The language-agnostic suite is downloaded and run black-box +against the built CLI/engine, in its own directory (`conformance/`) and its own CI job +(`conformance`, non-blocking by default). Run it locally with: + +```sh +make conformance # == pytest conformance +``` + +The suite is **not** a submodule. `conformance/conftest.py` clones `fruwehq/harel-conformance` at the release tag matching this package's version (falling back to `main` while the tag does not yet exist) into a gitignored `.cache/` directory and reuses it. To force a refresh, delete `.cache/`. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b806791 --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ +.PHONY: test conformance lint typecheck check all + +# Unit tests — the implementation's own suite. Hermetic and offline. +test: + pytest -q + +# Conformance — the language-agnostic suite from fruwehq/harel-conformance, run +# black-box against this implementation. Downloads the suite (pinned to the release +# tag matching this package's version) into .cache/ on first run. +# Offline / against a local checkout: HAREL_CONFORMANCE_DIR=/path/to/harel-conformance make conformance +conformance: + pytest conformance -q + +lint: + ruff check . + +typecheck: + mypy src + +# Everything a PR needs to pass locally (unit gate), plus conformance. +check: lint typecheck test + +all: check conformance diff --git a/README.md b/README.md index da170cd..1a19ba3 100644 --- a/README.md +++ b/README.md @@ -108,14 +108,22 @@ logging.basicConfig(level=logging.DEBUG) # or logging.getLogger("harel").setLe ## Layout - `src/harel/` — the package. -- `tests/` — unit tests and the conformance harness. +- `tests/` — the implementation's own **unit tests** (hermetic, offline). +- `conformance/` — the harness that runs the external **conformance suite** black-box + against this implementation (kept separate from the unit tests). ## Develop ``` python -m venv .venv && . .venv/bin/activate pip install -e '.[dev]' -ruff check . && mypy src/harel && pytest # the suite is fetched into .cache/ on first run + +make check # ruff + mypy + unit tests (hermetic, offline) — the PR gate +make conformance # download & run the language-agnostic conformance suite ``` +Equivalently: `pytest` runs the unit tests only; `pytest conformance` runs the +conformance suite (it fetches `harel-conformance` into `.cache/` on first run — set +`HAREL_CONFORMANCE_DIR` to use a local checkout offline). The two are **separate**: +unit tests never touch the network; conformance is opt-in. ## License MIT — see [LICENSE](LICENSE). diff --git a/conformance/__init__.py b/conformance/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/conformance/conftest.py similarity index 100% rename from tests/conftest.py rename to conformance/conftest.py diff --git a/tests/harness.py b/conformance/harness.py similarity index 100% rename from tests/harness.py rename to conformance/harness.py diff --git a/tests/test_conformance.py b/conformance/test_conformance.py similarity index 100% rename from tests/test_conformance.py rename to conformance/test_conformance.py