From 73d5eedefe3b1fff87bd1ef8ca3b64a312593405 Mon Sep 17 00:00:00 2001 From: "mr.Shu" Date: Mon, 6 Apr 2026 18:23:22 +0200 Subject: [PATCH 1/4] ci: add GitHub Actions workflow to run tests on PRs Previously no CI workflow ran pytest on pull requests, so regressions could land undetected. This adds a test.yml workflow triggered on PRs and pushes to main. - Split into three parallel jobs: core, inspect, helm - Core job runs tests that need no optional dependencies - Inspect and HELM jobs install their respective extras - Pin action versions to match regenerate_types.yml - Add timeout-minutes: 10 to guard against hung jobs - Update uv.lock Closes #88 --- .github/workflows/test.yml | 64 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..9d4f78f62 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,64 @@ +name: Tests + +on: + pull_request: + push: + branches: [main] + +jobs: + core: + name: Core tests + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6.0.2 + - uses: astral-sh/setup-uv@v7.6.0 + + - name: Install core dependencies + run: uv sync --locked + + - name: Run core tests + run: | + uv run pytest \ + tests/test_validate.py \ + tests/test_check_duplicate_entries.py \ + tests/test_inspect_uuid_utils.py \ + tests/test_cli_inspect_uuid.py \ + tests/test_lm_eval_adapter.py \ + -v + + inspect: + name: Inspect converter tests + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6.0.2 + - uses: astral-sh/setup-uv@v7.6.0 + + - name: Install dependencies with inspect extra + run: uv sync --locked --extra inspect + + - name: Run inspect tests + run: | + uv run pytest \ + tests/test_inspect_adapter.py \ + tests/test_inspect_instance_level_adapter.py \ + -v + + helm: + name: HELM converter tests + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6.0.2 + - uses: astral-sh/setup-uv@v7.6.0 + + - name: Install dependencies with helm extra + run: uv sync --locked --extra helm + + - name: Run HELM tests + run: | + uv run pytest \ + tests/test_helm_adapter.py \ + tests/test_helm_instance_level_adapter.py \ + -v From f3fbe384511c0709968c6ba8e27c8719da2fd49e Mon Sep 17 00:00:00 2001 From: joncrall Date: Wed, 13 May 2026 14:58:34 -0400 Subject: [PATCH 2/4] Port to a locked / loose system for the rebase --- .github/workflows/test.yml | 89 +++++++++++++++----------------------- 1 file changed, 36 insertions(+), 53 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9d4f78f62..b2fee83cc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,64 +1,47 @@ name: Tests - on: pull_request: push: branches: [main] - +permissions: + contents: read jobs: - core: - name: Core tests + tests: + name: ${{ matrix.deps }} / ${{ matrix.resolution }} runs-on: ubuntu-latest timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + include: + - deps: core + resolution: locked + sync: uv sync --locked + pytest_args: >- + tests + --ignore-glob=tests/test_inspect*.py + --ignore-glob=tests/test_helm*.py + -v + - deps: core + resolution: loose + sync: uv sync --upgrade + pytest_args: >- + tests + --ignore-glob=tests/test_inspect*.py + --ignore-glob=tests/test_helm*.py + -v + - deps: full + resolution: locked + sync: uv sync --locked --all-extras + pytest_args: tests -v + - deps: full + resolution: loose + sync: uv sync --upgrade --all-extras + pytest_args: tests -v steps: - uses: actions/checkout@v6.0.2 - uses: astral-sh/setup-uv@v7.6.0 - - - name: Install core dependencies - run: uv sync --locked - - - name: Run core tests - run: | - uv run pytest \ - tests/test_validate.py \ - tests/test_check_duplicate_entries.py \ - tests/test_inspect_uuid_utils.py \ - tests/test_cli_inspect_uuid.py \ - tests/test_lm_eval_adapter.py \ - -v - - inspect: - name: Inspect converter tests - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@v6.0.2 - - uses: astral-sh/setup-uv@v7.6.0 - - - name: Install dependencies with inspect extra - run: uv sync --locked --extra inspect - - - name: Run inspect tests - run: | - uv run pytest \ - tests/test_inspect_adapter.py \ - tests/test_inspect_instance_level_adapter.py \ - -v - - helm: - name: HELM converter tests - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@v6.0.2 - - uses: astral-sh/setup-uv@v7.6.0 - - - name: Install dependencies with helm extra - run: uv sync --locked --extra helm - - - name: Run HELM tests - run: | - uv run pytest \ - tests/test_helm_adapter.py \ - tests/test_helm_instance_level_adapter.py \ - -v + - name: Install dependencies + run: ${{ matrix.sync }} + - name: Run tests + run: uv run pytest ${{ matrix.pytest_args }} From 836b05c0c8f4c1462b2351897b52322a339b651e Mon Sep 17 00:00:00 2001 From: joncrall Date: Wed, 13 May 2026 15:04:26 -0400 Subject: [PATCH 3/4] Rely on pytest markers to skip tests --- .github/workflows/test.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b2fee83cc..5a1d66302 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,19 +17,11 @@ jobs: - deps: core resolution: locked sync: uv sync --locked - pytest_args: >- - tests - --ignore-glob=tests/test_inspect*.py - --ignore-glob=tests/test_helm*.py - -v + pytest_args: tests -v - deps: core resolution: loose sync: uv sync --upgrade - pytest_args: >- - tests - --ignore-glob=tests/test_inspect*.py - --ignore-glob=tests/test_helm*.py - -v + pytest_args: tests -v - deps: full resolution: locked sync: uv sync --locked --all-extras From 7c6a1f7d72d29a31eb021ee55de9aabc794f06d5 Mon Sep 17 00:00:00 2001 From: joncrall Date: Wed, 13 May 2026 15:25:42 -0400 Subject: [PATCH 4/4] Update HELM minimum to support ModelDeploymentNotFoundError --- pyproject.toml | 2 +- uv.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index da63d2c4f..d98b832c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ [project.optional-dependencies] inspect = ["inspect-ai>=0.3.160,<0.4.0"] helm = [ - "crfm-helm>=0.5.12", + "crfm-helm>=0.5.14", "typer>=0.12,<1.0", ] all = [ diff --git a/uv.lock b/uv.lock index 09af603a8..b945a2145 100644 --- a/uv.lock +++ b/uv.lock @@ -602,7 +602,7 @@ wheels = [ [[package]] name = "crfm-helm" -version = "0.5.12" +version = "0.5.16" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "bottle" }, @@ -631,9 +631,9 @@ dependencies = [ { name = "uncertainty-calibration" }, { name = "zstandard" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9e/57/8f0ffd8973d48f0430a9bfa9ed22f6f1f11fed6c547df7d03b468f8c9833/crfm_helm-0.5.12.tar.gz", hash = "sha256:7ac444123f8f1f25d71e3727bbd119a5dd53653236b1442a8cad9fc6bb5d6418", size = 8263909, upload-time = "2026-01-28T00:37:29.974Z" } +sdist = { url = "https://files.pythonhosted.org/packages/12/4d/395b742b9897a2966ee17e44c9d638ffb96716641acef86d80ef377bb72f/crfm_helm-0.5.16.tar.gz", hash = "sha256:2ce0e3e2b81298f93c2882b52a66c1619dc40460d7eadfb6b3d7e1717294a9d6", size = 8289168, upload-time = "2026-04-30T05:07:21.124Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/50/7cf798d9f0f6c18456be3bf281fee90e8932485234e8b1bf475e2fcee307/crfm_helm-0.5.12-py3-none-any.whl", hash = "sha256:01e83d17f7140cccf65443f0117a1375ccd3cec04c5d939587f9e16fcd77cba1", size = 8843831, upload-time = "2026-01-28T00:37:26.771Z" }, + { url = "https://files.pythonhosted.org/packages/7c/7e/b4eb7d63ccdd39856c39b89dfb7eb1c0548a694a2b57d7656440afa1d930/crfm_helm-0.5.16-py3-none-any.whl", hash = "sha256:e15190fc43ed61c648c6b3844eb4f3e434728630fb28184d2dc00f4607febe7d", size = 8889144, upload-time = "2026-04-30T05:07:17.998Z" }, ] [[package]] @@ -883,7 +883,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "crfm-helm", marker = "extra == 'helm'", specifier = ">=0.5.12" }, + { name = "crfm-helm", marker = "extra == 'helm'", specifier = ">=0.5.14" }, { name = "datamodel-code-generator", extras = ["ruff"], specifier = ">=0.52.2" }, { name = "duckdb", specifier = ">=1.5.2" }, { name = "every-eval-ever", extras = ["helm"], marker = "extra == 'all'" },