From 0d2e5397c212534d0993934ec017959986b18896 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Sat, 21 Mar 2026 23:02:46 +0400 Subject: [PATCH 01/36] Improve CodSpeed benchmark observability --- .github/workflows/cd-build.yaml | 15 +------- .github/workflows/cd-codspeed.yaml | 62 ++++++++++++++++++++++++++++++ .github/workflows/cd.yaml | 8 +++- pyproject.toml | 2 +- taskfile.yaml | 9 +++++ 5 files changed, 80 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/cd-codspeed.yaml diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index 61cc3a9..b90833a 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -112,19 +112,6 @@ jobs: if: ${{ matrix.build.os != 'windows-11-arm' || !contains(fromJSON('["3.10","3.11","3.12"]'), matrix.python.label) }} run: uv run --no-sync pytest tests/ -vvv - - name: Setup venv for codspeed - if: runner.os == 'Linux' && matrix.build.arch_label == 'x86_64' && contains(fromJSON('["3.13","3.14"]'), matrix.python.label) - run: | - uv venv --python ${{ matrix.python.version }} --clear - uv sync --extra test --no-install-project - - - name: Run codspeed benchmarks - if: runner.os == 'Linux' && matrix.build.arch_label == 'x86_64' && contains(fromJSON('["3.13","3.14"]'), matrix.python.label) - uses: CodSpeedHQ/action@v4 - with: - mode: simulation - run: uv run --with "${{ steps.wheel.outputs.path }}" --no-sync pytest tests/ --codspeed -k test_performance -v - - name: Setup venv for shuffle tests if: ${{ matrix.build.os != 'windows-11-arm' || !contains(fromJSON('["3.10","3.11","3.12"]'), matrix.python.label) }} run: | @@ -179,4 +166,4 @@ jobs: COPIUM_PATCH_ENABLE: "1" COPIUM_USE_DICT_MEMO: "1" PYTHONPATH: ${{ github.workspace }}/cpython-tests/Lib - run: uv run --no-sync python -m unittest test.test_copy -v \ No newline at end of file + run: uv run --no-sync python -m unittest test.test_copy -v diff --git a/.github/workflows/cd-codspeed.yaml b/.github/workflows/cd-codspeed.yaml new file mode 100644 index 0000000..324236c --- /dev/null +++ b/.github/workflows/cd-codspeed.yaml @@ -0,0 +1,62 @@ +name: CodSpeed Benchmarks + +on: + workflow_call: + workflow_dispatch: + +jobs: + codspeed: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v5 + with: + submodules: recursive + + - name: Setup uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + # CodSpeed tracing requires actions/setup-python rather than uv-managed Python installs. + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.14" + + - name: Download benchmark wheel + uses: actions/download-artifact@v4 + with: + name: Wheels-3.14-Linux-x86_64 + path: wheelhouse + + - name: Find benchmark wheel + id: wheel + shell: bash + run: | + wheel_path=$(find wheelhouse -name "*-cp314-*-manylinux*_x86_64.whl" -type f | head -n1) + if [[ -z "$wheel_path" ]]; then + wheel_path=$(find wheelhouse -name "*-cp314-*-musllinux*_x86_64.whl" -type f | head -n1) + fi + if [[ -z "$wheel_path" ]]; then + echo "No wheel found for cp314 on Linux x86_64" + exit 1 + fi + echo "path=$wheel_path" >> "$GITHUB_OUTPUT" + + - name: Install benchmark dependencies + run: | + uv venv --python 3.14 --clear + uv sync --extra test --no-install-project + uv pip install "${{ steps.wheel.outputs.path }}" + + # Wall-time mode is intentionally not enabled here because CodSpeed macro runners + # currently require the repository to live under a GitHub organization. + - name: Run CodSpeed benchmarks + env: + PYTHONHASHSEED: "0" + uses: CodSpeedHQ/action@v4 + with: + mode: simulation,memory + run: uv run --no-sync pytest -o addopts='' tests/test_performance.py --codspeed -v diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml index ad0c7fb..0d8e255 100644 --- a/.github/workflows/cd.yaml +++ b/.github/workflows/cd.yaml @@ -21,7 +21,13 @@ jobs: build: uses: ./.github/workflows/cd-build.yaml - # Main branch only: benchmarks + # PR and main branch observability benchmarks + codspeed: + if: github.event_name == 'pull_request' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' + needs: build + uses: ./.github/workflows/cd-codspeed.yaml + + # Main branch only: longer-running benchmarks and README assets pyperformance: if: github.ref == 'refs/heads/main' needs: build diff --git a/pyproject.toml b/pyproject.toml index e7019da..a54d589 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,7 +86,7 @@ test = [ "indifference>=0.2.0", "typing-extensions; python_version < '3.12'", "datamodelzoo", - "pytest-codspeed>=4.2.0", + "pytest-codspeed>=4.3.0", "pytest-test-groups>=1.2.1", "psutil>=5.9.0", "pytest-random-order>=1.2.0", diff --git a/taskfile.yaml b/taskfile.yaml index 65d9e7e..0c616e0 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -111,6 +111,15 @@ tasks: done PY + benchmark:codspeed: + desc: Run the CodSpeed benchmark suite locally with Python 3.14 + env: + PYTHONHASHSEED: "0" + cmds: + - uv venv --python 3.14 --clear + - uv sync --inexact --quiet --extra test + - uv run --no-sync pytest -o addopts='' tests/test_performance.py --codspeed -v + build:wheel: desc: Regular optimized wheel (LTO + -O3 by default) cmds: From 297258b6020fef0ed674af84ae0cb3dd87c01e1e Mon Sep 17 00:00:00 2001 From: Bobronium Date: Sat, 21 Mar 2026 23:08:36 +0400 Subject: [PATCH 02/36] Make CodSpeed workflow reusable-only --- .github/workflows/cd-codspeed.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cd-codspeed.yaml b/.github/workflows/cd-codspeed.yaml index 324236c..5369209 100644 --- a/.github/workflows/cd-codspeed.yaml +++ b/.github/workflows/cd-codspeed.yaml @@ -2,7 +2,6 @@ name: CodSpeed Benchmarks on: workflow_call: - workflow_dispatch: jobs: codspeed: From da42289e0144d01191ff373d8e0df401f00e3e29 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Sat, 21 Mar 2026 23:10:40 +0400 Subject: [PATCH 03/36] Run CodSpeed independently from the build matrix --- .github/workflows/cd-codspeed.yaml | 15 +++++++++------ .github/workflows/cd.yaml | 1 - 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cd-codspeed.yaml b/.github/workflows/cd-codspeed.yaml index 5369209..3c979f0 100644 --- a/.github/workflows/cd-codspeed.yaml +++ b/.github/workflows/cd-codspeed.yaml @@ -24,19 +24,22 @@ jobs: with: python-version: "3.14" - - name: Download benchmark wheel - uses: actions/download-artifact@v4 + - name: Build benchmark wheel + uses: PyO3/maturin-action@v1 with: - name: Wheels-3.14-Linux-x86_64 - path: wheelhouse + target: x86_64-unknown-linux-gnu + manylinux: auto + args: --release --out dist -i python3.14 + rust-toolchain: nightly + sccache: "true" - name: Find benchmark wheel id: wheel shell: bash run: | - wheel_path=$(find wheelhouse -name "*-cp314-*-manylinux*_x86_64.whl" -type f | head -n1) + wheel_path=$(find dist -name "*-cp314-*-manylinux*_x86_64.whl" -type f | head -n1) if [[ -z "$wheel_path" ]]; then - wheel_path=$(find wheelhouse -name "*-cp314-*-musllinux*_x86_64.whl" -type f | head -n1) + wheel_path=$(find dist -name "*-cp314-*-musllinux*_x86_64.whl" -type f | head -n1) fi if [[ -z "$wheel_path" ]]; then echo "No wheel found for cp314 on Linux x86_64" diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml index 0d8e255..d897603 100644 --- a/.github/workflows/cd.yaml +++ b/.github/workflows/cd.yaml @@ -24,7 +24,6 @@ jobs: # PR and main branch observability benchmarks codspeed: if: github.event_name == 'pull_request' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' - needs: build uses: ./.github/workflows/cd-codspeed.yaml # Main branch only: longer-running benchmarks and README assets From 220f0c88fc33986ec4588f616c15f7955742caa9 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Sat, 21 Mar 2026 23:14:26 +0400 Subject: [PATCH 04/36] Clear sccache wrapper before CodSpeed setup --- .github/workflows/cd-codspeed.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/cd-codspeed.yaml b/.github/workflows/cd-codspeed.yaml index 3c979f0..cd80172 100644 --- a/.github/workflows/cd-codspeed.yaml +++ b/.github/workflows/cd-codspeed.yaml @@ -33,6 +33,10 @@ jobs: rust-toolchain: nightly sccache: "true" + - name: Clear sccache wrapper + # uv still resolves project metadata for extras, so leave rustc callable directly. + run: echo "RUSTC_WRAPPER=" >> "$GITHUB_ENV" + - name: Find benchmark wheel id: wheel shell: bash From 1ae5ed427e92c39b76f012da90a9cef0f301a078 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Sat, 21 Mar 2026 23:40:09 +0400 Subject: [PATCH 05/36] Stabilize CodSpeed benchmark sequencing --- .github/workflows/cd-codspeed.yaml | 176 +++++++++++++++++++++++++---- 1 file changed, 152 insertions(+), 24 deletions(-) diff --git a/.github/workflows/cd-codspeed.yaml b/.github/workflows/cd-codspeed.yaml index cd80172..b64db9d 100644 --- a/.github/workflows/cd-codspeed.yaml +++ b/.github/workflows/cd-codspeed.yaml @@ -4,7 +4,8 @@ on: workflow_call: jobs: - codspeed: + simulation_python_3_13: + name: simulation (Python 3.13) runs-on: ubuntu-latest permissions: contents: read @@ -12,57 +13,184 @@ jobs: - uses: actions/checkout@v5 with: submodules: recursive + fetch-depth: 0 + fetch-tags: true - name: Setup uv uses: astral-sh/setup-uv@v6 with: enable-cache: true - # CodSpeed tracing requires actions/setup-python rather than uv-managed Python installs. - - name: Set up Python - uses: actions/setup-python@v6 + - name: Build benchmark wheel for Python 3.13 + uses: PyO3/maturin-action@v1 with: - python-version: "3.14" + target: x86_64-unknown-linux-gnu + manylinux: auto + args: --release --out dist/python-3-13 -i 3.13 + rust-toolchain: nightly + sccache: "true" - - name: Build benchmark wheel + - name: Clear sccache wrapper after Python 3.13 build + run: echo "RUSTC_WRAPPER=" >> "$GITHUB_ENV" + + - name: Find benchmark wheel for Python 3.13 + id: python_3_13_wheel + shell: bash + run: | + python_3_13_wheel_path=$(find dist/python-3-13 -name "*-cp313-*-manylinux*_x86_64.whl" -type f | head -n1) + if [[ -z "$python_3_13_wheel_path" ]]; then + python_3_13_wheel_path=$(find dist/python-3-13 -name "*-cp313-*-musllinux*_x86_64.whl" -type f | head -n1) + fi + if [[ -z "$python_3_13_wheel_path" ]]; then + echo "No wheel found for Python 3.13 on Linux x86_64" + exit 1 + fi + echo "path=$python_3_13_wheel_path" >> "$GITHUB_OUTPUT" + + - name: Setup test environment for Python 3.13 + run: | + uv venv --python 3.13 --clear + uv sync --extra test --no-install-project + uv pip install "${{ steps.python_3_13_wheel.outputs.path }}" + + - name: Test built wheel for Python 3.13 + run: uv run --no-sync pytest tests/ -vvv + + - name: Setup CodSpeed environment for Python 3.13 + run: | + uv venv --python 3.13 --clear + uv sync --extra test --no-install-project + + - name: Run CodSpeed simulation benchmarks for Python 3.13 + uses: CodSpeedHQ/action@v4 + with: + mode: simulation + run: uv run --with "${{ steps.python_3_13_wheel.outputs.path }}" --no-sync pytest tests/ --codspeed -k test_performance -v + + simulation_python_3_14: + name: simulation (Python 3.14) + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v5 + with: + submodules: recursive + fetch-depth: 0 + fetch-tags: true + + - name: Setup uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Build benchmark wheel for Python 3.14 uses: PyO3/maturin-action@v1 with: target: x86_64-unknown-linux-gnu manylinux: auto - args: --release --out dist -i python3.14 + args: --release --out dist/python-3-14 -i 3.14 rust-toolchain: nightly sccache: "true" - - name: Clear sccache wrapper - # uv still resolves project metadata for extras, so leave rustc callable directly. + - name: Clear sccache wrapper after Python 3.14 build run: echo "RUSTC_WRAPPER=" >> "$GITHUB_ENV" - - name: Find benchmark wheel - id: wheel + - name: Find benchmark wheel for Python 3.14 + id: python_3_14_wheel shell: bash run: | - wheel_path=$(find dist -name "*-cp314-*-manylinux*_x86_64.whl" -type f | head -n1) - if [[ -z "$wheel_path" ]]; then - wheel_path=$(find dist -name "*-cp314-*-musllinux*_x86_64.whl" -type f | head -n1) + python_3_14_wheel_path=$(find dist/python-3-14 -name "*-cp314-*-manylinux*_x86_64.whl" -type f | head -n1) + if [[ -z "$python_3_14_wheel_path" ]]; then + python_3_14_wheel_path=$(find dist/python-3-14 -name "*-cp314-*-musllinux*_x86_64.whl" -type f | head -n1) fi - if [[ -z "$wheel_path" ]]; then - echo "No wheel found for cp314 on Linux x86_64" + if [[ -z "$python_3_14_wheel_path" ]]; then + echo "No wheel found for Python 3.14 on Linux x86_64" exit 1 fi - echo "path=$wheel_path" >> "$GITHUB_OUTPUT" + echo "path=$python_3_14_wheel_path" >> "$GITHUB_OUTPUT" + + - name: Setup test environment for Python 3.14 + run: | + uv venv --python 3.14 --clear + uv sync --extra test --no-install-project + uv pip install "${{ steps.python_3_14_wheel.outputs.path }}" + + - name: Test built wheel for Python 3.14 + run: uv run --no-sync pytest tests/ -vvv + + - name: Setup CodSpeed environment for Python 3.14 + run: | + uv venv --python 3.14 --clear + uv sync --extra test --no-install-project + + - name: Run CodSpeed simulation benchmarks for Python 3.14 + uses: CodSpeedHQ/action@v4 + with: + mode: simulation + run: uv run --with "${{ steps.python_3_14_wheel.outputs.path }}" --no-sync pytest tests/ --codspeed -k test_performance -v + + memory_python_3_14: + name: memory (Python 3.14) + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v5 + with: + submodules: recursive + fetch-depth: 0 + fetch-tags: true + + - name: Setup uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Build benchmark wheel for Python 3.14 + uses: PyO3/maturin-action@v1 + with: + target: x86_64-unknown-linux-gnu + manylinux: auto + args: --release --out dist/python-3-14 -i 3.14 + rust-toolchain: nightly + sccache: "true" + + - name: Clear sccache wrapper after Python 3.14 build + run: echo "RUSTC_WRAPPER=" >> "$GITHUB_ENV" + + - name: Find benchmark wheel for Python 3.14 + id: python_3_14_wheel + shell: bash + run: | + python_3_14_wheel_path=$(find dist/python-3-14 -name "*-cp314-*-manylinux*_x86_64.whl" -type f | head -n1) + if [[ -z "$python_3_14_wheel_path" ]]; then + python_3_14_wheel_path=$(find dist/python-3-14 -name "*-cp314-*-musllinux*_x86_64.whl" -type f | head -n1) + fi + if [[ -z "$python_3_14_wheel_path" ]]; then + echo "No wheel found for Python 3.14 on Linux x86_64" + exit 1 + fi + echo "path=$python_3_14_wheel_path" >> "$GITHUB_OUTPUT" + + - name: Setup test environment for Python 3.14 + run: | + uv venv --python 3.14 --clear + uv sync --extra test --no-install-project + uv pip install "${{ steps.python_3_14_wheel.outputs.path }}" + + - name: Test built wheel for Python 3.14 + run: uv run --no-sync pytest tests/ -vvv - - name: Install benchmark dependencies + - name: Setup CodSpeed memory environment for Python 3.14 run: | uv venv --python 3.14 --clear uv sync --extra test --no-install-project - uv pip install "${{ steps.wheel.outputs.path }}" # Wall-time mode is intentionally not enabled here because CodSpeed macro runners # currently require the repository to live under a GitHub organization. - - name: Run CodSpeed benchmarks - env: - PYTHONHASHSEED: "0" + - name: Run CodSpeed memory benchmarks for Python 3.14 uses: CodSpeedHQ/action@v4 with: - mode: simulation,memory - run: uv run --no-sync pytest -o addopts='' tests/test_performance.py --codspeed -v + mode: memory + run: uv run --with "${{ steps.python_3_14_wheel.outputs.path }}" --no-sync pytest tests/ --codspeed -k test_performance -v From cf68572ff2c65d91d10176c2c1ca810db526ab4d Mon Sep 17 00:00:00 2001 From: Bobronium Date: Sat, 21 Mar 2026 23:42:07 +0400 Subject: [PATCH 06/36] Restore CodSpeed simulation to build lanes --- .github/workflows/cd-build.yaml | 13 +++ .github/workflows/cd-codspeed.yaml | 126 ----------------------------- 2 files changed, 13 insertions(+), 126 deletions(-) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index b90833a..dafb92a 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -112,6 +112,19 @@ jobs: if: ${{ matrix.build.os != 'windows-11-arm' || !contains(fromJSON('["3.10","3.11","3.12"]'), matrix.python.label) }} run: uv run --no-sync pytest tests/ -vvv + - name: Setup venv for CodSpeed + if: runner.os == 'Linux' && matrix.build.arch_label == 'x86_64' && contains(fromJSON('["3.13","3.14"]'), matrix.python.label) + run: | + uv venv --python ${{ matrix.python.version }} --clear + uv sync --extra test --no-install-project + + - name: Run CodSpeed simulation benchmarks + if: runner.os == 'Linux' && matrix.build.arch_label == 'x86_64' && contains(fromJSON('["3.13","3.14"]'), matrix.python.label) + uses: CodSpeedHQ/action@v4 + with: + mode: simulation + run: uv run --with "${{ steps.wheel.outputs.path }}" --no-sync pytest tests/ --codspeed -k test_performance -v + - name: Setup venv for shuffle tests if: ${{ matrix.build.os != 'windows-11-arm' || !contains(fromJSON('["3.10","3.11","3.12"]'), matrix.python.label) }} run: | diff --git a/.github/workflows/cd-codspeed.yaml b/.github/workflows/cd-codspeed.yaml index b64db9d..28e9b50 100644 --- a/.github/workflows/cd-codspeed.yaml +++ b/.github/workflows/cd-codspeed.yaml @@ -4,132 +4,6 @@ on: workflow_call: jobs: - simulation_python_3_13: - name: simulation (Python 3.13) - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - uses: actions/checkout@v5 - with: - submodules: recursive - fetch-depth: 0 - fetch-tags: true - - - name: Setup uv - uses: astral-sh/setup-uv@v6 - with: - enable-cache: true - - - name: Build benchmark wheel for Python 3.13 - uses: PyO3/maturin-action@v1 - with: - target: x86_64-unknown-linux-gnu - manylinux: auto - args: --release --out dist/python-3-13 -i 3.13 - rust-toolchain: nightly - sccache: "true" - - - name: Clear sccache wrapper after Python 3.13 build - run: echo "RUSTC_WRAPPER=" >> "$GITHUB_ENV" - - - name: Find benchmark wheel for Python 3.13 - id: python_3_13_wheel - shell: bash - run: | - python_3_13_wheel_path=$(find dist/python-3-13 -name "*-cp313-*-manylinux*_x86_64.whl" -type f | head -n1) - if [[ -z "$python_3_13_wheel_path" ]]; then - python_3_13_wheel_path=$(find dist/python-3-13 -name "*-cp313-*-musllinux*_x86_64.whl" -type f | head -n1) - fi - if [[ -z "$python_3_13_wheel_path" ]]; then - echo "No wheel found for Python 3.13 on Linux x86_64" - exit 1 - fi - echo "path=$python_3_13_wheel_path" >> "$GITHUB_OUTPUT" - - - name: Setup test environment for Python 3.13 - run: | - uv venv --python 3.13 --clear - uv sync --extra test --no-install-project - uv pip install "${{ steps.python_3_13_wheel.outputs.path }}" - - - name: Test built wheel for Python 3.13 - run: uv run --no-sync pytest tests/ -vvv - - - name: Setup CodSpeed environment for Python 3.13 - run: | - uv venv --python 3.13 --clear - uv sync --extra test --no-install-project - - - name: Run CodSpeed simulation benchmarks for Python 3.13 - uses: CodSpeedHQ/action@v4 - with: - mode: simulation - run: uv run --with "${{ steps.python_3_13_wheel.outputs.path }}" --no-sync pytest tests/ --codspeed -k test_performance -v - - simulation_python_3_14: - name: simulation (Python 3.14) - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - uses: actions/checkout@v5 - with: - submodules: recursive - fetch-depth: 0 - fetch-tags: true - - - name: Setup uv - uses: astral-sh/setup-uv@v6 - with: - enable-cache: true - - - name: Build benchmark wheel for Python 3.14 - uses: PyO3/maturin-action@v1 - with: - target: x86_64-unknown-linux-gnu - manylinux: auto - args: --release --out dist/python-3-14 -i 3.14 - rust-toolchain: nightly - sccache: "true" - - - name: Clear sccache wrapper after Python 3.14 build - run: echo "RUSTC_WRAPPER=" >> "$GITHUB_ENV" - - - name: Find benchmark wheel for Python 3.14 - id: python_3_14_wheel - shell: bash - run: | - python_3_14_wheel_path=$(find dist/python-3-14 -name "*-cp314-*-manylinux*_x86_64.whl" -type f | head -n1) - if [[ -z "$python_3_14_wheel_path" ]]; then - python_3_14_wheel_path=$(find dist/python-3-14 -name "*-cp314-*-musllinux*_x86_64.whl" -type f | head -n1) - fi - if [[ -z "$python_3_14_wheel_path" ]]; then - echo "No wheel found for Python 3.14 on Linux x86_64" - exit 1 - fi - echo "path=$python_3_14_wheel_path" >> "$GITHUB_OUTPUT" - - - name: Setup test environment for Python 3.14 - run: | - uv venv --python 3.14 --clear - uv sync --extra test --no-install-project - uv pip install "${{ steps.python_3_14_wheel.outputs.path }}" - - - name: Test built wheel for Python 3.14 - run: uv run --no-sync pytest tests/ -vvv - - - name: Setup CodSpeed environment for Python 3.14 - run: | - uv venv --python 3.14 --clear - uv sync --extra test --no-install-project - - - name: Run CodSpeed simulation benchmarks for Python 3.14 - uses: CodSpeedHQ/action@v4 - with: - mode: simulation - run: uv run --with "${{ steps.python_3_14_wheel.outputs.path }}" --no-sync pytest tests/ --codspeed -k test_performance -v - memory_python_3_14: name: memory (Python 3.14) runs-on: ubuntu-latest From c99851afeaea3e65379f9b879e31131d61c61600 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Mon, 23 Mar 2026 08:58:23 +0400 Subject: [PATCH 07/36] Add CodSpeed walltime benchmarks --- .github/workflows/cd-build.yaml | 80 +++++++++++++++++++++++++++++- .github/workflows/cd-codspeed.yaml | 70 -------------------------- .github/workflows/cd.yaml | 5 -- 3 files changed, 78 insertions(+), 77 deletions(-) delete mode 100644 .github/workflows/cd-codspeed.yaml diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index dafb92a..d214ecb 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -113,18 +113,31 @@ jobs: run: uv run --no-sync pytest tests/ -vvv - name: Setup venv for CodSpeed - if: runner.os == 'Linux' && matrix.build.arch_label == 'x86_64' && contains(fromJSON('["3.13","3.14"]'), matrix.python.label) + if: ${{ (github.event_name == 'pull_request' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') && runner.os == 'Linux' && matrix.build.arch_label == 'x86_64' && contains(fromJSON('["3.13","3.14"]'), matrix.python.label) }} run: | uv venv --python ${{ matrix.python.version }} --clear uv sync --extra test --no-install-project - name: Run CodSpeed simulation benchmarks - if: runner.os == 'Linux' && matrix.build.arch_label == 'x86_64' && contains(fromJSON('["3.13","3.14"]'), matrix.python.label) + if: ${{ (github.event_name == 'pull_request' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') && runner.os == 'Linux' && matrix.build.arch_label == 'x86_64' && contains(fromJSON('["3.13","3.14"]'), matrix.python.label) }} uses: CodSpeedHQ/action@v4 with: mode: simulation run: uv run --with "${{ steps.wheel.outputs.path }}" --no-sync pytest tests/ --codspeed -k test_performance -v + - name: Setup CodSpeed memory environment + if: ${{ (github.event_name == 'pull_request' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') && runner.os == 'Linux' && matrix.build.arch_label == 'x86_64' && matrix.python.label == '3.14' }} + run: | + uv venv --python ${{ matrix.python.version }} --clear + uv sync --extra test --no-install-project + + - name: Run CodSpeed memory benchmarks + if: ${{ (github.event_name == 'pull_request' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') && runner.os == 'Linux' && matrix.build.arch_label == 'x86_64' && matrix.python.label == '3.14' }} + uses: CodSpeedHQ/action@v4 + with: + mode: memory + run: uv run --with "${{ steps.wheel.outputs.path }}" --no-sync pytest tests/ --codspeed -k test_performance -v + - name: Setup venv for shuffle tests if: ${{ matrix.build.os != 'windows-11-arm' || !contains(fromJSON('["3.10","3.11","3.12"]'), matrix.python.label) }} run: | @@ -180,3 +193,66 @@ jobs: COPIUM_USE_DICT_MEMO: "1" PYTHONPATH: ${{ github.workspace }}/cpython-tests/Lib run: uv run --no-sync python -m unittest test.test_copy -v + + codspeed_walltime_python_3_14: + if: github.event_name == 'pull_request' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' + name: walltime (Python 3.14) + runs-on: codspeed-macro + + steps: + - uses: actions/checkout@v5 + with: + submodules: recursive + fetch-depth: 0 + fetch-tags: true + + - name: Setup uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Build walltime benchmark wheel for Python 3.14 + uses: PyO3/maturin-action@v1 + with: + target: aarch64-unknown-linux-gnu + manylinux: auto + args: --release --out dist/walltime-python-3-14 -i 3.14 + rust-toolchain: nightly + sccache: "true" + + - name: Clear sccache wrapper after walltime wheel build + run: echo "RUSTC_WRAPPER=" >> "$GITHUB_ENV" + + - name: Find walltime benchmark wheel for Python 3.14 + id: walltime_python_3_14_wheel + shell: bash + run: | + walltime_python_3_14_wheel_path=$(find dist/walltime-python-3-14 -name "*-cp314-*-manylinux*_aarch64.whl" -type f | head -n1) + if [[ -z "$walltime_python_3_14_wheel_path" ]]; then + walltime_python_3_14_wheel_path=$(find dist/walltime-python-3-14 -name "*-cp314-*-musllinux*_aarch64.whl" -type f | head -n1) + fi + if [[ -z "$walltime_python_3_14_wheel_path" ]]; then + echo "No wheel found for Python 3.14 on Linux ARM64" + exit 1 + fi + echo "path=$walltime_python_3_14_wheel_path" >> "$GITHUB_OUTPUT" + + - name: Setup test environment for walltime Python 3.14 + run: | + uv venv --python 3.14 --clear + uv sync --extra test --no-install-project + uv pip install "${{ steps.walltime_python_3_14_wheel.outputs.path }}" + + - name: Test built walltime wheel for Python 3.14 + run: uv run --no-sync pytest tests/ -vvv + + - name: Setup CodSpeed walltime environment for Python 3.14 + run: | + uv venv --python 3.14 --clear + uv sync --extra test --no-install-project + + - name: Run CodSpeed walltime benchmarks for Python 3.14 + uses: CodSpeedHQ/action@v4 + with: + mode: walltime + run: uv run --with "${{ steps.walltime_python_3_14_wheel.outputs.path }}" --no-sync pytest tests/ --codspeed -k test_performance -v diff --git a/.github/workflows/cd-codspeed.yaml b/.github/workflows/cd-codspeed.yaml deleted file mode 100644 index 28e9b50..0000000 --- a/.github/workflows/cd-codspeed.yaml +++ /dev/null @@ -1,70 +0,0 @@ -name: CodSpeed Benchmarks - -on: - workflow_call: - -jobs: - memory_python_3_14: - name: memory (Python 3.14) - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - uses: actions/checkout@v5 - with: - submodules: recursive - fetch-depth: 0 - fetch-tags: true - - - name: Setup uv - uses: astral-sh/setup-uv@v6 - with: - enable-cache: true - - - name: Build benchmark wheel for Python 3.14 - uses: PyO3/maturin-action@v1 - with: - target: x86_64-unknown-linux-gnu - manylinux: auto - args: --release --out dist/python-3-14 -i 3.14 - rust-toolchain: nightly - sccache: "true" - - - name: Clear sccache wrapper after Python 3.14 build - run: echo "RUSTC_WRAPPER=" >> "$GITHUB_ENV" - - - name: Find benchmark wheel for Python 3.14 - id: python_3_14_wheel - shell: bash - run: | - python_3_14_wheel_path=$(find dist/python-3-14 -name "*-cp314-*-manylinux*_x86_64.whl" -type f | head -n1) - if [[ -z "$python_3_14_wheel_path" ]]; then - python_3_14_wheel_path=$(find dist/python-3-14 -name "*-cp314-*-musllinux*_x86_64.whl" -type f | head -n1) - fi - if [[ -z "$python_3_14_wheel_path" ]]; then - echo "No wheel found for Python 3.14 on Linux x86_64" - exit 1 - fi - echo "path=$python_3_14_wheel_path" >> "$GITHUB_OUTPUT" - - - name: Setup test environment for Python 3.14 - run: | - uv venv --python 3.14 --clear - uv sync --extra test --no-install-project - uv pip install "${{ steps.python_3_14_wheel.outputs.path }}" - - - name: Test built wheel for Python 3.14 - run: uv run --no-sync pytest tests/ -vvv - - - name: Setup CodSpeed memory environment for Python 3.14 - run: | - uv venv --python 3.14 --clear - uv sync --extra test --no-install-project - - # Wall-time mode is intentionally not enabled here because CodSpeed macro runners - # currently require the repository to live under a GitHub organization. - - name: Run CodSpeed memory benchmarks for Python 3.14 - uses: CodSpeedHQ/action@v4 - with: - mode: memory - run: uv run --with "${{ steps.python_3_14_wheel.outputs.path }}" --no-sync pytest tests/ --codspeed -k test_performance -v diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml index d897603..848c903 100644 --- a/.github/workflows/cd.yaml +++ b/.github/workflows/cd.yaml @@ -21,11 +21,6 @@ jobs: build: uses: ./.github/workflows/cd-build.yaml - # PR and main branch observability benchmarks - codspeed: - if: github.event_name == 'pull_request' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' - uses: ./.github/workflows/cd-codspeed.yaml - # Main branch only: longer-running benchmarks and README assets pyperformance: if: github.ref == 'refs/heads/main' From 4e681cdb693c89e1191ea861e836820e906d0a9d Mon Sep 17 00:00:00 2001 From: Bobronium Date: Mon, 23 Mar 2026 09:02:14 +0400 Subject: [PATCH 08/36] Reuse build artifact for CodSpeed walltime --- .github/workflows/cd-build.yaml | 52 ++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index d214ecb..ca2098c 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -198,6 +198,9 @@ jobs: if: github.event_name == 'pull_request' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' name: walltime (Python 3.14) runs-on: codspeed-macro + permissions: + actions: read + contents: read steps: - uses: actions/checkout@v5 @@ -211,17 +214,46 @@ jobs: with: enable-cache: true - - name: Build walltime benchmark wheel for Python 3.14 - uses: PyO3/maturin-action@v1 + - name: Wait for the Python 3.14 ARM64 wheel artifact + id: wait_for_walltime_wheel_artifact + uses: actions/github-script@v8 with: - target: aarch64-unknown-linux-gnu - manylinux: auto - args: --release --out dist/walltime-python-3-14 -i 3.14 - rust-toolchain: nightly - sccache: "true" - - - name: Clear sccache wrapper after walltime wheel build - run: echo "RUSTC_WRAPPER=" >> "$GITHUB_ENV" + script: | + const walltimeWheelArtifactName = "Wheels-3.14-Linux-ARM64"; + const timeoutMilliseconds = 20 * 60 * 1000; + const pollIntervalMilliseconds = 10 * 1000; + const startedAtMilliseconds = Date.now(); + + while (Date.now() - startedAtMilliseconds < timeoutMilliseconds) { + const { data } = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + per_page: 100, + }); + + const matchingArtifact = data.artifacts.find( + (artifact) => artifact.name === walltimeWheelArtifactName && !artifact.expired, + ); + + if (matchingArtifact) { + core.setOutput("artifact_name", matchingArtifact.name); + return; + } + + core.info(`Waiting for ${walltimeWheelArtifactName} to be uploaded by the ARM64 wheel build job.`); + await new Promise((resolve) => setTimeout(resolve, pollIntervalMilliseconds)); + } + + core.setFailed("Timed out while waiting for the Python 3.14 ARM64 wheel artifact."); + + - name: Download the Python 3.14 ARM64 wheel artifact + uses: actions/download-artifact@v4 + with: + github-token: ${{ github.token }} + run-id: ${{ github.run_id }} + name: ${{ steps.wait_for_walltime_wheel_artifact.outputs.artifact_name }} + path: dist/walltime-python-3-14 - name: Find walltime benchmark wheel for Python 3.14 id: walltime_python_3_14_wheel From b1aa0e0471cf0d11c20d298cc51421e908aaa2a0 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Mon, 23 Mar 2026 09:05:47 +0400 Subject: [PATCH 09/36] Avoid waiting on CodSpeed walltime runner --- .github/workflows/cd-build.yaml | 152 ++++++++++++++++++++++++-------- 1 file changed, 113 insertions(+), 39 deletions(-) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index ca2098c..35a954d 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -30,6 +30,9 @@ jobs: - { tag: "cp313", version: "3.13.9", label: "3.13" } - { tag: "cp314", version: "3.14.0", label: "3.14" } - { tag: "cp314t", version: "3.14t", label: "3.14t", branch: "3.14" } + exclude: + - build: { os: ubuntu-24.04-arm, rust_target: aarch64-unknown-linux-gnu, manylinux: auto, wheel_arch: "manylinux*_aarch64", arch_label: "ARM64", platform_label: "Linux" } + python: { tag: "cp314", version: "3.14.0", label: "3.14" } runs-on: ${{ matrix.build.os }} @@ -194,13 +197,119 @@ jobs: PYTHONPATH: ${{ github.workspace }}/cpython-tests/Lib run: uv run --no-sync python -m unittest test.test_copy -v + wheel_python_3_14_linux_arm64: + runs-on: ubuntu-24.04-arm + name: "Python 3.14 - Linux - ARM64" + + steps: + - uses: actions/checkout@v5 + with: + submodules: recursive + fetch-depth: 0 + fetch-tags: true + + - name: Setup uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Build wheel + uses: PyO3/maturin-action@v1 + with: + target: aarch64-unknown-linux-gnu + manylinux: auto + args: --release --out dist -i 3.14 + rust-toolchain: nightly + sccache: 'true' + + - name: Clear sccache wrapper + run: echo "RUSTC_WRAPPER=" >> "$GITHUB_ENV" + + - uses: actions/upload-artifact@v4 + with: + name: "Wheels-3.14-Linux-ARM64" + path: dist/*.whl + + - name: Find built wheel + id: wheel + shell: bash + run: | + walltime_arm64_wheel_path=$(find dist -name "*-cp314-*-manylinux*_aarch64.whl" -type f | head -n1) + + if [[ -z "$walltime_arm64_wheel_path" ]]; then + walltime_arm64_wheel_path=$(find dist -name "*-cp314-*-musllinux*_aarch64.whl" -type f | head -n1) + fi + + if [[ -z "$walltime_arm64_wheel_path" ]]; then + echo "No compatible wheel found" + exit 1 + fi + + echo "Found: $walltime_arm64_wheel_path" + echo "path=$walltime_arm64_wheel_path" >> "$GITHUB_OUTPUT" + + - name: Setup test environment + run: | + uv venv --python 3.14 --clear + uv sync --extra test --no-install-project + uv pip install "${{ steps.wheel.outputs.path }}" + + - name: "Test: Built wheel" + run: uv run --no-sync pytest tests/ -vvv + + - name: Setup venv for shuffle tests + run: | + uv venv --python 3.14 --clear + uv sync --extra test --no-install-project + uv pip install "${{ steps.wheel.outputs.path }}" + + - name: "Test: Shuffle A" + run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=A + + - name: "Test: Shuffle B" + run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=B + + - name: "Test: Shuffle C" + run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=C + + - name: "Test: Shuffle D" + run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=D + + - name: "Cache CPython test suite" + id: cpython_cache + uses: actions/cache@v4 + with: + path: cpython-tests + key: cpython-tests-3.14 + + - name: "Checkout CPython test suite" + if: steps.cpython_cache.outputs.cache-hit != 'true' + uses: actions/checkout@v5 + with: + repository: python/cpython + ref: 3.14 + sparse-checkout: Lib/test + sparse-checkout-cone-mode: true + path: cpython-tests + + - name: "Test: CPython test_copy (patched)" + env: + COPIUM_PATCH_ENABLE: "1" + PYTHONPATH: ${{ github.workspace }}/cpython-tests/Lib + run: uv run --no-sync python -m unittest test.test_copy -v + + - name: "Test: CPython test_copy (patched, dict memo)" + env: + COPIUM_PATCH_ENABLE: "1" + COPIUM_USE_DICT_MEMO: "1" + PYTHONPATH: ${{ github.workspace }}/cpython-tests/Lib + run: uv run --no-sync python -m unittest test.test_copy -v + codspeed_walltime_python_3_14: if: github.event_name == 'pull_request' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' + needs: wheel_python_3_14_linux_arm64 name: walltime (Python 3.14) runs-on: codspeed-macro - permissions: - actions: read - contents: read steps: - uses: actions/checkout@v5 @@ -214,45 +323,10 @@ jobs: with: enable-cache: true - - name: Wait for the Python 3.14 ARM64 wheel artifact - id: wait_for_walltime_wheel_artifact - uses: actions/github-script@v8 - with: - script: | - const walltimeWheelArtifactName = "Wheels-3.14-Linux-ARM64"; - const timeoutMilliseconds = 20 * 60 * 1000; - const pollIntervalMilliseconds = 10 * 1000; - const startedAtMilliseconds = Date.now(); - - while (Date.now() - startedAtMilliseconds < timeoutMilliseconds) { - const { data } = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId, - per_page: 100, - }); - - const matchingArtifact = data.artifacts.find( - (artifact) => artifact.name === walltimeWheelArtifactName && !artifact.expired, - ); - - if (matchingArtifact) { - core.setOutput("artifact_name", matchingArtifact.name); - return; - } - - core.info(`Waiting for ${walltimeWheelArtifactName} to be uploaded by the ARM64 wheel build job.`); - await new Promise((resolve) => setTimeout(resolve, pollIntervalMilliseconds)); - } - - core.setFailed("Timed out while waiting for the Python 3.14 ARM64 wheel artifact."); - - name: Download the Python 3.14 ARM64 wheel artifact uses: actions/download-artifact@v4 with: - github-token: ${{ github.token }} - run-id: ${{ github.run_id }} - name: ${{ steps.wait_for_walltime_wheel_artifact.outputs.artifact_name }} + name: "Wheels-3.14-Linux-ARM64" path: dist/walltime-python-3-14 - name: Find walltime benchmark wheel for Python 3.14 From 1b741ed0ce0155af4f92b4ef0e4826cfc74871a1 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Mon, 23 Mar 2026 19:20:37 +0400 Subject: [PATCH 10/36] Rewrite benchmarks --- tests/test_performance.py | 658 ++++++++++++++++++++++++++++++++------ 1 file changed, 562 insertions(+), 96 deletions(-) diff --git a/tests/test_performance.py b/tests/test_performance.py index bc1ecbf..0468695 100644 --- a/tests/test_performance.py +++ b/tests/test_performance.py @@ -1,126 +1,592 @@ -# SPDX-FileCopyrightText: 2025-present Arseny Boykov (Bobronium) -# -# SPDX-License-Identifier: MIT +""" +copium.deepcopy benchmark suite for CodSpeed. + +Each synthetic group isolates one code path in the deepcopy pipeline. +Within a group, variants share identical structure but differ in the +measured signal. 3+ scale points per signal. Real-world cases detect +end-to-end regression across representative workloads. + +Deepcopy pipeline (from deepcopium.rs): + + 1. pre-memo atomic? → return immediately (None/int/str/bool/float/bytes) + 2. memo recall → hit: return cached; miss: continue + 3. type dispatch → tuple / dict / list / set (exact type) + 4. post-memo atomic? → return immediately (re.Pattern/type/range/function/…) + 5. specialized → frozenset / bytearray / bound method + 6. reduce fallback → __deepcopy__ or __reduce_ex__ +""" import copy as stdlib_copy import platform -import random +import re import sys -from itertools import chain -from typing import Any +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from typing import Any, NamedTuple import pytest - import copium import copium.patch -from datamodelzoo import CASES -from datamodelzoo import Case -BASE_CASES = [ - case - for case in CASES - if "raises" not in case.name and "thirdparty" not in case.name and "guard" not in case.name -] -GUARD_CASES = [case for case in CASES if "guard" in case.name] - -random.seed(1) - -COMBINED_CASES = [ - Case( - "all", - factory=lambda: (c := [case.obj for case in BASE_CASES] * 1000, random.shuffle(c), c)[-1], - ), - Case( - "cpython:91610", - factory=lambda: [case.obj for case in BASE_CASES if "91610" in case.name], - ), - Case( - "diverse_atomic", - factory=lambda: [case.obj for case in BASE_CASES if "atom:" in case.name] * 1000, - ), - Case( - "all_proto", - factory=lambda: [case.obj for case in BASE_CASES if "proto:" in case.name] * 1000, - ), - Case( - "all_reflexive", - factory=lambda: [case.obj for case in BASE_CASES if "reflexive" in case.name] * 10, - ), - Case( - "all_empty", - factory=lambda: [case.obj for case in BASE_CASES if "empty" in case.name] * 100, - ), - Case( - "all_stdlib", - factory=lambda: [case.obj for case in BASE_CASES if "stdlib" in case.name] * 1000, - ), -] + +class Case(NamedTuple): + name: str + obj: Any + + +def scaled(tag, factory, sizes): + return [Case(f"{tag}-n-{n}", factory(n)) for n in sizes] + + +def depth_scaled(tag, factory, depths): + return [Case(f"{tag}-d-{d}", factory(d)) for d in depths] + + +def generate_params(cases): + return pytest.mark.parametrize( + "case", [pytest.param(c, id=c.name) for c in cases] + ) + python_version = ".".join(map(str, sys.version_info[:2])) if not getattr(sys, "_is_gil_enabled", lambda: True)(): python_version += "t" python_version += f"-{platform.machine()}" -PYTHON_VERSION_PARAM = pytest.mark.parametrize("_python", [python_version]) +PYTHON_VERSION = pytest.mark.parametrize("_python", [python_version]) + +SIZES = (10, 100, 1000) +DEPTHS = (10, 100, 500) +ATOM_SIZES = (100, 1000, 10000) +REDUCE_SIZES = (10, 50, 200) + + +# ═══════════════════════════════════════════════════════════ +# MEMO ISOLATION +# +# Constant shape: {'a': (X, X, X), 'b': [X] * n} +# +# Outer dict (2 keys) and inner tuple/list are the same +# across all variants. X controls which memo path fires: +# +# shared_mut → memo hit after first (shallow leaf) +# shared_deep → memo hit after first (recursive leaf) +# shared_tuple_atom → tuple all_same path, never memoised +# shared_tuple_mut → tuple content changes → memo store + hits +# shared_atom → pre-memo atomic skip, no memo +# unique_atom → pre-memo atomic skip, distinct id()s +# unique_mut → memo store each, zero hits +# ═══════════════════════════════════════════════════════════ + + +def memo_shared_mut(n): + leaf = [1, 2, 3] + return {"a": (leaf, leaf, leaf), "b": [leaf] * n} + + +def memo_shared_deep(n): + leaf = [[1, 2], {"k": "v"}, [3, 4]] + return {"a": (leaf, leaf, leaf), "b": [leaf] * n} + + +def memo_shared_tuple_atom(n): + leaf = (1, 2, 3) + return {"a": (leaf, leaf, leaf), "b": [leaf] * n} + + +def memo_shared_tuple_mut(n): + leaf = ([],) + return {"a": (leaf, leaf, leaf), "b": [leaf] * n} + + +def memo_shared_atom(n): + return {"a": (None, None, None), "b": [None] * n} + + +def memo_unique_atom(n): + return {"a": (1, 2, 3), "b": list(range(n))} + -COMBINED_CASES_PARAMS = pytest.mark.parametrize( - "case", - [pytest.param(case, id=case.name) for case in COMBINED_CASES], +def memo_unique_mut(n): + return {"a": ([], [], []), "b": [[] for _ in range(n)]} + + +MEMO_CASES = ( + scaled("shared_mut", memo_shared_mut, SIZES) + + scaled("shared_deep", memo_shared_deep, SIZES) + + scaled("shared_tuple_atom", memo_shared_tuple_atom, SIZES) + + scaled("shared_tuple_mut", memo_shared_tuple_mut, SIZES) + + scaled("shared_atom", memo_shared_atom, SIZES) + + scaled("unique_atom", memo_unique_atom, SIZES) + + scaled("unique_mut", memo_unique_mut, SIZES) ) -BASE_CASES_PARAMS = pytest.mark.parametrize( - "case", - [pytest.param(case, id=case.name) for case in chain(BASE_CASES, GUARD_CASES)], + +# ═══════════════════════════════════════════════════════════ +# CONTAINER TRAVERSAL +# +# Flat container of n atomic ints. +# Isolates per-container creation + traversal cost. +# ═══════════════════════════════════════════════════════════ + +CONTAINER_CASES = ( + scaled("list", lambda n: list(range(n)), SIZES) + + scaled("tuple", lambda n: tuple(range(n)), SIZES) + + scaled("dict", lambda n: {i: i for i in range(n)}, SIZES) + + scaled("set", lambda n: set(range(n)), SIZES) + + scaled("frozenset", lambda n: frozenset(range(n)), SIZES) + + scaled("bytearray", lambda n: bytearray(n), (100, 10_000, 1_000_000)) ) -@BASE_CASES_PARAMS -@PYTHON_VERSION_PARAM -def test_individual_cases_warmup(case: Any, copy, _python, benchmark) -> None: - copy.deepcopy(case.obj) +# ═══════════════════════════════════════════════════════════ +# NESTING DEPTH +# +# Single chain d levels deep. Leaf = [1, 2, 3] (mutable) +# except tuple_atom which uses atomic leaf to trigger +# the all_same optimisation at every level. +# ═══════════════════════════════════════════════════════════ + + +def nested_list(d): + obj = [1, 2, 3] + for _ in range(d): + obj = [obj] + return obj + + +def nested_dict(d): + obj = [1, 2, 3] + for _ in range(d): + obj = {"k": obj} + return obj + + +def nested_tuple_mut(d): + obj = [1, 2, 3] + for _ in range(d): + obj = (obj,) + return obj + + +def nested_tuple_atom(d): + obj = 42 + for _ in range(d): + obj = (obj,) + return obj + + +DEPTH_CASES = ( + depth_scaled("list", nested_list, DEPTHS) + + depth_scaled("dict", nested_dict, DEPTHS) + + depth_scaled("tuple_mut", nested_tuple_mut, DEPTHS) + + depth_scaled("tuple_atom", nested_tuple_atom, DEPTHS) +) + + +# ═══════════════════════════════════════════════════════════ +# ATOMIC FAST PATH +# +# Outer list of n items. List overhead is constant across +# variants; we measure per-item dispatch cost. +# +# Pre-memo atomics: None, int, str, bool, float, bytes +# → is_literal_immutable fires before memo +# Post-memo atomics: re.Pattern, type objects +# → memo recall miss, then is_postmemo_atomic fires +# ═══════════════════════════════════════════════════════════ + +CACHED_RE = re.compile(r"^test$") + + +def mixed_prememo_atoms(n): + pool = [None, 42, "s", True, 3.14, b"b"] + return [pool[i % 6] for i in range(n)] + + +ATOMIC_CASES = ( + scaled("none", lambda n: [None] * n, ATOM_SIZES) + + scaled("int", lambda n: list(range(n)), ATOM_SIZES) + + scaled("str", lambda n: [f"s{i}" for i in range(n)], ATOM_SIZES) + + scaled("mixed_builtin_atomics", mixed_prememo_atoms, ATOM_SIZES) + + scaled("re.Pattern", lambda n: [CACHED_RE] * n, ATOM_SIZES) + + scaled("type", lambda n: [int] * n, ATOM_SIZES) +) + + +# ═══════════════════════════════════════════════════════════ +# REDUCE PROTOCOL +# +# Objects going through __reduce_ex__ / __deepcopy__. +# List of n instances to scale. +# ═══════════════════════════════════════════════════════════ + + +@dataclass +class SimpleDataclass: + x: int + y: str + + +@dataclass +class MutableDataclass: + x: int + items: list = field(default_factory=list) + mapping: dict = field(default_factory=dict) + +@dataclass +class NestedDataclass: + inner: SimpleDataclass + items: list = field(default_factory=list) -@COMBINED_CASES_PARAMS -@PYTHON_VERSION_PARAM -def test_combined_cases_warmup(case: Any, copy, _python, benchmark) -> None: - copy.deepcopy(case.obj) +class SlotsObject: + __slots__ = ("x", "y", "z") -# Initially tests were only running on 3.13 x86_64 -if python_version == "3.13-x86_64": - # backwards compatibility with previous benchmarks runs + def __init__(self, x, y, z): + self.x = x + self.y = y + self.z = z - @BASE_CASES_PARAMS - def test_individual_cases(case: Any, copy, benchmark) -> None: - benchmark(copy.deepcopy, case.obj) - @COMBINED_CASES_PARAMS - def test_combined_cases(case: Any, copy, benchmark) -> None: - benchmark(copy.deepcopy, case.obj) +class CustomDeepcopyObject: + def __init__(self, v): + self.v = v -else: - assert sys.version_info >= (3, 14) or "--codspeed" not in sys.argv, ( - "This block assumed to have newer versions only." + def __deepcopy__(self, memo): + return CustomDeepcopyObject(stdlib_copy.deepcopy(self.v, memo)) + + +REDUCE_CASES = ( + scaled( + "dataclass_simple", + lambda n: [SimpleDataclass(i, f"v{i}") for i in range(n)], + REDUCE_SIZES, + ) + + scaled( + "dataclass_mutable", + lambda n: [MutableDataclass(i, [i], {"k": i}) for i in range(n)], + REDUCE_SIZES, + ) + + scaled( + "dataclass_nested", + lambda n: [NestedDataclass(SimpleDataclass(i, f"v{i}"), [i]) for i in range(n)], + REDUCE_SIZES, + ) + + scaled( + "slots", + lambda n: [SlotsObject(i, f"v{i}", float(i)) for i in range(n)], + REDUCE_SIZES, ) + + scaled( + "datetime", + lambda n: [datetime(2024, 1, 1) + timedelta(days=i) for i in range(n)], + REDUCE_SIZES, + ) + + scaled( + "custom_deepcopy", + lambda n: [CustomDeepcopyObject([i]) for i in range(n)], + REDUCE_SIZES, + ) +) + + +# ═══════════════════════════════════════════════════════════ +# EDGE CASES +# +# Structural pathologies: cycles, empties, dense sharing, +# all_same tuples at scale. +# ═══════════════════════════════════════════════════════════ + + +def make_cyclic_list(): + a = [1, 2, 3] + a.append(a) + return a + + +def make_cyclic_dict(): + d = {"k": "v"} + d["self"] = d + return d + + +def make_dense_refs(): + nodes = [[i] for i in range(50)] + return [nodes[i % 50] for i in range(2500)] + + +def wide_dict(n): + return {f"k{i}": [i] for i in range(n)} + + +EDGE_CASES = [ + Case("cyclic_list", make_cyclic_list()), + Case("cyclic_dict", make_cyclic_dict()), + Case("empties", [[], (), {}, set(), frozenset(), bytearray()]), + Case("tuple_allsame_10k", (None,) * 10000), + Case("tuple_alldiff_1k", tuple([] for _ in range(1000))), + Case("dense_refs_50x50", make_dense_refs()), + *scaled("wide_dict", wide_dict, (100, 1000, 5000)), +] + + +# ═══════════════════════════════════════════════════════════ +# REAL-WORLD +# +# Representative production deepcopy patterns. +# Data is self-contained and deterministic. +# ═══════════════════════════════════════════════════════════ + + +def make_json_api_response(): + return { + "status": "ok", + "pagination": {"page": 1, "per_page": 20, "total": 142}, + "data": [ + { + "id": i, + "type": "user", + "attributes": { + "name": f"User {i}", + "email": f"u{i}@x.com", + "active": i % 3 != 0, + "score": float(i * 17 % 100), + "tags": ["admin", "verified"] if i % 5 == 0 else ["user"], + "metadata": {"joined": "2024-01-15", "logins": i * 7}, + }, + "relationships": { + "team": {"data": {"type": "team", "id": i % 4}}, + "projects": { + "data": [ + {"type": "project", "id": i * 10 + j} for j in range(3) + ] + }, + }, + } + for i in range(20) + ], + "included": [ + {"type": "team", "id": t, "attributes": {"name": f"Team {t}"}} + for t in range(4) + ], + "meta": {"request_id": "abc-123", "timing_ms": 42.5}, + } + + +def make_config_with_shared_defaults(): + defaults = {"timeout": 30, "retries": 3, "backoff": 1.5} + return { + "version": "2.1.0", + "environments": { + env: { + "database": { + "host": f"db-{env}", + "port": 5432, + "pool_size": pool_size, + "options": defaults, + }, + "cache": {"host": f"redis-{env}", "port": 6379, "options": defaults}, + "features": { + "oauth": env != "dev", + "debug": env == "dev", + "providers": ["google", "github"] if env != "dev" else [], + }, + } + for env, pool_size in [("dev", 2), ("staging", 5), ("prod", 20)] + }, + "shared": { + "origins": ["https://app.example.com", "https://api.example.com"], + "headers": ("Content-Type", "Authorization", "X-Request-ID"), + "error_codes": frozenset({400, 401, 403, 404, 500}), + }, + } + + +def make_openapi_fragment(): + def schema(name, fields): + return { + "type": "object", + "title": name, + "properties": {f: {"type": t} for f, t in fields}, + "required": [f for f, _ in fields], + } + + base_fields = [ + ("id", "integer"), + ("name", "string"), + ("created_at", "string"), + ("updated_at", "string"), + ("metadata", "object"), + ] + + schemas = {} + for model in ("User", "Project", "Task", "Comment"): + schemas[model] = schema(model, base_fields) + schemas[f"{model}List"] = { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": {"$ref": f"#/components/schemas/{model}"}, + }, + "total": {"type": "integer"}, + "page": {"type": "integer"}, + }, + } + + paths = {} + for resource in ("users", "projects", "tasks"): + paths[f"/api/v1/{resource}"] = { + method: { + "operationId": f"{method}_{resource}", + "tags": [resource], + "parameters": [ + {"name": "page", "in": "query", "schema": {"type": "integer"}}, + {"name": "per_page", "in": "query", "schema": {"type": "integer"}}, + ], + "responses": { + "200": {"description": "OK"}, + "404": {"description": "Not found"}, + }, + } + for method in ("get", "post") + } + + return { + "openapi": "3.0.3", + "info": {"title": "Example API", "version": "1.0.0"}, + "paths": paths, + "components": {"schemas": schemas}, + } + + +def make_tabular_data(n): + categories = ("A", "B", "C", "D") + return [ + { + "id": i, + "name": f"item_{i}", + "value": float(i * 3.14), + "category": categories[i % 4], + "active": i % 7 != 0, + "tags": [f"t{j}" for j in range(i % 4)], + } + for i in range(n) + ] + + +def make_grayscale_image_1024x1024(): + return [[(r * 4 + c) % 256 for c in range(1024)] for r in range(1024)] + + +@dataclass +class OrmUser: + id: int + name: str + prefs: dict = field(default_factory=dict) + sessions: list = field(default_factory=list) + + +@dataclass +class OrmSession: + token: str + created: datetime + data: dict = field(default_factory=dict) + + +def make_orm_graph(): + shared_prefs = {"theme": "dark", "lang": "en", "notifications": True} + return [ + OrmUser( + i, + f"u{i}", + shared_prefs, + [ + OrmSession( + f"t{i}{j}", + datetime(2024, 1, 1 + j), + {"ip": f"10.0.{i}.{j}"}, + ) + for j in range(3) + ], + ) + for i in range(10) + ] + + +REAL_WORLD_CASES = [ + Case("json_api_response", make_json_api_response()), + Case("config_shared_defaults", make_config_with_shared_defaults()), + Case("openapi_schema", make_openapi_fragment()), + Case("tabular_100", make_tabular_data(100)), + Case("tabular_1000", make_tabular_data(1000)), + Case("image_1024x1024", make_grayscale_image_1024x1024()), + Case("orm_graph_10u3s", make_orm_graph()), +] + + +# ═══════════════════════════════════════════════════════════ +# TESTS +# ═══════════════════════════════════════════════════════════ + + +@generate_params(MEMO_CASES) +@PYTHON_VERSION +def test_memo(case: Case, _python, benchmark): + benchmark(copium.deepcopy, case.obj) + + +@generate_params(CONTAINER_CASES) +@PYTHON_VERSION +def test_container(case: Case, _python, benchmark): + benchmark(copium.deepcopy, case.obj) + + +@generate_params(DEPTH_CASES) +@PYTHON_VERSION +def test_depth(case: Case, _python, benchmark): + benchmark(copium.deepcopy, case.obj) + + +@generate_params(ATOMIC_CASES) +@PYTHON_VERSION +def test_atomic(case: Case, _python, benchmark): + benchmark(copium.deepcopy, case.obj) + + +@generate_params(REDUCE_CASES) +@PYTHON_VERSION +def test_reduce(case: Case, _python, benchmark): + benchmark(copium.deepcopy, case.obj) + + +@generate_params(EDGE_CASES) +@PYTHON_VERSION +def test_edge(case: Case, _python, benchmark): + benchmark(copium.deepcopy, case.obj) + + +@generate_params(REAL_WORLD_CASES) +@PYTHON_VERSION +def test_real(case: Case, _python, benchmark): + benchmark(copium.deepcopy, case.obj) + + +@generate_params(REAL_WORLD_CASES) +@PYTHON_VERSION +def test_real_dict_memo(case: Case, _python, benchmark): + benchmark(copium.deepcopy, case.obj, {}) + + +@generate_params(REAL_WORLD_CASES) +@PYTHON_VERSION +def test_real_stdlib_patched(case: Case, _python, benchmark, copium_patch_enabled): + benchmark(stdlib_copy.deepcopy, case.obj) + - @BASE_CASES_PARAMS - @PYTHON_VERSION_PARAM - def test_individual_cases(case: Any, copy, benchmark, _python) -> None: - benchmark(copy.deepcopy, case.obj) - - @COMBINED_CASES_PARAMS - @PYTHON_VERSION_PARAM - def test_combined_cases(case: Any, copy, benchmark, _python) -> None: - benchmark(copy.deepcopy, case.obj) - - @COMBINED_CASES_PARAMS - @PYTHON_VERSION_PARAM - def test_combined_cases_copium_dict_memo(case: Any, benchmark, _python) -> None: - benchmark(copium.deepcopy, case.obj, {}) - - @COMBINED_CASES_PARAMS - @PYTHON_VERSION_PARAM - def test_combined_cases_stdlib_patched( - case: Any, benchmark, _python, copium_patch_enabled - ) -> None: - benchmark(stdlib_copy.deepcopy, case.obj) +@generate_params(REAL_WORLD_CASES) +@PYTHON_VERSION +def test_real_stdlib(case: Case, _python, benchmark): + benchmark(stdlib_copy.deepcopy, case.obj) From 02ab4ed36ec33d7fd7475d4044511a1b06d85799 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Mon, 23 Mar 2026 19:27:23 +0400 Subject: [PATCH 11/36] Fix linting --- tests/test_performance.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/tests/test_performance.py b/tests/test_performance.py index 0468695..47d1edf 100644 --- a/tests/test_performance.py +++ b/tests/test_performance.py @@ -20,11 +20,16 @@ import platform import re import sys -from dataclasses import dataclass, field -from datetime import datetime, timedelta -from typing import Any, NamedTuple +from dataclasses import dataclass +from dataclasses import field +from datetime import UTC +from datetime import datetime +from datetime import timedelta +from typing import Any +from typing import NamedTuple import pytest + import copium import copium.patch @@ -43,9 +48,7 @@ def depth_scaled(tag, factory, depths): def generate_params(cases): - return pytest.mark.parametrize( - "case", [pytest.param(c, id=c.name) for c in cases] - ) + return pytest.mark.parametrize("case", [pytest.param(c, id=c.name) for c in cases]) python_version = ".".join(map(str, sys.version_info[:2])) @@ -281,7 +284,7 @@ def __deepcopy__(self, memo): ) + scaled( "datetime", - lambda n: [datetime(2024, 1, 1) + timedelta(days=i) for i in range(n)], + lambda n: [datetime(2024, 1, 1, tzinfo=UTC) + timedelta(days=i) for i in range(n)], REDUCE_SIZES, ) + scaled( @@ -358,18 +361,13 @@ def make_json_api_response(): }, "relationships": { "team": {"data": {"type": "team", "id": i % 4}}, - "projects": { - "data": [ - {"type": "project", "id": i * 10 + j} for j in range(3) - ] - }, + "projects": {"data": [{"type": "project", "id": i * 10 + j} for j in range(3)]}, }, } for i in range(20) ], "included": [ - {"type": "team", "id": t, "attributes": {"name": f"Team {t}"}} - for t in range(4) + {"type": "team", "id": t, "attributes": {"name": f"Team {t}"}} for t in range(4) ], "meta": {"request_id": "abc-123", "timing_ms": 42.5}, } @@ -506,7 +504,7 @@ def make_orm_graph(): [ OrmSession( f"t{i}{j}", - datetime(2024, 1, 1 + j), + datetime(2024, 1, 1 + j, tzinfo=UTC), {"ip": f"10.0.{i}.{j}"}, ) for j in range(3) From 4c588fd04072b8bd5c6a08a9a4fb65b980bec6c3 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Mon, 23 Mar 2026 20:44:24 +0400 Subject: [PATCH 12/36] Update cd-build.yaml --- .github/workflows/cd-build.yaml | 259 ++++++++++++++------------------ 1 file changed, 109 insertions(+), 150 deletions(-) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index 35a954d..fb64c0a 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -14,38 +14,46 @@ jobs: max-parallel: 64 matrix: build: - # Linux - { os: ubuntu-latest, rust_target: x86_64-unknown-linux-gnu, manylinux: auto, wheel_arch: "manylinux*_x86_64", arch_label: "x86_64", platform_label: "Linux" } - { os: ubuntu-24.04-arm, rust_target: aarch64-unknown-linux-gnu, manylinux: auto, wheel_arch: "manylinux*_aarch64", arch_label: "ARM64", platform_label: "Linux" } - # Windows - { os: windows-latest, rust_target: x86_64-pc-windows-msvc, wheel_arch: "win_amd64", arch_label: "x64", platform_label: "Windows" } - { os: windows-11-arm, rust_target: aarch64-pc-windows-msvc, wheel_arch: "win_arm64", arch_label: "ARM64", platform_label: "Windows" } - # macOS - { os: macos-15, rust_target: aarch64-apple-darwin, wheel_arch: "macosx*_arm64", arch_label: "Apple Silicon", platform_label: "macOS" } - { os: macos-15-intel, rust_target: x86_64-apple-darwin, wheel_arch: "macosx*_x86_64", arch_label: "Intel", platform_label: "macOS" } python: - - { tag: "cp310", version: "3.10.11", label: "3.10" } - - { tag: "cp311", version: "3.11.9", label: "3.11" } - - { tag: "cp312", version: "3.12.10", label: "3.12" } - - { tag: "cp313", version: "3.13.9", label: "3.13" } - - { tag: "cp314", version: "3.14.0", label: "3.14" } - - { tag: "cp314t", version: "3.14t", label: "3.14t", branch: "3.14" } + - { tag: "cp310", version: "3.10.11", label: "3.10" } + - { tag: "cp311", version: "3.11.9", label: "3.11" } + - { tag: "cp312", version: "3.12.10", label: "3.12" } + - { tag: "cp313", version: "3.13.9", label: "3.13" } + - { tag: "cp314", version: "3.14.0", label: "3.14" } + - { tag: "cp314t", version: "3.14t", label: "3.14t", branch: "3.14" } exclude: - build: { os: ubuntu-24.04-arm, rust_target: aarch64-unknown-linux-gnu, manylinux: auto, wheel_arch: "manylinux*_aarch64", arch_label: "ARM64", platform_label: "Linux" } python: { tag: "cp314", version: "3.14.0", label: "3.14" } runs-on: ${{ matrix.build.os }} - name: "Python ${{ matrix.python.label }} - ${{ matrix.build.platform_label }} - ${{ matrix.build.arch_label }}" steps: - - uses: actions/checkout@v5 + - &checkout + uses: actions/checkout@v5 with: submodules: recursive fetch-depth: 0 fetch-tags: true - - name: Setup uv + - name: Compute job parameters + id: meta + shell: bash + run: | + skip_tests=false + if [[ "${{ matrix.build.os }}" == "windows-11-arm" && "${{ matrix.python.label }}" =~ ^3\.(10|11|12)$ ]]; then + skip_tests=true + fi + echo "skip_tests=$skip_tests" >> "$GITHUB_OUTPUT" + + - &setup-uv + name: Setup uv uses: astral-sh/setup-uv@v6 with: enable-cache: true @@ -54,8 +62,7 @@ jobs: if: runner.os != 'Linux' shell: bash run: | - if [[ "${{ matrix.build.os }}" == "windows-11-arm" && "${{ matrix.python.label }}" =~ ^3\.(10|11|12)$ ]]; - then + if [[ "${{ matrix.build.os }}" == "windows-11-arm" && "${{ matrix.python.label }}" =~ ^3\.(10|11|12)$ ]]; then uv python install ${{ matrix.python.version }} echo "PYTHON=python${{ matrix.python.label }}" >> "$GITHUB_ENV" else @@ -74,23 +81,25 @@ jobs: manylinux: ${{ matrix.build.manylinux || 'auto' }} args: --release --out dist -i ${{ env.PYTHON || matrix.python.label }} rust-toolchain: nightly - sccache: 'true' + sccache: "true" env: CL: ${{ runner.os == 'Windows' && '/experimental:c11atomics' || '' }} MACOSX_DEPLOYMENT_TARGET: ${{ runner.os == 'macOS' && '10.12' || '' }} - - name: Clear sccache wrapper - # this is required due to maturin failing to find sccache later during getting metadata on uv sync. + - &clear-sccache-linux + name: Clear sccache wrapper if: runner.os == 'Linux' - run: echo "RUSTC_WRAPPER=" >> "$GITHUB_ENV" + run: | + echo "RUSTC_WRAPPER=" >> "$GITHUB_ENV" - - uses: actions/upload-artifact@v4 + - name: Upload wheel artifact + uses: actions/upload-artifact@v4 with: name: "Wheels-${{ matrix.python.label }}-${{ matrix.build.platform_label }}-${{ matrix.build.arch_label }}" path: dist/*.whl - name: Find built wheel - if: ${{ matrix.build.os != 'windows-11-arm' || !contains(fromJSON('["3.10","3.11","3.12"]'), matrix.python.label) }} + if: steps.meta.outputs.skip_tests != 'true' id: wheel shell: bash run: | @@ -101,79 +110,45 @@ jobs: exit 1 fi - echo "Found: $wheel" echo "path=$wheel" >> "$GITHUB_OUTPUT" - name: Setup test environment - if: ${{ matrix.build.os != 'windows-11-arm' || !contains(fromJSON('["3.10","3.11","3.12"]'), matrix.python.label) }} + if: steps.meta.outputs.skip_tests != 'true' run: | uv venv --python ${{ matrix.python.label }} --clear uv sync --extra test --no-install-project uv pip install "${{ steps.wheel.outputs.path }}" - - name: "Test: Built wheel" - if: ${{ matrix.build.os != 'windows-11-arm' || !contains(fromJSON('["3.10","3.11","3.12"]'), matrix.python.label) }} + - name: Test built wheel + if: steps.meta.outputs.skip_tests != 'true' run: uv run --no-sync pytest tests/ -vvv - - name: Setup venv for CodSpeed - if: ${{ (github.event_name == 'pull_request' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') && runner.os == 'Linux' && matrix.build.arch_label == 'x86_64' && contains(fromJSON('["3.13","3.14"]'), matrix.python.label) }} - run: | - uv venv --python ${{ matrix.python.version }} --clear - uv sync --extra test --no-install-project - - - name: Run CodSpeed simulation benchmarks - if: ${{ (github.event_name == 'pull_request' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') && runner.os == 'Linux' && matrix.build.arch_label == 'x86_64' && contains(fromJSON('["3.13","3.14"]'), matrix.python.label) }} - uses: CodSpeedHQ/action@v4 - with: - mode: simulation - run: uv run --with "${{ steps.wheel.outputs.path }}" --no-sync pytest tests/ --codspeed -k test_performance -v - - - name: Setup CodSpeed memory environment - if: ${{ (github.event_name == 'pull_request' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') && runner.os == 'Linux' && matrix.build.arch_label == 'x86_64' && matrix.python.label == '3.14' }} - run: | - uv venv --python ${{ matrix.python.version }} --clear - uv sync --extra test --no-install-project - - - name: Run CodSpeed memory benchmarks - if: ${{ (github.event_name == 'pull_request' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') && runner.os == 'Linux' && matrix.build.arch_label == 'x86_64' && matrix.python.label == '3.14' }} - uses: CodSpeedHQ/action@v4 - with: - mode: memory - run: uv run --with "${{ steps.wheel.outputs.path }}" --no-sync pytest tests/ --codspeed -k test_performance -v - - - name: Setup venv for shuffle tests - if: ${{ matrix.build.os != 'windows-11-arm' || !contains(fromJSON('["3.10","3.11","3.12"]'), matrix.python.label) }} - run: | - uv venv --python ${{ matrix.python.label }} --clear - uv sync --extra test --no-install-project - uv pip install "${{ steps.wheel.outputs.path }}" - - - name: "Test: Shuffle A" - if: ${{ matrix.build.os != 'windows-11-arm' || !contains(fromJSON('["3.10","3.11","3.12"]'), matrix.python.label) }} + - name: Test shuffle A + if: steps.meta.outputs.skip_tests != 'true' run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=A - - name: "Test: Shuffle B" - if: ${{ matrix.build.os != 'windows-11-arm' || !contains(fromJSON('["3.10","3.11","3.12"]'), matrix.python.label) }} + - name: Test shuffle B + if: steps.meta.outputs.skip_tests != 'true' run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=B - - name: "Test: Shuffle C" - if: ${{ matrix.build.os != 'windows-11-arm' || !contains(fromJSON('["3.10","3.11","3.12"]'), matrix.python.label) }} + - name: Test shuffle C + if: steps.meta.outputs.skip_tests != 'true' run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=C - - name: "Test: Shuffle D" - if: ${{ matrix.build.os != 'windows-11-arm' || !contains(fromJSON('["3.10","3.11","3.12"]'), matrix.python.label) }} + - name: Test shuffle D + if: steps.meta.outputs.skip_tests != 'true' run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=D - - name: "Cache CPython test suite" - if: ${{ matrix.build.os != 'windows-11-arm' || !contains(fromJSON('["3.10","3.11","3.12"]'), matrix.python.label) }} + - name: Cache CPython test suite + if: steps.meta.outputs.skip_tests != 'true' id: cpython-cache uses: actions/cache@v4 with: path: cpython-tests key: cpython-tests-${{ matrix.python.branch || matrix.python.label }} - - name: "Checkout CPython test suite" - if: ${{ (matrix.build.os != 'windows-11-arm' || !contains(fromJSON('["3.10","3.11","3.12"]'), matrix.python.label)) && steps.cpython-cache.outputs.cache-hit != 'true' }} + - name: Checkout CPython test suite + if: steps.meta.outputs.skip_tests != 'true' && steps.cpython-cache.outputs.cache-hit != 'true' uses: actions/checkout@v5 with: repository: python/cpython @@ -182,15 +157,15 @@ jobs: sparse-checkout-cone-mode: true path: cpython-tests - - name: "Test: CPython test_copy (patched)" - if: ${{ matrix.build.os != 'windows-11-arm' || !contains(fromJSON('["3.10","3.11","3.12"]'), matrix.python.label) }} + - name: Test CPython test_copy (patched) + if: steps.meta.outputs.skip_tests != 'true' env: COPIUM_PATCH_ENABLE: "1" PYTHONPATH: ${{ github.workspace }}/cpython-tests/Lib run: uv run --no-sync python -m unittest test.test_copy -v - - name: "Test: CPython test_copy (patched, dict memo)" - if: ${{ matrix.build.os != 'windows-11-arm' || !contains(fromJSON('["3.10","3.11","3.12"]'), matrix.python.label) }} + - name: Test CPython test_copy (patched, dict memo) + if: steps.meta.outputs.skip_tests != 'true' env: COPIUM_PATCH_ENABLE: "1" COPIUM_USE_DICT_MEMO: "1" @@ -202,16 +177,8 @@ jobs: name: "Python 3.14 - Linux - ARM64" steps: - - uses: actions/checkout@v5 - with: - submodules: recursive - fetch-depth: 0 - fetch-tags: true - - - name: Setup uv - uses: astral-sh/setup-uv@v6 - with: - enable-cache: true + - *checkout + - *setup-uv - name: Build wheel uses: PyO3/maturin-action@v1 @@ -220,12 +187,12 @@ jobs: manylinux: auto args: --release --out dist -i 3.14 rust-toolchain: nightly - sccache: 'true' + sccache: "true" - - name: Clear sccache wrapper - run: echo "RUSTC_WRAPPER=" >> "$GITHUB_ENV" + - *clear-sccache-linux - - uses: actions/upload-artifact@v4 + - name: Upload wheel artifact + uses: actions/upload-artifact@v4 with: name: "Wheels-3.14-Linux-ARM64" path: dist/*.whl @@ -234,19 +201,15 @@ jobs: id: wheel shell: bash run: | - walltime_arm64_wheel_path=$(find dist -name "*-cp314-*-manylinux*_aarch64.whl" -type f | head -n1) - - if [[ -z "$walltime_arm64_wheel_path" ]]; then - walltime_arm64_wheel_path=$(find dist -name "*-cp314-*-musllinux*_aarch64.whl" -type f | head -n1) + wheel=$(find dist -name "*-cp314-*-manylinux*_aarch64.whl" -type f | head -n1) + if [[ -z "$wheel" ]]; then + wheel=$(find dist -name "*-cp314-*-musllinux*_aarch64.whl" -type f | head -n1) fi - - if [[ -z "$walltime_arm64_wheel_path" ]]; then + if [[ -z "$wheel" ]]; then echo "No compatible wheel found" exit 1 fi - - echo "Found: $walltime_arm64_wheel_path" - echo "path=$walltime_arm64_wheel_path" >> "$GITHUB_OUTPUT" + echo "path=$wheel" >> "$GITHUB_OUTPUT" - name: Setup test environment run: | @@ -254,36 +217,30 @@ jobs: uv sync --extra test --no-install-project uv pip install "${{ steps.wheel.outputs.path }}" - - name: "Test: Built wheel" + - name: Test built wheel run: uv run --no-sync pytest tests/ -vvv - - name: Setup venv for shuffle tests - run: | - uv venv --python 3.14 --clear - uv sync --extra test --no-install-project - uv pip install "${{ steps.wheel.outputs.path }}" - - - name: "Test: Shuffle A" + - name: Test shuffle A run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=A - - name: "Test: Shuffle B" + - name: Test shuffle B run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=B - - name: "Test: Shuffle C" + - name: Test shuffle C run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=C - - name: "Test: Shuffle D" + - name: Test shuffle D run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=D - - name: "Cache CPython test suite" - id: cpython_cache + - name: Cache CPython test suite + id: cpython-cache uses: actions/cache@v4 with: path: cpython-tests key: cpython-tests-3.14 - - name: "Checkout CPython test suite" - if: steps.cpython_cache.outputs.cache-hit != 'true' + - name: Checkout CPython test suite + if: steps.cpython-cache.outputs.cache-hit != 'true' uses: actions/checkout@v5 with: repository: python/cpython @@ -292,73 +249,75 @@ jobs: sparse-checkout-cone-mode: true path: cpython-tests - - name: "Test: CPython test_copy (patched)" + - name: Test CPython test_copy (patched) env: COPIUM_PATCH_ENABLE: "1" PYTHONPATH: ${{ github.workspace }}/cpython-tests/Lib run: uv run --no-sync python -m unittest test.test_copy -v - - name: "Test: CPython test_copy (patched, dict memo)" + - name: Test CPython test_copy (patched, dict memo) env: COPIUM_PATCH_ENABLE: "1" COPIUM_USE_DICT_MEMO: "1" PYTHONPATH: ${{ github.workspace }}/cpython-tests/Lib run: uv run --no-sync python -m unittest test.test_copy -v - codspeed_walltime_python_3_14: - if: github.event_name == 'pull_request' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' + codspeed: + name: "CodSpeed ${{ matrix.mode.label }} - Python 3.14 - ARM64 - Shard ${{ matrix.shard.name }}" needs: wheel_python_3_14_linux_arm64 - name: walltime (Python 3.14) - runs-on: codspeed-macro + runs-on: ${{ matrix.mode.runner }} - steps: - - uses: actions/checkout@v5 - with: - submodules: recursive - fetch-depth: 0 - fetch-tags: true + strategy: + fail-fast: false + matrix: + mode: + - { id: simulation, label: "CPU Simulation", runner: "ubuntu-24.04-arm", extra_env: "" } + - { id: memory, label: "Memory", runner: "ubuntu-24.04-arm", extra_env: "CODSPEED_MEMORY=1" } + - { id: walltime, label: "WallTime", runner: "codspeed-macro", extra_env: "" } + shard: + - { name: "core", expr: "test_performance and (test_memo or test_container or test_depth)" } + - { name: "mid", expr: "test_performance and (test_atomic or test_reduce or test_edge)" } + - { name: "real", expr: "test_performance and (test_real or test_real_dict_memo or test_real_stdlib_patched or test_real_stdlib)" } - - name: Setup uv - uses: astral-sh/setup-uv@v6 - with: - enable-cache: true + steps: + - *checkout + - *setup-uv - - name: Download the Python 3.14 ARM64 wheel artifact + - name: Download benchmark wheel artifact uses: actions/download-artifact@v4 with: name: "Wheels-3.14-Linux-ARM64" - path: dist/walltime-python-3-14 + path: dist/benchmark-wheel - - name: Find walltime benchmark wheel for Python 3.14 - id: walltime_python_3_14_wheel + - name: Find benchmark wheel + id: wheel shell: bash run: | - walltime_python_3_14_wheel_path=$(find dist/walltime-python-3-14 -name "*-cp314-*-manylinux*_aarch64.whl" -type f | head -n1) - if [[ -z "$walltime_python_3_14_wheel_path" ]]; then - walltime_python_3_14_wheel_path=$(find dist/walltime-python-3-14 -name "*-cp314-*-musllinux*_aarch64.whl" -type f | head -n1) + wheel=$(find dist/benchmark-wheel -name "*-cp314-*-manylinux*_aarch64.whl" -type f | head -n1) + if [[ -z "$wheel" ]]; then + wheel=$(find dist/benchmark-wheel -name "*-cp314-*-musllinux*_aarch64.whl" -type f | head -n1) fi - if [[ -z "$walltime_python_3_14_wheel_path" ]]; then - echo "No wheel found for Python 3.14 on Linux ARM64" + if [[ -z "$wheel" ]]; then + echo "No compatible wheel found" exit 1 fi - echo "path=$walltime_python_3_14_wheel_path" >> "$GITHUB_OUTPUT" - - - name: Setup test environment for walltime Python 3.14 - run: | - uv venv --python 3.14 --clear - uv sync --extra test --no-install-project - uv pip install "${{ steps.walltime_python_3_14_wheel.outputs.path }}" - - - name: Test built walltime wheel for Python 3.14 - run: uv run --no-sync pytest tests/ -vvv + echo "path=$wheel" >> "$GITHUB_OUTPUT" - - name: Setup CodSpeed walltime environment for Python 3.14 + - name: Setup benchmark environment run: | uv venv --python 3.14 --clear uv sync --extra test --no-install-project + uv pip install "${{ steps.wheel.outputs.path }}" - - name: Run CodSpeed walltime benchmarks for Python 3.14 + - name: Run CodSpeed benchmarks uses: CodSpeedHQ/action@v4 + env: + CODSPEED_MEMORY: ${{ matrix.mode.id == 'memory' && '1' || '' }} with: - mode: walltime - run: uv run --with "${{ steps.walltime_python_3_14_wheel.outputs.path }}" --no-sync pytest tests/ --codspeed -k test_performance -v + mode: ${{ matrix.mode.id }} + run: > + uv run --no-sync pytest tests/ + --codspeed + -k "${{ matrix.shard.expr }}" + ${{ matrix.mode.id == 'walltime' && '--codspeed-warmup-time=0.2 --codspeed-max-time=1 --codspeed-max-rounds=20' || '' }} + -v \ No newline at end of file From 01287d33c8a8f40c82f5e24e20dd558ca9f80f25 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Tue, 24 Mar 2026 11:50:57 +0400 Subject: [PATCH 13/36] Update cd-build.yaml --- .github/workflows/cd-build.yaml | 54 ++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index fb64c0a..0d27e00 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -104,12 +104,10 @@ jobs: shell: bash run: | wheel=$(find dist -name "*-${{ matrix.python.tag }}-${{ matrix.build.wheel_arch }}.whl" -type f | head -n1) - if [[ -z "$wheel" ]]; then echo "No compatible wheel found" exit 1 fi - echo "path=$wheel" >> "$GITHUB_OUTPUT" - name: Setup test environment @@ -263,33 +261,34 @@ jobs: run: uv run --no-sync python -m unittest test.test_copy -v codspeed: - name: "CodSpeed ${{ matrix.mode.label }} - Python 3.14 - ARM64 - Shard ${{ matrix.shard.name }}" + name: "CodSpeed ${{ matrix.mode.label }} - Python 3.14 - ARM64 - ${{ matrix.shard.name }}" needs: wheel_python_3_14_linux_arm64 - runs-on: ${{ matrix.mode.runner }} + runs-on: ubuntu-24.04-arm strategy: fail-fast: false matrix: mode: - - { id: simulation, label: "CPU Simulation", runner: "ubuntu-24.04-arm", extra_env: "" } - - { id: memory, label: "Memory", runner: "ubuntu-24.04-arm", extra_env: "CODSPEED_MEMORY=1" } - - { id: walltime, label: "WallTime", runner: "codspeed-macro", extra_env: "" } + - { id: simulation, label: "CPU Simulation" } + - { id: memory, label: "Memory" } shard: - - { name: "core", expr: "test_performance and (test_memo or test_container or test_depth)" } - - { name: "mid", expr: "test_performance and (test_atomic or test_reduce or test_edge)" } - - { name: "real", expr: "test_performance and (test_real or test_real_dict_memo or test_real_stdlib_patched or test_real_stdlib)" } + - { name: "Core", expr: "test_memo or test_container or test_depth" } + - { name: "Mid", expr: "test_atomic or test_reduce or test_edge" } + - { name: "Real Data", expr: "test_real" } steps: - *checkout - *setup-uv - - name: Download benchmark wheel artifact + - &download-benchmark-wheel + name: Download benchmark wheel artifact uses: actions/download-artifact@v4 with: name: "Wheels-3.14-Linux-ARM64" path: dist/benchmark-wheel - - name: Find benchmark wheel + - &find-benchmark-wheel + name: Find benchmark wheel id: wheel shell: bash run: | @@ -303,7 +302,8 @@ jobs: fi echo "path=$wheel" >> "$GITHUB_OUTPUT" - - name: Setup benchmark environment + - &setup-benchmark-env + name: Setup benchmark environment run: | uv venv --python 3.14 --clear uv sync --extra test --no-install-project @@ -316,8 +316,32 @@ jobs: with: mode: ${{ matrix.mode.id }} run: > - uv run --no-sync pytest tests/ + uv run --no-sync pytest tests/test_performance.py --codspeed -k "${{ matrix.shard.expr }}" - ${{ matrix.mode.id == 'walltime' && '--codspeed-warmup-time=0.2 --codspeed-max-time=1 --codspeed-max-rounds=20' || '' }} + -v + + codspeed_walltime: + name: "CodSpeed WallTime - Python 3.14 - ARM64" + needs: wheel_python_3_14_linux_arm64 + runs-on: codspeed-macro + + steps: + - *checkout + - *setup-uv + - *download-benchmark-wheel + - *find-benchmark-wheel + - *setup-benchmark-env + + - name: Run CodSpeed walltime benchmarks + uses: CodSpeedHQ/action@v4 + with: + mode: walltime + run: > + uv run --no-sync pytest tests/test_performance.py + --codspeed + -k "test_performance" + --codspeed-warmup-time=0.2 + --codspeed-max-time=1 + --codspeed-max-rounds=20 -v \ No newline at end of file From 838c7037a4c3f1d79ac3b4ec69107524cee2d441 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Tue, 24 Mar 2026 11:54:05 +0400 Subject: [PATCH 14/36] Remove tzinfo from test data --- tests/test_performance.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_performance.py b/tests/test_performance.py index 47d1edf..323b601 100644 --- a/tests/test_performance.py +++ b/tests/test_performance.py @@ -22,7 +22,6 @@ import sys from dataclasses import dataclass from dataclasses import field -from datetime import UTC from datetime import datetime from datetime import timedelta from typing import Any @@ -284,7 +283,7 @@ def __deepcopy__(self, memo): ) + scaled( "datetime", - lambda n: [datetime(2024, 1, 1, tzinfo=UTC) + timedelta(days=i) for i in range(n)], + lambda n: [datetime(2024, 1, 1) + timedelta(days=i) for i in range(n)], # noqa: DTZ001 REDUCE_SIZES, ) + scaled( @@ -504,7 +503,7 @@ def make_orm_graph(): [ OrmSession( f"t{i}{j}", - datetime(2024, 1, 1 + j, tzinfo=UTC), + datetime(2024, 1, 1 + j), # noqa: DTZ001 {"ip": f"10.0.{i}.{j}"}, ) for j in range(3) From b623086442da24c28fc1f28d52835a7c97bdc398 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Tue, 24 Mar 2026 12:20:34 +0400 Subject: [PATCH 15/36] Bisect codspeed walltime --- .github/workflows/cd-build.yaml | 8 +- tools/bisect_codspeed_walltime.py | 339 ++++++++++++++++++++++++++++++ 2 files changed, 340 insertions(+), 7 deletions(-) create mode 100644 tools/bisect_codspeed_walltime.py diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index 0d27e00..d54e08c 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -338,10 +338,4 @@ jobs: with: mode: walltime run: > - uv run --no-sync pytest tests/test_performance.py - --codspeed - -k "test_performance" - --codspeed-warmup-time=0.2 - --codspeed-max-time=1 - --codspeed-max-rounds=20 - -v \ No newline at end of file + uv run --no-sync python tools/bisect_codspeed_walltime.py \ No newline at end of file diff --git a/tools/bisect_codspeed_walltime.py b/tools/bisect_codspeed_walltime.py new file mode 100644 index 0000000..5c6eeea --- /dev/null +++ b/tools/bisect_codspeed_walltime.py @@ -0,0 +1,339 @@ +from __future__ import annotations + +import json +import os +import shlex +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + +TEST_FILE = "tests/test_performance.py" +TEST_K = "test_performance" + +PYTEST_BASE = [ + "uv", + "run", + "--no-sync", + "pytest", + TEST_FILE, + "--codspeed", + "-k", + TEST_K, + "--codspeed-warmup-time=0", + "--codspeed-max-time=0.001", + "--codspeed-max-rounds=1", + "-q", + "-p", + "no:random-order", + "--override-ini=addopts=", +] + +CRASH_CODES = {139, -11} +CACHE_FILE = Path(".ci-codspeed-bisect-cache.json") +OUT_DIR = Path(".ci-codspeed-bisect") +OUT_DIR.mkdir(exist_ok=True) + + +@dataclass(frozen=True) +class RunResult: + code: int + crashed: bool + stdout_tail: str + stderr_tail: str + + +def sh(cmd: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + cmd, + text=True, + capture_output=True, + check=False, + env=os.environ.copy(), + ) + + +def collect_nodeids() -> list[str]: + cmd = [ + "uv", + "run", + "--no-sync", + "pytest", + TEST_FILE, + "-k", + TEST_K, + "--collect-only", + "-q", + "-p", + "no:random-order", + "--override-ini=addopts=", + ] + proc = sh(cmd) + if proc.returncode != 0: + print(proc.stdout) + print(proc.stderr, file=sys.stderr) + raise SystemExit(f"collect failed: exit={proc.returncode}") + + nodeids: list[str] = [] + for line in proc.stdout.splitlines(): + line = line.strip() + if line.startswith("tests/") and "::" in line: + nodeids.append(line) + + if not nodeids: + raise SystemExit("no nodeids collected") + + return nodeids + + +def load_cache() -> dict[str, dict]: + if CACHE_FILE.exists(): + return json.loads(CACHE_FILE.read_text()) + return {} + + +def save_cache(cache: dict[str, dict]) -> None: + CACHE_FILE.write_text(json.dumps(cache, indent=2, sort_keys=True)) + + +def key_for_subset(nodeids: Iterable[str]) -> str: + return "\n".join(nodeids) + + +def write_nodeids_file(path: Path, nodeids: list[str]) -> None: + path.write_text("".join(f"{x}\n" for x in nodeids)) + + +def run_subset(nodeids: list[str], *, label: str, use_cache: bool = True) -> RunResult: + cache = load_cache() + key = key_for_subset(nodeids) + if use_cache and key in cache: + entry = cache[key] + return RunResult( + code=entry["code"], + crashed=entry["crashed"], + stdout_tail=entry["stdout_tail"], + stderr_tail=entry["stderr_tail"], + ) + + listfile = OUT_DIR / f"{label.replace('/', '_').replace(' ', '_')}.txt" + write_nodeids_file(listfile, nodeids) + + cmd = PYTEST_BASE + [f"@{listfile}"] + + print(f"\n=== RUN {label} ===") + print(f"tests: {len(nodeids)}") + print("cmd:", shlex.join(cmd)) + + proc = sh(cmd) + + stdout_tail = "\n".join(proc.stdout.splitlines()[-40:]) + stderr_tail = "\n".join(proc.stderr.splitlines()[-40:]) + crashed = proc.returncode in CRASH_CODES + + result = RunResult( + code=proc.returncode, + crashed=crashed, + stdout_tail=stdout_tail, + stderr_tail=stderr_tail, + ) + + if use_cache: + cache[key] = { + "code": result.code, + "crashed": result.crashed, + "stdout_tail": result.stdout_tail, + "stderr_tail": result.stderr_tail, + } + save_cache(cache) + + print(f"exit={result.code} crashed={result.crashed}") + if result.stdout_tail: + print("--- stdout tail ---") + print(result.stdout_tail) + if result.stderr_tail: + print("--- stderr tail ---", file=sys.stderr) + print(result.stderr_tail, file=sys.stderr) + + return result + + +def partitions(xs: list[str], n: int) -> list[list[str]]: + size = len(xs) + out: list[list[str]] = [] + start = 0 + for i in range(n): + end = start + (size - start + (n - i) - 1) // (n - i) + out.append(xs[start:end]) + start = end + return [p for p in out if p] + + +def without(xs: list[str], ys: Iterable[str]) -> list[str]: + ys_set = set(ys) + return [x for x in xs if x not in ys_set] + + +def ddmin(nodeids: list[str]) -> list[str]: + current = nodeids[:] + n = 2 + + while len(current) >= 2: + chunks = partitions(current, n) + reduced = False + + for i, chunk in enumerate(chunks, 1): + res = run_subset(chunk, label=f"ddmin_subset_{len(current)}_{i}_of_{len(chunks)}") + if res.crashed: + current = chunk + n = 2 + reduced = True + print(f"reduced to crashing subset of {len(current)} tests") + break + if reduced: + continue + + for i, chunk in enumerate(chunks, 1): + comp = without(current, chunk) + if not comp: + continue + res = run_subset(comp, label=f"ddmin_complement_{len(current)}_{i}_of_{len(chunks)}") + if res.crashed: + current = comp + n = max(n - 1, 2) + reduced = True + print(f"reduced to crashing complement of {len(current)} tests") + break + if reduced: + continue + + if n >= len(current): + break + n = min(len(current), n * 2) + + return current + + +def find_last_passing_prefix(nodeids: list[str]) -> int: + lo = 0 + hi = len(nodeids) + while lo < hi: + mid = (lo + hi + 1) // 2 + res = run_subset(nodeids[:mid], label=f"prefix_0_{mid}") + if res.crashed: + hi = mid - 1 + else: + lo = mid + return lo + + +def verify_isolated_group(group: list[str], idx: int) -> None: + res = run_subset(group, label=f"verify_group_{idx}") + if not res.crashed: + raise SystemExit( + f"group {idx} is not independently crashing; " + "this means the crash depends on interaction with outside tests" + ) + + +def verify_remainder_passes(remainder: list[str]) -> None: + res = run_subset(remainder, label="verify_remainder") + if res.crashed: + raise SystemExit( + "suite still crashes after excluding all discovered failing groups; " + "partition is incomplete" + ) + + +def verify_each_group_against_clean_remainder(remainder: list[str], groups: list[list[str]]) -> None: + for i, group in enumerate(groups, 1): + combo = remainder + group + res = run_subset(combo, label=f"verify_remainder_plus_group_{i}") + if not res.crashed: + raise SystemExit( + f"clean remainder + failing group {i} did not crash; " + "group is not sufficient in the final partition" + ) + + +def main() -> int: + all_nodeids = collect_nodeids() + print(f"collected {len(all_nodeids)} nodeids") + + full = run_subset(all_nodeids, label="full_suite", use_cache=False) + if not full.crashed: + print("full suite did not reproduce crash") + return 2 + + last_ok = find_last_passing_prefix(all_nodeids) + print(f"last passing prefix length: {last_ok}") + if last_ok < len(all_nodeids): + print("first suspect after passing prefix:") + print(all_nodeids[last_ok]) + + remaining = all_nodeids[:] + failing_groups: list[list[str]] = [] + + iteration = 0 + while True: + iteration += 1 + res = run_subset(remaining, label=f"remaining_{iteration}", use_cache=False) + if not res.crashed: + break + + print(f"\n### extracting crashing group #{iteration} from {len(remaining)} tests") + group = ddmin(remaining) + + group_path = OUT_DIR / f"failing_group_{iteration}.txt" + write_nodeids_file(group_path, group) + print(f"wrote {group_path}") + + verify_isolated_group(group, iteration) + + new_remaining = without(remaining, group) + remaining_res = run_subset(new_remaining, label=f"remaining_after_group_{iteration}", use_cache=False) + + failing_groups.append(group) + remaining = new_remaining + + print( + f"group #{iteration}: {len(group)} tests; " + f"remaining: {len(remaining)}; " + f"remaining_crashes={remaining_res.crashed}" + ) + + if not remaining_res.crashed: + break + + passing_tests = remaining[:] + + print("\n=== FINAL PARTITION ===") + print(f"failing groups: {len(failing_groups)}") + for i, group in enumerate(failing_groups, 1): + print(f" group {i}: {len(group)} tests") + for t in group: + print(f" {t}") + print(f"passing remainder: {len(passing_tests)} tests") + + verify_remainder_passes(passing_tests) + verify_each_group_against_clean_remainder(passing_tests, failing_groups) + + write_nodeids_file(OUT_DIR / "passing_tests.txt", passing_tests) + for i, group in enumerate(failing_groups, 1): + write_nodeids_file(OUT_DIR / f"failing_group_{i}.txt", group) + + summary = { + "total_tests": len(all_nodeids), + "failing_group_count": len(failing_groups), + "failing_groups": failing_groups, + "passing_tests": passing_tests, + } + (OUT_DIR / "summary.json").write_text(json.dumps(summary, indent=2)) + + print("\npartition complete") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file From a0888c6e4ae34a9fea01d1e71eb852d3c1b42ef8 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Tue, 24 Mar 2026 12:20:49 +0400 Subject: [PATCH 16/36] Skip CI tests on PRs --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7792ccb..c264095 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,6 +17,7 @@ permissions: jobs: test: + if: github.event_name != 'pull_request' uses: ./.github/workflows/ci-test.yaml lint: From c1e01c81003bd50f1cfde910c3e362d1d7ba5816 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Tue, 24 Mar 2026 12:49:19 +0400 Subject: [PATCH 17/36] Try running walltime with minimal params --- .github/workflows/cd-build.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index d54e08c..61efa5a 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -338,4 +338,10 @@ jobs: with: mode: walltime run: > - uv run --no-sync python tools/bisect_codspeed_walltime.py \ No newline at end of file + uv run --no-sync pytest tests/test_performance.py + --codspeed + -k "test_performance" + --codspeed-warmup-time=0 + --codspeed-max-time=0.001 + --codspeed-max-rounds=1 + -v \ No newline at end of file From 277e4f2e64c88da00773ca70b847fd3a522ae0f7 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Tue, 24 Mar 2026 13:13:54 +0400 Subject: [PATCH 18/36] Add a small warmup and increase rounds to two --- .github/workflows/cd-build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index 61efa5a..d1164bd 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -341,7 +341,7 @@ jobs: uv run --no-sync pytest tests/test_performance.py --codspeed -k "test_performance" - --codspeed-warmup-time=0 + --codspeed-warmup-time=0.001 --codspeed-max-time=0.001 - --codspeed-max-rounds=1 + --codspeed-max-rounds=2 -v \ No newline at end of file From a1dee19b743bc0c31d81b8b0635d957a9a9dbc2f Mon Sep 17 00:00:00 2001 From: Bobronium Date: Tue, 24 Mar 2026 13:19:49 +0400 Subject: [PATCH 19/36] Remove custom parameters for the walltime --- .github/workflows/cd-build.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index d1164bd..496b77c 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -341,7 +341,4 @@ jobs: uv run --no-sync pytest tests/test_performance.py --codspeed -k "test_performance" - --codspeed-warmup-time=0.001 - --codspeed-max-time=0.001 - --codspeed-max-rounds=2 -v \ No newline at end of file From 4f935b97a82f193d8b10d67e81a8ff3a845950e6 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Tue, 24 Mar 2026 13:42:12 +0400 Subject: [PATCH 20/36] Use max time of 1 and warmup time of 0.1 --- .github/workflows/cd-build.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index 496b77c..41fd4ab 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -341,4 +341,6 @@ jobs: uv run --no-sync pytest tests/test_performance.py --codspeed -k "test_performance" + --codspeed-warmup-time=0.1 + --codspeed-max-time=1 -v \ No newline at end of file From 343c0477f264277f02417e2120b1c5c3e2dbef87 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Tue, 24 Mar 2026 15:34:04 +0400 Subject: [PATCH 21/36] Use max time 3 --- .github/workflows/cd-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index 41fd4ab..b9a3771 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -342,5 +342,5 @@ jobs: --codspeed -k "test_performance" --codspeed-warmup-time=0.1 - --codspeed-max-time=1 + --codspeed-max-time=3 -v \ No newline at end of file From dcb2f6cfdd57ccb39dc3d3999e76c4efe22b9ee6 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Tue, 24 Mar 2026 15:48:11 +0400 Subject: [PATCH 22/36] Use max time 2 --- .github/workflows/cd-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index b9a3771..fedf2ea 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -342,5 +342,5 @@ jobs: --codspeed -k "test_performance" --codspeed-warmup-time=0.1 - --codspeed-max-time=3 + --codspeed-max-time=2 -v \ No newline at end of file From 177bb3d21bc4f88788ccd9a1981ac99734a06ccd Mon Sep 17 00:00:00 2001 From: Bobronium Date: Tue, 24 Mar 2026 21:19:27 +0400 Subject: [PATCH 23/36] Give benchmarks a headstart --- .github/workflows/cd-build.yaml | 291 +++++++++++++++++--------------- 1 file changed, 154 insertions(+), 137 deletions(-) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index fedf2ea..3c0fd1f 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -8,28 +8,15 @@ env: CARGO_TERM_COLOR: always jobs: - wheels: + leading_wheel: strategy: fail-fast: false - max-parallel: 64 + max-parallel: 1 matrix: build: - - { os: ubuntu-latest, rust_target: x86_64-unknown-linux-gnu, manylinux: auto, wheel_arch: "manylinux*_x86_64", arch_label: "x86_64", platform_label: "Linux" } - - { os: ubuntu-24.04-arm, rust_target: aarch64-unknown-linux-gnu, manylinux: auto, wheel_arch: "manylinux*_aarch64", arch_label: "ARM64", platform_label: "Linux" } - - { os: windows-latest, rust_target: x86_64-pc-windows-msvc, wheel_arch: "win_amd64", arch_label: "x64", platform_label: "Windows" } - - { os: windows-11-arm, rust_target: aarch64-pc-windows-msvc, wheel_arch: "win_arm64", arch_label: "ARM64", platform_label: "Windows" } - - { os: macos-15, rust_target: aarch64-apple-darwin, wheel_arch: "macosx*_arm64", arch_label: "Apple Silicon", platform_label: "macOS" } - - { os: macos-15-intel, rust_target: x86_64-apple-darwin, wheel_arch: "macosx*_x86_64", arch_label: "Intel", platform_label: "macOS" } + - { os: ubuntu-24.04-arm, rust_target: aarch64-unknown-linux-gnu, manylinux: auto, wheel_arch: "manylinux*_aarch64", arch_label: "ARM64", platform_label: "Linux" } python: - - { tag: "cp310", version: "3.10.11", label: "3.10" } - - { tag: "cp311", version: "3.11.9", label: "3.11" } - - { tag: "cp312", version: "3.12.10", label: "3.12" } - - { tag: "cp313", version: "3.13.9", label: "3.13" } - - { tag: "cp314", version: "3.14.0", label: "3.14" } - - { tag: "cp314t", version: "3.14t", label: "3.14t", branch: "3.14" } - exclude: - - build: { os: ubuntu-24.04-arm, rust_target: aarch64-unknown-linux-gnu, manylinux: auto, wheel_arch: "manylinux*_aarch64", arch_label: "ARM64", platform_label: "Linux" } - python: { tag: "cp314", version: "3.14.0", label: "3.14" } + - { tag: "cp314", version: "3.14.0", label: "3.14" } runs-on: ${{ matrix.build.os }} name: "Python ${{ matrix.python.label }} - ${{ matrix.build.platform_label }} - ${{ matrix.build.arch_label }}" @@ -42,23 +29,14 @@ jobs: fetch-depth: 0 fetch-tags: true - - name: Compute job parameters - id: meta - shell: bash - run: | - skip_tests=false - if [[ "${{ matrix.build.os }}" == "windows-11-arm" && "${{ matrix.python.label }}" =~ ^3\.(10|11|12)$ ]]; then - skip_tests=true - fi - echo "skip_tests=$skip_tests" >> "$GITHUB_OUTPUT" - - &setup-uv name: Setup uv uses: astral-sh/setup-uv@v6 with: enable-cache: true - - name: Install Python (non-Linux) + - &install-python-non-linux + name: Install Python (non-Linux) if: runner.os != 'Linux' shell: bash run: | @@ -74,7 +52,8 @@ jobs: echo "PYTHON=$(uv python find ${{ matrix.python.version }})" >> "$GITHUB_ENV" fi - - name: Build wheel + - &build-wheel + name: Build wheel uses: PyO3/maturin-action@v1 with: target: ${{ matrix.build.rust_target }} @@ -92,61 +71,101 @@ jobs: run: | echo "RUSTC_WRAPPER=" >> "$GITHUB_ENV" - - name: Upload wheel artifact + - &upload-wheel-artifacts + name: Upload wheel artifact uses: actions/upload-artifact@v4 with: name: "Wheels-${{ matrix.python.label }}-${{ matrix.build.platform_label }}-${{ matrix.build.arch_label }}" path: dist/*.whl - - name: Find built wheel - if: steps.meta.outputs.skip_tests != 'true' + - *checkout + - *setup-uv + - *build-wheel + - *clear-sccache-linux + - *upload-wheel-artifacts + + leading_tests: + needs: leading_wheel + strategy: + fail-fast: false + max-parallel: 1 + matrix: + build: + - { os: ubuntu-24.04-arm, wheel_arch: "manylinux*_aarch64", arch_label: "ARM64", platform_label: "Linux" } + python: + - { tag: "cp314", version: "3.14.0", label: "3.14", branch: "3.14" } + + runs-on: ${{ matrix.build.os }} + name: "Tests - Python ${{ matrix.python.label }} - ${{ matrix.build.platform_label }} - ${{ matrix.build.arch_label }}" + + steps: + - &download-test-wheel + name: Download wheel artifact + uses: actions/download-artifact@v4 + with: + name: "Wheels-${{ matrix.python.label }}-${{ matrix.build.platform_label }}-${{ matrix.build.arch_label }}" + path: dist/test-wheel + + - &find-test-wheel + name: Find built wheel id: wheel shell: bash run: | - wheel=$(find dist -name "*-${{ matrix.python.tag }}-${{ matrix.build.wheel_arch }}.whl" -type f | head -n1) + wheel=$(find dist/test-wheel -name "*-${{ matrix.python.tag }}-${{ matrix.build.wheel_arch }}.whl" -type f | head -n1) + if [[ -z "$wheel" && "${{ matrix.build.platform_label }}" == "Linux" ]]; then + case "${{ matrix.build.arch_label }}" in + "x86_64") + wheel=$(find dist/test-wheel -name "*-${{ matrix.python.tag }}-*-musllinux*_x86_64.whl" -type f | head -n1) + ;; + "ARM64") + wheel=$(find dist/test-wheel -name "*-${{ matrix.python.tag }}-*-musllinux*_aarch64.whl" -type f | head -n1) + ;; + esac + fi if [[ -z "$wheel" ]]; then echo "No compatible wheel found" exit 1 fi echo "path=$wheel" >> "$GITHUB_OUTPUT" - - name: Setup test environment - if: steps.meta.outputs.skip_tests != 'true' + - &setup-test-env + name: Setup test environment run: | - uv venv --python ${{ matrix.python.label }} --clear + uv venv --python ${{ env.PYTHON || matrix.python.label }} --clear uv sync --extra test --no-install-project uv pip install "${{ steps.wheel.outputs.path }}" - - name: Test built wheel - if: steps.meta.outputs.skip_tests != 'true' + - &test-built-wheel + name: Test built wheel run: uv run --no-sync pytest tests/ -vvv - - name: Test shuffle A - if: steps.meta.outputs.skip_tests != 'true' + - &test-shuffle-a + name: Test shuffle A run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=A - - name: Test shuffle B - if: steps.meta.outputs.skip_tests != 'true' + - &test-shuffle-b + name: Test shuffle B run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=B - - name: Test shuffle C - if: steps.meta.outputs.skip_tests != 'true' + - &test-shuffle-c + name: Test shuffle C run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=C - - name: Test shuffle D - if: steps.meta.outputs.skip_tests != 'true' + - &test-shuffle-d + name: Test shuffle D run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=D - - name: Cache CPython test suite - if: steps.meta.outputs.skip_tests != 'true' + - &cache-cpython-tests + name: Cache CPython test suite id: cpython-cache uses: actions/cache@v4 with: path: cpython-tests key: cpython-tests-${{ matrix.python.branch || matrix.python.label }} - - name: Checkout CPython test suite - if: steps.meta.outputs.skip_tests != 'true' && steps.cpython-cache.outputs.cache-hit != 'true' + - &checkout-cpython-tests + name: Checkout CPython test suite + if: steps.cpython-cache.outputs.cache-hit != 'true' uses: actions/checkout@v5 with: repository: python/cpython @@ -155,110 +174,108 @@ jobs: sparse-checkout-cone-mode: true path: cpython-tests - - name: Test CPython test_copy (patched) - if: steps.meta.outputs.skip_tests != 'true' + - &test-cpython-copy-patched + name: Test CPython test_copy (patched) env: COPIUM_PATCH_ENABLE: "1" PYTHONPATH: ${{ github.workspace }}/cpython-tests/Lib run: uv run --no-sync python -m unittest test.test_copy -v - - name: Test CPython test_copy (patched, dict memo) - if: steps.meta.outputs.skip_tests != 'true' + - &test-cpython-copy-patched-dict-memo + name: Test CPython test_copy (patched, dict memo) env: COPIUM_PATCH_ENABLE: "1" COPIUM_USE_DICT_MEMO: "1" PYTHONPATH: ${{ github.workspace }}/cpython-tests/Lib run: uv run --no-sync python -m unittest test.test_copy -v - wheel_python_3_14_linux_arm64: - runs-on: ubuntu-24.04-arm - name: "Python 3.14 - Linux - ARM64" + wheels: + needs: + - leading_tests + if: ${{ always() }} + strategy: + fail-fast: false + max-parallel: 64 + matrix: + build: + - { os: ubuntu-latest, rust_target: x86_64-unknown-linux-gnu, manylinux: auto, wheel_arch: "manylinux*_x86_64", arch_label: "x86_64", platform_label: "Linux" } + - { os: ubuntu-24.04-arm, rust_target: aarch64-unknown-linux-gnu, manylinux: auto, wheel_arch: "manylinux*_aarch64", arch_label: "ARM64", platform_label: "Linux" } + - { os: windows-latest, rust_target: x86_64-pc-windows-msvc, wheel_arch: "win_amd64", arch_label: "x64", platform_label: "Windows" } + - { os: windows-11-arm, rust_target: aarch64-pc-windows-msvc, wheel_arch: "win_arm64", arch_label: "ARM64", platform_label: "Windows" } + - { os: macos-15, rust_target: aarch64-apple-darwin, wheel_arch: "macosx*_arm64", arch_label: "Apple Silicon", platform_label: "macOS" } + - { os: macos-15-intel, rust_target: x86_64-apple-darwin, wheel_arch: "macosx*_x86_64", arch_label: "Intel", platform_label: "macOS" } + python: + - { tag: "cp310", version: "3.10.11", label: "3.10" } + - { tag: "cp311", version: "3.11.9", label: "3.11" } + - { tag: "cp312", version: "3.12.10", label: "3.12" } + - { tag: "cp313", version: "3.13.9", label: "3.13" } + - { tag: "cp314", version: "3.14.0", label: "3.14" } + - { tag: "cp314t", version: "3.14t", label: "3.14t", branch: "3.14" } + exclude: + - build: { os: ubuntu-24.04-arm, rust_target: aarch64-unknown-linux-gnu, manylinux: auto, wheel_arch: "manylinux*_aarch64", arch_label: "ARM64", platform_label: "Linux" } + python: { tag: "cp314", version: "3.14.0", label: "3.14" } + + runs-on: ${{ matrix.build.os }} + name: "Python ${{ matrix.python.label }} - ${{ matrix.build.platform_label }} - ${{ matrix.build.arch_label }}" steps: - *checkout - *setup-uv - - - name: Build wheel - uses: PyO3/maturin-action@v1 - with: - target: aarch64-unknown-linux-gnu - manylinux: auto - args: --release --out dist -i 3.14 - rust-toolchain: nightly - sccache: "true" - + - *install-python-non-linux + - *build-wheel - *clear-sccache-linux + - *upload-wheel-artifacts - - name: Upload wheel artifact - uses: actions/upload-artifact@v4 - with: - name: "Wheels-3.14-Linux-ARM64" - path: dist/*.whl - - - name: Find built wheel - id: wheel - shell: bash - run: | - wheel=$(find dist -name "*-cp314-*-manylinux*_aarch64.whl" -type f | head -n1) - if [[ -z "$wheel" ]]; then - wheel=$(find dist -name "*-cp314-*-musllinux*_aarch64.whl" -type f | head -n1) - fi - if [[ -z "$wheel" ]]; then - echo "No compatible wheel found" - exit 1 - fi - echo "path=$wheel" >> "$GITHUB_OUTPUT" - - - name: Setup test environment - run: | - uv venv --python 3.14 --clear - uv sync --extra test --no-install-project - uv pip install "${{ steps.wheel.outputs.path }}" - - - name: Test built wheel - run: uv run --no-sync pytest tests/ -vvv - - - name: Test shuffle A - run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=A - - - name: Test shuffle B - run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=B - - - name: Test shuffle C - run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=C - - - name: Test shuffle D - run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=D - - - name: Cache CPython test suite - id: cpython-cache - uses: actions/cache@v4 - with: - path: cpython-tests - key: cpython-tests-3.14 - - - name: Checkout CPython test suite - if: steps.cpython-cache.outputs.cache-hit != 'true' - uses: actions/checkout@v5 - with: - repository: python/cpython - ref: 3.14 - sparse-checkout: Lib/test - sparse-checkout-cone-mode: true - path: cpython-tests + tests: + needs: + - wheels + strategy: + fail-fast: false + max-parallel: 64 + matrix: + build: + - { os: ubuntu-latest, wheel_arch: "manylinux*_x86_64", arch_label: "x86_64", platform_label: "Linux" } + - { os: ubuntu-24.04-arm, wheel_arch: "manylinux*_aarch64", arch_label: "ARM64", platform_label: "Linux" } + - { os: windows-latest, wheel_arch: "win_amd64", arch_label: "x64", platform_label: "Windows" } + - { os: windows-11-arm, wheel_arch: "win_arm64", arch_label: "ARM64", platform_label: "Windows" } + - { os: macos-15, wheel_arch: "macosx*_arm64", arch_label: "Apple Silicon", platform_label: "macOS" } + - { os: macos-15-intel, wheel_arch: "macosx*_x86_64", arch_label: "Intel", platform_label: "macOS" } + python: + - { tag: "cp310", version: "3.10.11", label: "3.10" } + - { tag: "cp311", version: "3.11.9", label: "3.11" } + - { tag: "cp312", version: "3.12.10", label: "3.12" } + - { tag: "cp313", version: "3.13.9", label: "3.13" } + - { tag: "cp314", version: "3.14.0", label: "3.14" } + - { tag: "cp314t", version: "3.14t", label: "3.14t", branch: "3.14" } + exclude: + - build: { os: ubuntu-24.04-arm, wheel_arch: "manylinux*_aarch64", arch_label: "ARM64", platform_label: "Linux" } + python: { tag: "cp314", version: "3.14.0", label: "3.14" } + - build: { os: windows-11-arm, wheel_arch: "win_arm64", arch_label: "ARM64", platform_label: "Windows" } + python: { tag: "cp310", version: "3.10.11", label: "3.10" } + - build: { os: windows-11-arm, wheel_arch: "win_arm64", arch_label: "ARM64", platform_label: "Windows" } + python: { tag: "cp311", version: "3.11.9", label: "3.11" } + - build: { os: windows-11-arm, wheel_arch: "win_arm64", arch_label: "ARM64", platform_label: "Windows" } + python: { tag: "cp312", version: "3.12.10", label: "3.12" } - - name: Test CPython test_copy (patched) - env: - COPIUM_PATCH_ENABLE: "1" - PYTHONPATH: ${{ github.workspace }}/cpython-tests/Lib - run: uv run --no-sync python -m unittest test.test_copy -v + runs-on: ${{ matrix.build.os }} + name: "Tests - Python ${{ matrix.python.label }} - ${{ matrix.build.platform_label }} - ${{ matrix.build.arch_label }}" - - name: Test CPython test_copy (patched, dict memo) - env: - COPIUM_PATCH_ENABLE: "1" - COPIUM_USE_DICT_MEMO: "1" - PYTHONPATH: ${{ github.workspace }}/cpython-tests/Lib - run: uv run --no-sync python -m unittest test.test_copy -v + steps: + - *checkout + - *setup-uv + - *install-python-non-linux + - *download-test-wheel + - *find-test-wheel + - *setup-test-env + - *test-built-wheel + - *test-shuffle-a + - *test-shuffle-b + - *test-shuffle-c + - *test-shuffle-d + - *cache-cpython-tests + - *checkout-cpython-tests + - *test-cpython-copy-patched + - *test-cpython-copy-patched-dict-memo codspeed: name: "CodSpeed ${{ matrix.mode.label }} - Python 3.14 - ARM64 - ${{ matrix.shard.name }}" @@ -273,7 +290,7 @@ jobs: - { id: memory, label: "Memory" } shard: - { name: "Core", expr: "test_memo or test_container or test_depth" } - - { name: "Mid", expr: "test_atomic or test_reduce or test_edge" } + - { name: "Mid", expr: "test_atomic or test_reduce or test_edge" } - { name: "Real Data", expr: "test_real" } steps: @@ -323,7 +340,7 @@ jobs: codspeed_walltime: name: "CodSpeed WallTime - Python 3.14 - ARM64" - needs: wheel_python_3_14_linux_arm64 + needs: leading_wheel runs-on: codspeed-macro steps: From 7430c2dba08fedd4b426cdda4813a432d857c750 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Tue, 24 Mar 2026 21:21:28 +0400 Subject: [PATCH 24/36] Fix broken dependency --- .github/workflows/cd-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index 3c0fd1f..9927fa2 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -279,7 +279,7 @@ jobs: codspeed: name: "CodSpeed ${{ matrix.mode.label }} - Python 3.14 - ARM64 - ${{ matrix.shard.name }}" - needs: wheel_python_3_14_linux_arm64 + needs: leading_wheel runs-on: ubuntu-24.04-arm strategy: From b185feb7a7670466bfecadc5c2c46f8454a98173 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Tue, 24 Mar 2026 21:28:05 +0400 Subject: [PATCH 25/36] Remove duplicated steps --- .github/workflows/cd-build.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index 9927fa2..8e0d190 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -78,12 +78,6 @@ jobs: name: "Wheels-${{ matrix.python.label }}-${{ matrix.build.platform_label }}-${{ matrix.build.arch_label }}" path: dist/*.whl - - *checkout - - *setup-uv - - *build-wheel - - *clear-sccache-linux - - *upload-wheel-artifacts - leading_tests: needs: leading_wheel strategy: From a3fa1e0179369b0b589b2b62bff9b98ab7b10293 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Tue, 24 Mar 2026 21:31:30 +0400 Subject: [PATCH 26/36] Add missing steps to leading_tests --- .github/workflows/cd-build.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index 8e0d190..ea9a3ee 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -77,7 +77,6 @@ jobs: with: name: "Wheels-${{ matrix.python.label }}-${{ matrix.build.platform_label }}-${{ matrix.build.arch_label }}" path: dist/*.whl - leading_tests: needs: leading_wheel strategy: @@ -93,6 +92,8 @@ jobs: name: "Tests - Python ${{ matrix.python.label }} - ${{ matrix.build.platform_label }} - ${{ matrix.build.arch_label }}" steps: + - *checkout + - *setup-uv - &download-test-wheel name: Download wheel artifact uses: actions/download-artifact@v4 From b8760a3ea9f24a80264748083beec37419d32df8 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Tue, 24 Mar 2026 21:31:56 +0400 Subject: [PATCH 27/36] Rename leading_wheel -> leading_wheels --- .github/workflows/cd-build.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index ea9a3ee..139bb50 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -8,7 +8,7 @@ env: CARGO_TERM_COLOR: always jobs: - leading_wheel: + leading_wheels: strategy: fail-fast: false max-parallel: 1 @@ -274,7 +274,7 @@ jobs: codspeed: name: "CodSpeed ${{ matrix.mode.label }} - Python 3.14 - ARM64 - ${{ matrix.shard.name }}" - needs: leading_wheel + needs: leading_wheels runs-on: ubuntu-24.04-arm strategy: @@ -335,7 +335,7 @@ jobs: codspeed_walltime: name: "CodSpeed WallTime - Python 3.14 - ARM64" - needs: leading_wheel + needs: leading_wheels runs-on: codspeed-macro steps: From e2f2150445c467576f5146bd8a445b5bb3cad460 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Tue, 24 Mar 2026 22:54:42 +0400 Subject: [PATCH 28/36] Allow leading tests to fail, but tests to run --- .github/workflows/cd-build.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index 139bb50..8acc7c3 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -224,6 +224,7 @@ jobs: tests: needs: - wheels + if: ${{ always() && needs.wheels.result == 'success' }} strategy: fail-fast: false max-parallel: 64 From c095216b9990e7dccf20fd0638b54e038b43e369 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Wed, 25 Mar 2026 00:54:07 +0400 Subject: [PATCH 29/36] Fix leading wheels dependency --- .github/workflows/cd-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index 8acc7c3..08a680e 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -78,7 +78,7 @@ jobs: name: "Wheels-${{ matrix.python.label }}-${{ matrix.build.platform_label }}-${{ matrix.build.arch_label }}" path: dist/*.whl leading_tests: - needs: leading_wheel + needs: leading_wheels strategy: fail-fast: false max-parallel: 1 From 3b5ec2de93f61fd7c5e79fbf35293c5f77625849 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Wed, 25 Mar 2026 01:22:42 +0400 Subject: [PATCH 30/36] Run sumulatiob & memory on x86 --- .github/workflows/cd-build.yaml | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index 08a680e..da5efb8 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -15,6 +15,7 @@ jobs: matrix: build: - { os: ubuntu-24.04-arm, rust_target: aarch64-unknown-linux-gnu, manylinux: auto, wheel_arch: "manylinux*_aarch64", arch_label: "ARM64", platform_label: "Linux" } + - { os: ubuntu-latest, rust_target: x86_64-unknown-linux-gnu, manylinux: auto, wheel_arch: "manylinux*_x86_64", arch_label: "x86_64", platform_label: "Linux" } python: - { tag: "cp314", version: "3.14.0", label: "3.14" } @@ -209,6 +210,8 @@ jobs: exclude: - build: { os: ubuntu-24.04-arm, rust_target: aarch64-unknown-linux-gnu, manylinux: auto, wheel_arch: "manylinux*_aarch64", arch_label: "ARM64", platform_label: "Linux" } python: { tag: "cp314", version: "3.14.0", label: "3.14" } + - build: { os: ubuntu-latest, rust_target: x86_64-unknown-linux-gnu, manylinux: auto, wheel_arch: "manylinux*_x86_64", arch_label: "x86_64", platform_label: "Linux" } + python: { tag: "cp314", version: "3.14.0", label: "3.14" } runs-on: ${{ matrix.build.os }} name: "Python ${{ matrix.python.label }} - ${{ matrix.build.platform_label }} - ${{ matrix.build.arch_label }}" @@ -274,9 +277,13 @@ jobs: - *test-cpython-copy-patched-dict-memo codspeed: - name: "CodSpeed ${{ matrix.mode.label }} - Python 3.14 - ARM64 - ${{ matrix.shard.name }}" + name: "CodSpeed ${{ matrix.mode.label }} - Python 3.14 - ${{ matrix.shard.name }}" needs: leading_wheels - runs-on: ubuntu-24.04-arm + runs-on: ubuntu-latest + env: + BENCH_ARTIFACT: "Wheels-3.14-Linux-x86_64" + BENCH_WHL_GLOB: "*-cp314-*-manylinux*_x86_64.whl" + BENCH_WHL_FALLBACK: "*-cp314-*-musllinux*_x86_64.whl" strategy: fail-fast: false @@ -297,7 +304,7 @@ jobs: name: Download benchmark wheel artifact uses: actions/download-artifact@v4 with: - name: "Wheels-3.14-Linux-ARM64" + name: ${{ env.BENCH_ARTIFACT }} path: dist/benchmark-wheel - &find-benchmark-wheel @@ -305,15 +312,10 @@ jobs: id: wheel shell: bash run: | - wheel=$(find dist/benchmark-wheel -name "*-cp314-*-manylinux*_aarch64.whl" -type f | head -n1) - if [[ -z "$wheel" ]]; then - wheel=$(find dist/benchmark-wheel -name "*-cp314-*-musllinux*_aarch64.whl" -type f | head -n1) - fi + wheel=$(find dist/benchmark-wheel -name "${{ env.BENCH_WHL_GLOB }}" -type f | head -n1) if [[ -z "$wheel" ]]; then - echo "No compatible wheel found" - exit 1 + wheel=$(find dist/benchmark-wheel -name "${{ env.BENCH_WHL_FALLBACK }}" -type f | head -n1) fi - echo "path=$wheel" >> "$GITHUB_OUTPUT" - &setup-benchmark-env name: Setup benchmark environment @@ -338,6 +340,10 @@ jobs: name: "CodSpeed WallTime - Python 3.14 - ARM64" needs: leading_wheels runs-on: codspeed-macro + env: + BENCH_ARTIFACT: "Wheels-3.14-Linux-ARM64" + BENCH_WHL_GLOB: "*-cp314-*-manylinux*_aarch64.whl" + BENCH_WHL_FALLBACK: "*-cp314-*-musllinux*_aarch64.whl" steps: - *checkout From 19a48f43c2d7ff4219c7419c2e93b00c1c6ab664 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Wed, 25 Mar 2026 01:29:52 +0400 Subject: [PATCH 31/36] Fix missing wheel for benchmarks --- .github/workflows/cd-build.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index da5efb8..dcc6bb4 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -316,6 +316,11 @@ jobs: if [[ -z "$wheel" ]]; then wheel=$(find dist/benchmark-wheel -name "${{ env.BENCH_WHL_FALLBACK }}" -type f | head -n1) fi + if [[ -z "$wheel" ]]; then + echo "No compatible wheel found" + exit 1 + fi + echo "path=$wheel" >> "$GITHUB_OUTPUT" - &setup-benchmark-env name: Setup benchmark environment From 3a38613667a236f1a03b576278636c00339d6a71 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Wed, 25 Mar 2026 07:13:01 +0400 Subject: [PATCH 32/36] Don't run memory benchmarks for small cases --- tests/test_performance.py | 95 +++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 44 deletions(-) diff --git a/tests/test_performance.py b/tests/test_performance.py index 323b601..bab0cf3 100644 --- a/tests/test_performance.py +++ b/tests/test_performance.py @@ -17,6 +17,7 @@ """ import copy as stdlib_copy +import os import platform import re import sys @@ -24,6 +25,7 @@ from dataclasses import field from datetime import datetime from datetime import timedelta +from itertools import chain from typing import Any from typing import NamedTuple @@ -36,18 +38,25 @@ class Case(NamedTuple): name: str obj: Any + memory: bool = True def scaled(tag, factory, sizes): - return [Case(f"{tag}-n-{n}", factory(n)) for n in sizes] + return (Case(f"{tag}-n-{n}", factory(n), memory=n >= 1000) for n in sizes) def depth_scaled(tag, factory, depths): - return [Case(f"{tag}-d-{d}", factory(d)) for d in depths] + return (Case(f"{tag}-d-{d}", factory(d), memory=d >= 100) for d in depths) + + +CODSPEED_MEMORY = bool(os.getenv("CODSPEED_MEMORY")) def generate_params(cases): - return pytest.mark.parametrize("case", [pytest.param(c, id=c.name) for c in cases]) + return pytest.mark.parametrize( + "case", + (pytest.param(c, id=c.name) for c in cases if not CODSPEED_MEMORY or c.memory), + ) python_version = ".".join(map(str, sys.version_info[:2])) @@ -113,17 +122,16 @@ def memo_unique_mut(n): return {"a": ([], [], []), "b": [[] for _ in range(n)]} -MEMO_CASES = ( - scaled("shared_mut", memo_shared_mut, SIZES) - + scaled("shared_deep", memo_shared_deep, SIZES) - + scaled("shared_tuple_atom", memo_shared_tuple_atom, SIZES) - + scaled("shared_tuple_mut", memo_shared_tuple_mut, SIZES) - + scaled("shared_atom", memo_shared_atom, SIZES) - + scaled("unique_atom", memo_unique_atom, SIZES) - + scaled("unique_mut", memo_unique_mut, SIZES) +MEMO_CASES = chain( + scaled("shared_mut", memo_shared_mut, SIZES), + scaled("shared_deep", memo_shared_deep, SIZES), + scaled("shared_tuple_atom", memo_shared_tuple_atom, SIZES), + scaled("shared_tuple_mut", memo_shared_tuple_mut, SIZES), + scaled("shared_atom", memo_shared_atom, SIZES), + scaled("unique_atom", memo_unique_atom, SIZES), + scaled("unique_mut", memo_unique_mut, SIZES), ) - # ═══════════════════════════════════════════════════════════ # CONTAINER TRAVERSAL # @@ -131,13 +139,13 @@ def memo_unique_mut(n): # Isolates per-container creation + traversal cost. # ═══════════════════════════════════════════════════════════ -CONTAINER_CASES = ( - scaled("list", lambda n: list(range(n)), SIZES) - + scaled("tuple", lambda n: tuple(range(n)), SIZES) - + scaled("dict", lambda n: {i: i for i in range(n)}, SIZES) - + scaled("set", lambda n: set(range(n)), SIZES) - + scaled("frozenset", lambda n: frozenset(range(n)), SIZES) - + scaled("bytearray", lambda n: bytearray(n), (100, 10_000, 1_000_000)) +CONTAINER_CASES = chain( + scaled("list", lambda n: list(range(n)), SIZES), + scaled("tuple", lambda n: tuple(range(n)), SIZES), + scaled("dict", lambda n: {i: i for i in range(n)}, SIZES), + scaled("set", lambda n: set(range(n)), SIZES), + scaled("frozenset", lambda n: frozenset(range(n)), SIZES), + scaled("bytearray", lambda n: bytearray(n), (100, 10_000, 1_000_000)), ) @@ -178,14 +186,13 @@ def nested_tuple_atom(d): return obj -DEPTH_CASES = ( - depth_scaled("list", nested_list, DEPTHS) - + depth_scaled("dict", nested_dict, DEPTHS) - + depth_scaled("tuple_mut", nested_tuple_mut, DEPTHS) - + depth_scaled("tuple_atom", nested_tuple_atom, DEPTHS) +DEPTH_CASES = chain( + depth_scaled("list", nested_list, DEPTHS), + depth_scaled("dict", nested_dict, DEPTHS), + depth_scaled("tuple_mut", nested_tuple_mut, DEPTHS), + depth_scaled("tuple_atom", nested_tuple_atom, DEPTHS), ) - # ═══════════════════════════════════════════════════════════ # ATOMIC FAST PATH # @@ -206,13 +213,13 @@ def mixed_prememo_atoms(n): return [pool[i % 6] for i in range(n)] -ATOMIC_CASES = ( - scaled("none", lambda n: [None] * n, ATOM_SIZES) - + scaled("int", lambda n: list(range(n)), ATOM_SIZES) - + scaled("str", lambda n: [f"s{i}" for i in range(n)], ATOM_SIZES) - + scaled("mixed_builtin_atomics", mixed_prememo_atoms, ATOM_SIZES) - + scaled("re.Pattern", lambda n: [CACHED_RE] * n, ATOM_SIZES) - + scaled("type", lambda n: [int] * n, ATOM_SIZES) +ATOMIC_CASES = chain( + scaled("none", lambda n: [None] * n, ATOM_SIZES), + scaled("int", lambda n: list(range(n)), ATOM_SIZES), + scaled("str", lambda n: [f"s{i}" for i in range(n)], ATOM_SIZES), + scaled("mixed_builtin_atomics", mixed_prememo_atoms, ATOM_SIZES), + scaled("re.Pattern", lambda n: [CACHED_RE] * n, ATOM_SIZES), + scaled("type", lambda n: [int] * n, ATOM_SIZES), ) @@ -260,37 +267,37 @@ def __deepcopy__(self, memo): return CustomDeepcopyObject(stdlib_copy.deepcopy(self.v, memo)) -REDUCE_CASES = ( +REDUCE_CASES = chain( scaled( "dataclass_simple", lambda n: [SimpleDataclass(i, f"v{i}") for i in range(n)], REDUCE_SIZES, - ) - + scaled( + ), + scaled( "dataclass_mutable", lambda n: [MutableDataclass(i, [i], {"k": i}) for i in range(n)], REDUCE_SIZES, - ) - + scaled( + ), + scaled( "dataclass_nested", lambda n: [NestedDataclass(SimpleDataclass(i, f"v{i}"), [i]) for i in range(n)], REDUCE_SIZES, - ) - + scaled( + ), + scaled( "slots", lambda n: [SlotsObject(i, f"v{i}", float(i)) for i in range(n)], REDUCE_SIZES, - ) - + scaled( + ), + scaled( "datetime", lambda n: [datetime(2024, 1, 1) + timedelta(days=i) for i in range(n)], # noqa: DTZ001 REDUCE_SIZES, - ) - + scaled( + ), + scaled( "custom_deepcopy", lambda n: [CustomDeepcopyObject([i]) for i in range(n)], REDUCE_SIZES, - ) + ), ) From e35e8d2e41765ddfef8464ccdb3a4fa087eb9dc5 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Thu, 26 Mar 2026 02:51:37 +0400 Subject: [PATCH 33/36] Collapse wheels and tests jobs --- .github/workflows/cd-build.yaml | 72 ++++++++++++--------------------- 1 file changed, 25 insertions(+), 47 deletions(-) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index dcc6bb4..6a201bc 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -78,6 +78,7 @@ jobs: with: name: "Wheels-${{ matrix.python.label }}-${{ matrix.build.platform_label }}-${{ matrix.build.arch_label }}" path: dist/*.whl + leading_tests: needs: leading_wheels strategy: @@ -100,21 +101,29 @@ jobs: uses: actions/download-artifact@v4 with: name: "Wheels-${{ matrix.python.label }}-${{ matrix.build.platform_label }}-${{ matrix.build.arch_label }}" - path: dist/test-wheel + path: dist + + - &should-test + name: Check test eligibility + id: should_test + shell: bash + run: echo "run=true" >> "$GITHUB_OUTPUT" + if: matrix.build.os != 'windows-11-arm' || !contains(fromJSON('["cp310","cp311","cp312"]'), matrix.python.tag) - &find-test-wheel name: Find built wheel id: wheel + if: steps.should_test.outputs.run == 'true' shell: bash run: | - wheel=$(find dist/test-wheel -name "*-${{ matrix.python.tag }}-${{ matrix.build.wheel_arch }}.whl" -type f | head -n1) + wheel=$(find dist -name "*-${{ matrix.python.tag }}-${{ matrix.build.wheel_arch }}.whl" -type f | head -n1) if [[ -z "$wheel" && "${{ matrix.build.platform_label }}" == "Linux" ]]; then case "${{ matrix.build.arch_label }}" in "x86_64") - wheel=$(find dist/test-wheel -name "*-${{ matrix.python.tag }}-*-musllinux*_x86_64.whl" -type f | head -n1) + wheel=$(find dist -name "*-${{ matrix.python.tag }}-*-musllinux*_x86_64.whl" -type f | head -n1) ;; "ARM64") - wheel=$(find dist/test-wheel -name "*-${{ matrix.python.tag }}-*-musllinux*_aarch64.whl" -type f | head -n1) + wheel=$(find dist -name "*-${{ matrix.python.tag }}-*-musllinux*_aarch64.whl" -type f | head -n1) ;; esac fi @@ -126,6 +135,7 @@ jobs: - &setup-test-env name: Setup test environment + if: steps.should_test.outputs.run == 'true' run: | uv venv --python ${{ env.PYTHON || matrix.python.label }} --clear uv sync --extra test --no-install-project @@ -133,27 +143,33 @@ jobs: - &test-built-wheel name: Test built wheel + if: steps.should_test.outputs.run == 'true' run: uv run --no-sync pytest tests/ -vvv - &test-shuffle-a name: Test shuffle A + if: steps.should_test.outputs.run == 'true' run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=A - &test-shuffle-b name: Test shuffle B + if: steps.should_test.outputs.run == 'true' run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=B - &test-shuffle-c name: Test shuffle C + if: steps.should_test.outputs.run == 'true' run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=C - &test-shuffle-d name: Test shuffle D + if: steps.should_test.outputs.run == 'true' run: uv run --no-sync pytest tests/ -v --random-order --random-order-seed=D - &cache-cpython-tests name: Cache CPython test suite id: cpython-cache + if: steps.should_test.outputs.run == 'true' uses: actions/cache@v4 with: path: cpython-tests @@ -161,7 +177,7 @@ jobs: - &checkout-cpython-tests name: Checkout CPython test suite - if: steps.cpython-cache.outputs.cache-hit != 'true' + if: steps.should_test.outputs.run == 'true' && steps.cpython-cache.outputs.cache-hit != 'true' uses: actions/checkout@v5 with: repository: python/cpython @@ -172,6 +188,7 @@ jobs: - &test-cpython-copy-patched name: Test CPython test_copy (patched) + if: steps.should_test.outputs.run == 'true' env: COPIUM_PATCH_ENABLE: "1" PYTHONPATH: ${{ github.workspace }}/cpython-tests/Lib @@ -179,13 +196,14 @@ jobs: - &test-cpython-copy-patched-dict-memo name: Test CPython test_copy (patched, dict memo) + if: steps.should_test.outputs.run == 'true' env: COPIUM_PATCH_ENABLE: "1" COPIUM_USE_DICT_MEMO: "1" PYTHONPATH: ${{ github.workspace }}/cpython-tests/Lib run: uv run --no-sync python -m unittest test.test_copy -v - wheels: + build_and_test: needs: - leading_tests if: ${{ always() }} @@ -223,47 +241,7 @@ jobs: - *build-wheel - *clear-sccache-linux - *upload-wheel-artifacts - - tests: - needs: - - wheels - if: ${{ always() && needs.wheels.result == 'success' }} - strategy: - fail-fast: false - max-parallel: 64 - matrix: - build: - - { os: ubuntu-latest, wheel_arch: "manylinux*_x86_64", arch_label: "x86_64", platform_label: "Linux" } - - { os: ubuntu-24.04-arm, wheel_arch: "manylinux*_aarch64", arch_label: "ARM64", platform_label: "Linux" } - - { os: windows-latest, wheel_arch: "win_amd64", arch_label: "x64", platform_label: "Windows" } - - { os: windows-11-arm, wheel_arch: "win_arm64", arch_label: "ARM64", platform_label: "Windows" } - - { os: macos-15, wheel_arch: "macosx*_arm64", arch_label: "Apple Silicon", platform_label: "macOS" } - - { os: macos-15-intel, wheel_arch: "macosx*_x86_64", arch_label: "Intel", platform_label: "macOS" } - python: - - { tag: "cp310", version: "3.10.11", label: "3.10" } - - { tag: "cp311", version: "3.11.9", label: "3.11" } - - { tag: "cp312", version: "3.12.10", label: "3.12" } - - { tag: "cp313", version: "3.13.9", label: "3.13" } - - { tag: "cp314", version: "3.14.0", label: "3.14" } - - { tag: "cp314t", version: "3.14t", label: "3.14t", branch: "3.14" } - exclude: - - build: { os: ubuntu-24.04-arm, wheel_arch: "manylinux*_aarch64", arch_label: "ARM64", platform_label: "Linux" } - python: { tag: "cp314", version: "3.14.0", label: "3.14" } - - build: { os: windows-11-arm, wheel_arch: "win_arm64", arch_label: "ARM64", platform_label: "Windows" } - python: { tag: "cp310", version: "3.10.11", label: "3.10" } - - build: { os: windows-11-arm, wheel_arch: "win_arm64", arch_label: "ARM64", platform_label: "Windows" } - python: { tag: "cp311", version: "3.11.9", label: "3.11" } - - build: { os: windows-11-arm, wheel_arch: "win_arm64", arch_label: "ARM64", platform_label: "Windows" } - python: { tag: "cp312", version: "3.12.10", label: "3.12" } - - runs-on: ${{ matrix.build.os }} - name: "Tests - Python ${{ matrix.python.label }} - ${{ matrix.build.platform_label }} - ${{ matrix.build.arch_label }}" - - steps: - - *checkout - - *setup-uv - - *install-python-non-linux - - *download-test-wheel + - *should-test - *find-test-wheel - *setup-test-env - *test-built-wheel From b1034ea1ecdfb70236997a8c24cdab2b001f8145 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Thu, 26 Mar 2026 15:13:39 +0400 Subject: [PATCH 34/36] Increase max time for walltime and allow it to fail --- .github/workflows/cd-build.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cd-build.yaml b/.github/workflows/cd-build.yaml index 6a201bc..c6dc9a1 100644 --- a/.github/workflows/cd-build.yaml +++ b/.github/workflows/cd-build.yaml @@ -337,6 +337,7 @@ jobs: - name: Run CodSpeed walltime benchmarks uses: CodSpeedHQ/action@v4 + continue-on-error: true with: mode: walltime run: > @@ -344,5 +345,5 @@ jobs: --codspeed -k "test_performance" --codspeed-warmup-time=0.1 - --codspeed-max-time=2 + --codspeed-max-time=3 -v \ No newline at end of file From 89e2bc323478076d77c2110d7e47b83194469d25 Mon Sep 17 00:00:00 2001 From: Bobronium Date: Thu, 26 Mar 2026 15:35:08 +0400 Subject: [PATCH 35/36] Remove bisect_codspeed_walltime.py --- tools/bisect_codspeed_walltime.py | 339 ------------------------------ 1 file changed, 339 deletions(-) delete mode 100644 tools/bisect_codspeed_walltime.py diff --git a/tools/bisect_codspeed_walltime.py b/tools/bisect_codspeed_walltime.py deleted file mode 100644 index 5c6eeea..0000000 --- a/tools/bisect_codspeed_walltime.py +++ /dev/null @@ -1,339 +0,0 @@ -from __future__ import annotations - -import json -import os -import shlex -import subprocess -import sys -from dataclasses import dataclass -from pathlib import Path -from typing import Iterable - -TEST_FILE = "tests/test_performance.py" -TEST_K = "test_performance" - -PYTEST_BASE = [ - "uv", - "run", - "--no-sync", - "pytest", - TEST_FILE, - "--codspeed", - "-k", - TEST_K, - "--codspeed-warmup-time=0", - "--codspeed-max-time=0.001", - "--codspeed-max-rounds=1", - "-q", - "-p", - "no:random-order", - "--override-ini=addopts=", -] - -CRASH_CODES = {139, -11} -CACHE_FILE = Path(".ci-codspeed-bisect-cache.json") -OUT_DIR = Path(".ci-codspeed-bisect") -OUT_DIR.mkdir(exist_ok=True) - - -@dataclass(frozen=True) -class RunResult: - code: int - crashed: bool - stdout_tail: str - stderr_tail: str - - -def sh(cmd: list[str]) -> subprocess.CompletedProcess[str]: - return subprocess.run( - cmd, - text=True, - capture_output=True, - check=False, - env=os.environ.copy(), - ) - - -def collect_nodeids() -> list[str]: - cmd = [ - "uv", - "run", - "--no-sync", - "pytest", - TEST_FILE, - "-k", - TEST_K, - "--collect-only", - "-q", - "-p", - "no:random-order", - "--override-ini=addopts=", - ] - proc = sh(cmd) - if proc.returncode != 0: - print(proc.stdout) - print(proc.stderr, file=sys.stderr) - raise SystemExit(f"collect failed: exit={proc.returncode}") - - nodeids: list[str] = [] - for line in proc.stdout.splitlines(): - line = line.strip() - if line.startswith("tests/") and "::" in line: - nodeids.append(line) - - if not nodeids: - raise SystemExit("no nodeids collected") - - return nodeids - - -def load_cache() -> dict[str, dict]: - if CACHE_FILE.exists(): - return json.loads(CACHE_FILE.read_text()) - return {} - - -def save_cache(cache: dict[str, dict]) -> None: - CACHE_FILE.write_text(json.dumps(cache, indent=2, sort_keys=True)) - - -def key_for_subset(nodeids: Iterable[str]) -> str: - return "\n".join(nodeids) - - -def write_nodeids_file(path: Path, nodeids: list[str]) -> None: - path.write_text("".join(f"{x}\n" for x in nodeids)) - - -def run_subset(nodeids: list[str], *, label: str, use_cache: bool = True) -> RunResult: - cache = load_cache() - key = key_for_subset(nodeids) - if use_cache and key in cache: - entry = cache[key] - return RunResult( - code=entry["code"], - crashed=entry["crashed"], - stdout_tail=entry["stdout_tail"], - stderr_tail=entry["stderr_tail"], - ) - - listfile = OUT_DIR / f"{label.replace('/', '_').replace(' ', '_')}.txt" - write_nodeids_file(listfile, nodeids) - - cmd = PYTEST_BASE + [f"@{listfile}"] - - print(f"\n=== RUN {label} ===") - print(f"tests: {len(nodeids)}") - print("cmd:", shlex.join(cmd)) - - proc = sh(cmd) - - stdout_tail = "\n".join(proc.stdout.splitlines()[-40:]) - stderr_tail = "\n".join(proc.stderr.splitlines()[-40:]) - crashed = proc.returncode in CRASH_CODES - - result = RunResult( - code=proc.returncode, - crashed=crashed, - stdout_tail=stdout_tail, - stderr_tail=stderr_tail, - ) - - if use_cache: - cache[key] = { - "code": result.code, - "crashed": result.crashed, - "stdout_tail": result.stdout_tail, - "stderr_tail": result.stderr_tail, - } - save_cache(cache) - - print(f"exit={result.code} crashed={result.crashed}") - if result.stdout_tail: - print("--- stdout tail ---") - print(result.stdout_tail) - if result.stderr_tail: - print("--- stderr tail ---", file=sys.stderr) - print(result.stderr_tail, file=sys.stderr) - - return result - - -def partitions(xs: list[str], n: int) -> list[list[str]]: - size = len(xs) - out: list[list[str]] = [] - start = 0 - for i in range(n): - end = start + (size - start + (n - i) - 1) // (n - i) - out.append(xs[start:end]) - start = end - return [p for p in out if p] - - -def without(xs: list[str], ys: Iterable[str]) -> list[str]: - ys_set = set(ys) - return [x for x in xs if x not in ys_set] - - -def ddmin(nodeids: list[str]) -> list[str]: - current = nodeids[:] - n = 2 - - while len(current) >= 2: - chunks = partitions(current, n) - reduced = False - - for i, chunk in enumerate(chunks, 1): - res = run_subset(chunk, label=f"ddmin_subset_{len(current)}_{i}_of_{len(chunks)}") - if res.crashed: - current = chunk - n = 2 - reduced = True - print(f"reduced to crashing subset of {len(current)} tests") - break - if reduced: - continue - - for i, chunk in enumerate(chunks, 1): - comp = without(current, chunk) - if not comp: - continue - res = run_subset(comp, label=f"ddmin_complement_{len(current)}_{i}_of_{len(chunks)}") - if res.crashed: - current = comp - n = max(n - 1, 2) - reduced = True - print(f"reduced to crashing complement of {len(current)} tests") - break - if reduced: - continue - - if n >= len(current): - break - n = min(len(current), n * 2) - - return current - - -def find_last_passing_prefix(nodeids: list[str]) -> int: - lo = 0 - hi = len(nodeids) - while lo < hi: - mid = (lo + hi + 1) // 2 - res = run_subset(nodeids[:mid], label=f"prefix_0_{mid}") - if res.crashed: - hi = mid - 1 - else: - lo = mid - return lo - - -def verify_isolated_group(group: list[str], idx: int) -> None: - res = run_subset(group, label=f"verify_group_{idx}") - if not res.crashed: - raise SystemExit( - f"group {idx} is not independently crashing; " - "this means the crash depends on interaction with outside tests" - ) - - -def verify_remainder_passes(remainder: list[str]) -> None: - res = run_subset(remainder, label="verify_remainder") - if res.crashed: - raise SystemExit( - "suite still crashes after excluding all discovered failing groups; " - "partition is incomplete" - ) - - -def verify_each_group_against_clean_remainder(remainder: list[str], groups: list[list[str]]) -> None: - for i, group in enumerate(groups, 1): - combo = remainder + group - res = run_subset(combo, label=f"verify_remainder_plus_group_{i}") - if not res.crashed: - raise SystemExit( - f"clean remainder + failing group {i} did not crash; " - "group is not sufficient in the final partition" - ) - - -def main() -> int: - all_nodeids = collect_nodeids() - print(f"collected {len(all_nodeids)} nodeids") - - full = run_subset(all_nodeids, label="full_suite", use_cache=False) - if not full.crashed: - print("full suite did not reproduce crash") - return 2 - - last_ok = find_last_passing_prefix(all_nodeids) - print(f"last passing prefix length: {last_ok}") - if last_ok < len(all_nodeids): - print("first suspect after passing prefix:") - print(all_nodeids[last_ok]) - - remaining = all_nodeids[:] - failing_groups: list[list[str]] = [] - - iteration = 0 - while True: - iteration += 1 - res = run_subset(remaining, label=f"remaining_{iteration}", use_cache=False) - if not res.crashed: - break - - print(f"\n### extracting crashing group #{iteration} from {len(remaining)} tests") - group = ddmin(remaining) - - group_path = OUT_DIR / f"failing_group_{iteration}.txt" - write_nodeids_file(group_path, group) - print(f"wrote {group_path}") - - verify_isolated_group(group, iteration) - - new_remaining = without(remaining, group) - remaining_res = run_subset(new_remaining, label=f"remaining_after_group_{iteration}", use_cache=False) - - failing_groups.append(group) - remaining = new_remaining - - print( - f"group #{iteration}: {len(group)} tests; " - f"remaining: {len(remaining)}; " - f"remaining_crashes={remaining_res.crashed}" - ) - - if not remaining_res.crashed: - break - - passing_tests = remaining[:] - - print("\n=== FINAL PARTITION ===") - print(f"failing groups: {len(failing_groups)}") - for i, group in enumerate(failing_groups, 1): - print(f" group {i}: {len(group)} tests") - for t in group: - print(f" {t}") - print(f"passing remainder: {len(passing_tests)} tests") - - verify_remainder_passes(passing_tests) - verify_each_group_against_clean_remainder(passing_tests, failing_groups) - - write_nodeids_file(OUT_DIR / "passing_tests.txt", passing_tests) - for i, group in enumerate(failing_groups, 1): - write_nodeids_file(OUT_DIR / f"failing_group_{i}.txt", group) - - summary = { - "total_tests": len(all_nodeids), - "failing_group_count": len(failing_groups), - "failing_groups": failing_groups, - "passing_tests": passing_tests, - } - (OUT_DIR / "summary.json").write_text(json.dumps(summary, indent=2)) - - print("\npartition complete") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) \ No newline at end of file From 91c1ec586de5c0caf41f6071a22ea5e9579ef9da Mon Sep 17 00:00:00 2001 From: Bobronium Date: Thu, 26 Mar 2026 15:37:00 +0400 Subject: [PATCH 36/36] Remove codpseed entry in taskfile.yaml --- taskfile.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/taskfile.yaml b/taskfile.yaml index 0c616e0..65d9e7e 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -111,15 +111,6 @@ tasks: done PY - benchmark:codspeed: - desc: Run the CodSpeed benchmark suite locally with Python 3.14 - env: - PYTHONHASHSEED: "0" - cmds: - - uv venv --python 3.14 --clear - - uv sync --inexact --quiet --extra test - - uv run --no-sync pytest -o addopts='' tests/test_performance.py --codspeed -v - build:wheel: desc: Regular optimized wheel (LTO + -O3 by default) cmds: