From 349250b26336f6db3aeef4cdcd7506e8751b9628 Mon Sep 17 00:00:00 2001 From: "Kruszynski, MarcinX" Date: Thu, 14 May 2026 14:25:42 +0200 Subject: [PATCH 1/8] coverage part 1 --- .github/coverage/tests_python.yml | 134 ++++++++++ .github/workflows/coverage.yml | 392 ++++++++++++++++++++++++++++++ 2 files changed, 526 insertions(+) create mode 100644 .github/coverage/tests_python.yml create mode 100644 .github/workflows/coverage.yml diff --git a/.github/coverage/tests_python.yml b/.github/coverage/tests_python.yml new file mode 100644 index 00000000000..029665e4014 --- /dev/null +++ b/.github/coverage/tests_python.yml @@ -0,0 +1,134 @@ +schema_version: 1 +suite: python +reporting: + label: Python + artifact_group: python + artifact_name_template: coverage-{suite}-{lane} + selection_env: PY_TEST_NAMES + stats_file: python-coverage-stats.env + duration_file: python-test-durations.csv + coverage_file: python-coverage.xml + debug_dirs: + - src: .tmp/python-coverage + dest: python-coverage-debug + uploads: + - name: NNCF Python + artifact_pattern: coverage-python-* + file: python-coverage.xml + key: nncf_python_xml + flag: nncf-python + +tests: + - name: common + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -ra --durations=30 -n2 ${PYTEST_COVERAGE_ARGS} + tests/common --junitxml nncf-tests.xml + + - name: onnx + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -ra --durations=30 -n4 ${PYTEST_COVERAGE_ARGS} + tests/onnx --junitxml nncf-tests.xml + + - name: openvino + kind: command + profiles: [cpu] + command: >- + ONEDNN_MAX_CPU_ISA=AVX2 python3 -m pytest -ra --durations=30 -n4 --dist loadscope + ${PYTEST_COVERAGE_ARGS} tests/openvino --junitxml nncf-tests.xml + + - name: torch_cpu + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -ra -n2 --durations=30 ${PYTEST_COVERAGE_ARGS} + tests/torch -m "not cuda" --junitxml nncf-tests.xml + + - name: tools + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -ra --durations=30 ${PYTEST_COVERAGE_ARGS} + tests/tools --junitxml nncf-tests.xml + + - name: executorch + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -ra --durations=30 ${PYTEST_COVERAGE_ARGS} + tests/executorch --junitxml nncf-tests.xml + + - name: wc_1 + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -s -ra ${PYTEST_COVERAGE_ARGS} + tests/post_training/test_quantize_conformance.py::test_weight_compression + --junit-xml=nncf-tests.xml + --durations-path=tests/post_training/data/wc_test_durations.json + --splitting-algorithm=least_duration + --splits 6 + --group 1 + + - name: wc_2 + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -s -ra ${PYTEST_COVERAGE_ARGS} + tests/post_training/test_quantize_conformance.py::test_weight_compression + --junit-xml=nncf-tests.xml + --durations-path=tests/post_training/data/wc_test_durations.json + --splitting-algorithm=least_duration + --splits 6 + --group 2 + + - name: wc_3 + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -s -ra ${PYTEST_COVERAGE_ARGS} + tests/post_training/test_quantize_conformance.py::test_weight_compression + --junit-xml=nncf-tests.xml + --durations-path=tests/post_training/data/wc_test_durations.json + --splitting-algorithm=least_duration + --splits 6 + --group 3 + + - name: wc_4 + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -s -ra ${PYTEST_COVERAGE_ARGS} + tests/post_training/test_quantize_conformance.py::test_weight_compression + --junit-xml=nncf-tests.xml + --durations-path=tests/post_training/data/wc_test_durations.json + --splitting-algorithm=least_duration + --splits 6 + --group 4 + + - name: wc_5 + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -s -ra ${PYTEST_COVERAGE_ARGS} + tests/post_training/test_quantize_conformance.py::test_weight_compression + --junit-xml=nncf-tests.xml + --durations-path=tests/post_training/data/wc_test_durations.json + --splitting-algorithm=least_duration + --splits 6 + --group 5 + + - name: wc_6 + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -s -ra ${PYTEST_COVERAGE_ARGS} + tests/post_training/test_quantize_conformance.py::test_weight_compression + --junit-xml=nncf-tests.xml + --durations-path=tests/post_training/data/wc_test_durations.json + --splitting-algorithm=least_duration + --splits 6 + --group 6 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 00000000000..c31e182b7dc --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,392 @@ +name: Code coverage + +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * *' + +permissions: read-all + +concurrency: + group: ${{ github.event_name == 'pull_request' && github.run_id || format('{0}-{1}', github.workflow, github.ref) }} + cancel-in-progress: false + +env: + PYTHON_VERSION: '3.12' + ENABLE_BRANCH_COVERAGE: 'false' + +jobs: + CoveragePythonCore: + name: Coverage Python ${{ matrix.test.name }} (CPU) + timeout-minutes: 120 + runs-on: ${{ matrix.test.runner }} + strategy: + fail-fast: false + matrix: + test: + - name: common + runner: ubuntu-latest + requirements: tests/common/requirements.txt + - name: onnx + runner: ubuntu-latest-8-cores + requirements: tests/onnx/requirements.txt + - name: openvino + runner: ubuntu-latest-8-cores + requirements: tests/openvino/requirements.txt + - name: torch_cpu + runner: ubuntu-latest + requirements: tests/torch/requirements.txt + - name: tools + runner: ubuntu-latest + requirements: tests/tools/requirements.txt + defaults: + run: + shell: bash + env: + TEST_PROFILE: cpu + OV_WORKSPACE: ${{ github.workspace }} + PY_TEST_NAMES: ${{ matrix.test.name }} + COVERAGE_WRITE_STEP_SUMMARY: 'false' + steps: + - name: Checkout sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + lfs: true + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + + - name: Install coverage Python dependencies + uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + with: + command: install-requirements + + - name: Install NNCF and test requirements + run: uv pip install --system . -r ${{ matrix.test.requirements }} + + - name: Run Python coverage tests + uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + with: + command: run-suite + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + suite: python + profile: cpu + lane: ${{ matrix.test.name }} + test-names: ${{ matrix.test.name }} + python-package: nncf + python-source-dir: src/nncf + python-path-alias: nncf + python-coverage-omit: | + */tests/* + */docs/* + */examples/* + */tools/* + */build/* + */.venv/* + */venv/* + + - name: Collect Python results + id: cpu_artifact + if: ${{ always() }} + uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + with: + command: collect-results + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + suite: python + profile: cpu + lane: ${{ matrix.test.name }} + test-names: ${{ matrix.test.name }} + + - name: Upload Python artifact + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ steps.cpu_artifact.outputs['artifact-name'] }} + path: ${{ steps.cpu_artifact.outputs.dir }} + if-no-files-found: warn + retention-days: 7 + compression-level: 0 + overwrite: true + + CoveragePythonExecuTorch: + name: Coverage Python ExecuTorch (CPU) + timeout-minutes: 60 + runs-on: ubuntu-latest-8-cores + defaults: + run: + shell: bash + env: + TEST_PROFILE: cpu + OV_WORKSPACE: ${{ github.workspace }} + PY_TEST_NAMES: executorch + COVERAGE_WRITE_STEP_SUMMARY: 'false' + DEBIAN_FRONTEND: noninteractive + steps: + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get --assume-yes install gcc g++ build-essential ninja-build libgl1-mesa-dev libglib2.0-0 + + - name: Checkout sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + lfs: true + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.10.14' + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + + - name: Install coverage Python dependencies + uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + with: + command: install-requirements + + - name: Install NNCF and test requirements + run: pip install . -r tests/executorch/requirements.txt + + - name: Run ExecuTorch coverage tests + uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + with: + command: run-suite + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + suite: python + profile: cpu + lane: executorch + test-names: executorch + python-package: nncf + python-source-dir: src/nncf + python-path-alias: nncf + python-coverage-omit: | + */tests/* + */docs/* + */examples/* + */tools/* + */build/* + */.venv/* + */venv/* + + - name: Collect ExecuTorch results + id: cpu_artifact + if: ${{ always() }} + uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + with: + command: collect-results + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + suite: python + profile: cpu + lane: executorch + test-names: executorch + + - name: Upload ExecuTorch artifact + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ steps.cpu_artifact.outputs['artifact-name'] }} + path: ${{ steps.cpu_artifact.outputs.dir }} + if-no-files-found: warn + retention-days: 7 + compression-level: 0 + overwrite: true + + CoveragePythonWeightCompression: + name: Coverage Python Weight Compression ${{ matrix.group }}/6 (CPU) + timeout-minutes: 120 + runs-on: ubuntu-latest-16-cores + strategy: + fail-fast: false + matrix: + group: [1, 2, 3, 4, 5, 6] + defaults: + run: + shell: bash + env: + TEST_PROFILE: cpu + OV_WORKSPACE: ${{ github.workspace }} + PY_TEST_NAMES: wc_${{ matrix.group }} + COVERAGE_WRITE_STEP_SUMMARY: 'false' + TQDM_DISABLE: 1 + HOME_HF: /home/runner/hf_home + steps: + - name: Checkout sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + + - name: Install coverage Python dependencies + uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + with: + command: install-requirements + + - name: Install NNCF and test requirements + run: uv pip install --system . -r tests/post_training/requirements.txt + + - name: Run weight-compression coverage tests + uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + with: + command: run-suite + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + suite: python + profile: cpu + lane: wc_${{ matrix.group }} + test-names: wc_${{ matrix.group }} + python-package: nncf + python-source-dir: src/nncf + python-path-alias: nncf + python-coverage-omit: | + */tests/* + */docs/* + */examples/* + */tools/* + */build/* + */.venv/* + */venv/* + + - name: Collect weight-compression results + id: cpu_artifact + if: ${{ always() }} + uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + with: + command: collect-results + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + suite: python + profile: cpu + lane: wc_${{ matrix.group }} + test-names: wc_${{ matrix.group }} + + - name: Upload weight-compression artifact + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ steps.cpu_artifact.outputs['artifact-name'] }} + path: ${{ steps.cpu_artifact.outputs.dir }} + if-no-files-found: warn + retention-days: 7 + compression-level: 0 + overwrite: true + + CoverageSummary: + name: Coverage Summary + if: ${{ always() && !cancelled() }} + needs: [CoveragePythonCore, CoveragePythonExecuTorch, CoveragePythonWeightCompression] + outputs: + upload_matrix_json: ${{ steps.coverage_files.outputs['upload-matrix-json'] || '[]' }} + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + lfs: false + + - name: Install coverage Python dependencies + uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + with: + command: install-requirements + + - name: Download Python artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: coverage-python-* + path: ${{ github.workspace }}/artifacts/python + + - name: Generate final summary + if: ${{ always() }} + uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + with: + command: render-summary + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + selection: phase-1 + selected-lanes: common,onnx,openvino,torch_cpu,tools,executorch,wc_1,wc_2,wc_3,wc_4,wc_5,wc_6 + + - name: Merge duration reports + if: ${{ always() }} + uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + with: + command: merge-durations + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + duration-output: ${{ github.workspace }}/coverage-test-durations-all.csv + + - name: Upload merged duration artifact + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-test-durations + path: coverage-test-durations-all.csv + if-no-files-found: warn + retention-days: 7 + compression-level: 0 + overwrite: true + + - name: Resolve Codecov upload files + id: coverage_files + if: ${{ always() }} + uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + with: + command: resolve-uploads + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + + CoverageCodecovUploads: + name: Coverage Codecov Upload (${{ matrix.upload.name }}) + if: ${{ always() && needs.CoverageSummary.result == 'success' && needs.CoverageSummary.outputs.upload_matrix_json != '[]' }} + needs: CoverageSummary + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + upload: ${{ fromJSON(needs.CoverageSummary.outputs.upload_matrix_json || '[]') }} + steps: + - name: Checkout sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + lfs: false + + - name: Download coverage artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: ${{ matrix.upload.artifact_pattern }} + path: ${{ github.workspace }}/artifacts/${{ matrix.upload.artifact_group }} + merge-multiple: false + + - name: Resolve Codecov upload files + id: upload_files + uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + with: + command: resolve-upload-files + workspace: ${{ github.workspace }} + artifact-group: ${{ matrix.upload.artifact_group }} + artifact-names-json: ${{ toJSON(matrix.upload.artifact_names) }} + upload-key: ${{ matrix.upload.key }} + upload-file: ${{ matrix.upload.file }} + + - name: Upload Codecov (${{ matrix.upload.name }}) + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ${{ steps.upload_files.outputs.files }} + disable_search: true + plugins: noop + flags: ${{ matrix.upload.flag }} + override_commit: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + override_pr: ${{ github.event_name == 'pull_request' && github.event.pull_request.number || '' }} + fail_ci_if_error: true + verbose: true From 46c0d3ab4e27fa87baabf50b3c970933a390bda3 Mon Sep 17 00:00:00 2001 From: "Kruszynski, MarcinX" Date: Thu, 14 May 2026 14:38:17 +0200 Subject: [PATCH 2/8] enable pull_request trigger for testing purposes --- .github/workflows/coverage.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index c31e182b7dc..59901524f10 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -2,8 +2,7 @@ name: Code coverage on: workflow_dispatch: - schedule: - - cron: '0 0 * * *' + pull_request: permissions: read-all From 33352958e55d765b6af4dd8babf9208ab9dc7bce Mon Sep 17 00:00:00 2001 From: "Kruszynski, MarcinX" Date: Thu, 14 May 2026 14:43:27 +0200 Subject: [PATCH 3/8] fix python tests cmd --- .github/coverage/tests_python.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/coverage/tests_python.yml b/.github/coverage/tests_python.yml index 029665e4014..5b15ce4e902 100644 --- a/.github/coverage/tests_python.yml +++ b/.github/coverage/tests_python.yml @@ -67,7 +67,7 @@ tests: command: >- python3 -m pytest -s -ra ${PYTEST_COVERAGE_ARGS} tests/post_training/test_quantize_conformance.py::test_weight_compression - --junit-xml=nncf-tests.xml + --junitxml=nncf-tests.xml --durations-path=tests/post_training/data/wc_test_durations.json --splitting-algorithm=least_duration --splits 6 @@ -79,7 +79,7 @@ tests: command: >- python3 -m pytest -s -ra ${PYTEST_COVERAGE_ARGS} tests/post_training/test_quantize_conformance.py::test_weight_compression - --junit-xml=nncf-tests.xml + --junitxml=nncf-tests.xml --durations-path=tests/post_training/data/wc_test_durations.json --splitting-algorithm=least_duration --splits 6 @@ -91,7 +91,7 @@ tests: command: >- python3 -m pytest -s -ra ${PYTEST_COVERAGE_ARGS} tests/post_training/test_quantize_conformance.py::test_weight_compression - --junit-xml=nncf-tests.xml + --junitxml=nncf-tests.xml --durations-path=tests/post_training/data/wc_test_durations.json --splitting-algorithm=least_duration --splits 6 @@ -103,7 +103,7 @@ tests: command: >- python3 -m pytest -s -ra ${PYTEST_COVERAGE_ARGS} tests/post_training/test_quantize_conformance.py::test_weight_compression - --junit-xml=nncf-tests.xml + --junitxml=nncf-tests.xml --durations-path=tests/post_training/data/wc_test_durations.json --splitting-algorithm=least_duration --splits 6 @@ -115,7 +115,7 @@ tests: command: >- python3 -m pytest -s -ra ${PYTEST_COVERAGE_ARGS} tests/post_training/test_quantize_conformance.py::test_weight_compression - --junit-xml=nncf-tests.xml + --junitxml=nncf-tests.xml --durations-path=tests/post_training/data/wc_test_durations.json --splitting-algorithm=least_duration --splits 6 @@ -127,7 +127,7 @@ tests: command: >- python3 -m pytest -s -ra ${PYTEST_COVERAGE_ARGS} tests/post_training/test_quantize_conformance.py::test_weight_compression - --junit-xml=nncf-tests.xml + --junitxml=nncf-tests.xml --durations-path=tests/post_training/data/wc_test_durations.json --splitting-algorithm=least_duration --splits 6 From 76f5200544635be7cbdd3557f2c7fc11e7538e51 Mon Sep 17 00:00:00 2001 From: "Kruszynski, MarcinX" Date: Wed, 20 May 2026 10:35:45 +0200 Subject: [PATCH 4/8] enable cuda tests --- .github/coverage/tests_python.yml | 14 ++++ .github/workflows/coverage.yml | 120 +++++++++++++++++++++++++++++- 2 files changed, 131 insertions(+), 3 deletions(-) diff --git a/.github/coverage/tests_python.yml b/.github/coverage/tests_python.yml index 5b15ce4e902..1af2173cb10 100644 --- a/.github/coverage/tests_python.yml +++ b/.github/coverage/tests_python.yml @@ -47,6 +47,20 @@ tests: python3 -m pytest -ra -n2 --durations=30 ${PYTEST_COVERAGE_ARGS} tests/torch -m "not cuda" --junitxml nncf-tests.xml + - name: torch_cuda + kind: command + profiles: [gpu] + command: >- + python3 -m pytest -ra --durations=30 ${PYTEST_COVERAGE_ARGS} + tests/torch -m cuda --junitxml nncf-tests.xml + + - name: gptqmodel_cuda + kind: command + profiles: [gpu] + command: >- + python3 -m pytest -ra --durations=30 ${PYTEST_COVERAGE_ARGS} + tests/integration/gptq_model/ --junitxml nncf-tests.xml + - name: tools kind: command profiles: [cpu] diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 59901524f10..816d57e8407 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -281,10 +281,124 @@ jobs: compression-level: 0 overwrite: true + CoveragePythonCuda: + name: Coverage Python ${{ matrix.test.name }} (CUDA) + timeout-minutes: 120 + runs-on: aks-linux-6-cores-55gb-gpu-a10 + strategy: + fail-fast: false + matrix: + test: + - name: torch_cuda + requirements: tests/torch/requirements.txt + extra_requirements: '' + - name: gptqmodel_cuda + requirements: tests/integration/gptq_model/requirements.txt + extra_requirements: tests/integration/gptq_model/requirements_extra.txt + defaults: + run: + shell: bash + env: + TEST_PROFILE: gpu + OV_WORKSPACE: ${{ github.workspace }} + PY_TEST_NAMES: ${{ matrix.test.name }} + COVERAGE_WRITE_STEP_SUMMARY: 'false' + PYTHONDONTWRITEBYTECODE: '1' + DEBIAN_FRONTEND: noninteractive + steps: + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get --assume-yes install build-essential ninja-build libgl1-mesa-dev libglib2.0-0 wget make + + - name: Download CUDA + run: | + wget -q https://developer.download.nvidia.com/compute/cuda/12.6.3/local_installers/cuda_12.6.3_560.35.05_linux.run + sudo sh cuda_12.6.3_560.35.05_linux.run --toolkit --silent + + - name: Runner info + continue-on-error: true + run: | + nvidia-smi + cat /proc/cpuinfo + + - name: Checkout sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + lfs: true + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + + - name: Install coverage Python dependencies + uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + with: + command: install-requirements + + - name: Install NNCF and test requirements + run: uv pip install --system . -r ${{ matrix.test.requirements }} + + - name: Install extra requirements + if: ${{ matrix.test.extra_requirements != '' }} + run: uv pip install --system -r ${{ matrix.test.extra_requirements }} + + - name: Check CUDA + run: python -c "import torch; print(torch.cuda.is_available())" + + - name: Run CUDA Python coverage tests + uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + with: + command: run-suite + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + suite: python + profile: gpu + lane: ${{ matrix.test.name }} + test-names: ${{ matrix.test.name }} + python-package: nncf + python-source-dir: src/nncf + python-path-alias: nncf + python-coverage-omit: | + */tests/* + */docs/* + */examples/* + */tools/* + */build/* + */.venv/* + */venv/* + + - name: Collect CUDA Python results + id: gpu_artifact + if: ${{ always() }} + uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + with: + command: collect-results + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + suite: python + profile: gpu + lane: ${{ matrix.test.name }} + test-names: ${{ matrix.test.name }} + + - name: Upload CUDA Python artifact + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ steps.gpu_artifact.outputs['artifact-name'] }} + path: ${{ steps.gpu_artifact.outputs.dir }} + if-no-files-found: warn + retention-days: 7 + compression-level: 0 + overwrite: true + CoverageSummary: name: Coverage Summary if: ${{ always() && !cancelled() }} - needs: [CoveragePythonCore, CoveragePythonExecuTorch, CoveragePythonWeightCompression] + needs: [CoveragePythonCore, CoveragePythonExecuTorch, CoveragePythonWeightCompression, CoveragePythonCuda] outputs: upload_matrix_json: ${{ steps.coverage_files.outputs['upload-matrix-json'] || '[]' }} runs-on: ubuntu-latest @@ -312,8 +426,8 @@ jobs: command: render-summary workspace: ${{ github.workspace }} config-dir: ${{ github.workspace }}/.github/coverage - selection: phase-1 - selected-lanes: common,onnx,openvino,torch_cpu,tools,executorch,wc_1,wc_2,wc_3,wc_4,wc_5,wc_6 + selection: phase-1,phase-2 + selected-lanes: common,onnx,openvino,torch_cpu,tools,executorch,wc_1,wc_2,wc_3,wc_4,wc_5,wc_6,torch_cuda,gptqmodel_cuda - name: Merge duration reports if: ${{ always() }} From b62e1228e393f77dea06edd706f175d034f213c9 Mon Sep 17 00:00:00 2001 From: "Kruszynski, MarcinX" Date: Fri, 22 May 2026 14:49:07 +0200 Subject: [PATCH 5/8] enable fuzz test --- .github/coverage/tests_python.yml | 7 +++ .github/workflows/coverage.yml | 85 +++++++++++++++++++++++++++++-- 2 files changed, 89 insertions(+), 3 deletions(-) diff --git a/.github/coverage/tests_python.yml b/.github/coverage/tests_python.yml index 1af2173cb10..b940c9cbb52 100644 --- a/.github/coverage/tests_python.yml +++ b/.github/coverage/tests_python.yml @@ -68,6 +68,13 @@ tests: python3 -m pytest -ra --durations=30 ${PYTEST_COVERAGE_ARGS} tests/tools --junitxml nncf-tests.xml + - name: fuzz + kind: command + profiles: [cpu] + command: >- + python3 -m coverage run --rcfile=${PY_COV_CONFIG} --append + tests/cross_fw/sdl/fuzz/fuzz_target.py -runs=10 -seed=1 + - name: executorch kind: command profiles: [cpu] diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 816d57e8407..8a6d1720c45 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -281,6 +281,85 @@ jobs: compression-level: 0 overwrite: true + CoveragePythonFuzz: + name: Coverage Python Fuzz (CPU) + timeout-minutes: 30 + runs-on: ubuntu-latest + defaults: + run: + shell: bash + env: + TEST_PROFILE: cpu + OV_WORKSPACE: ${{ github.workspace }} + PY_TEST_NAMES: fuzz + COVERAGE_WRITE_STEP_SUMMARY: 'false' + steps: + - name: Checkout sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + lfs: true + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + + - name: Install coverage Python dependencies + uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + with: + command: install-requirements + + - name: Install NNCF and fuzz requirements + run: uv pip install --system . -r tests/cross_fw/sdl/fuzz/requirements.txt + + - name: Run fuzz coverage test + uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + with: + command: run-suite + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + suite: python + profile: cpu + lane: fuzz + test-names: fuzz + python-package: nncf + python-source-dir: src/nncf + python-path-alias: nncf + python-coverage-omit: | + */tests/* + */docs/* + */examples/* + */tools/* + */build/* + */.venv/* + */venv/* + + - name: Collect fuzz results + id: cpu_artifact + if: ${{ always() }} + uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + with: + command: collect-results + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + suite: python + profile: cpu + lane: fuzz + test-names: fuzz + + - name: Upload fuzz artifact + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ steps.cpu_artifact.outputs['artifact-name'] }} + path: ${{ steps.cpu_artifact.outputs.dir }} + if-no-files-found: warn + retention-days: 7 + compression-level: 0 + overwrite: true + CoveragePythonCuda: name: Coverage Python ${{ matrix.test.name }} (CUDA) timeout-minutes: 120 @@ -398,7 +477,7 @@ jobs: CoverageSummary: name: Coverage Summary if: ${{ always() && !cancelled() }} - needs: [CoveragePythonCore, CoveragePythonExecuTorch, CoveragePythonWeightCompression, CoveragePythonCuda] + needs: [CoveragePythonCore, CoveragePythonExecuTorch, CoveragePythonWeightCompression, CoveragePythonFuzz, CoveragePythonCuda] outputs: upload_matrix_json: ${{ steps.coverage_files.outputs['upload-matrix-json'] || '[]' }} runs-on: ubuntu-latest @@ -426,8 +505,8 @@ jobs: command: render-summary workspace: ${{ github.workspace }} config-dir: ${{ github.workspace }}/.github/coverage - selection: phase-1,phase-2 - selected-lanes: common,onnx,openvino,torch_cpu,tools,executorch,wc_1,wc_2,wc_3,wc_4,wc_5,wc_6,torch_cuda,gptqmodel_cuda + selection: common,onnx,openvino,torch_cpu,tools,executorch,wc_1,wc_2,wc_3,wc_4,wc_5,wc_6,fuzz,torch_cuda,gptqmodel_cuda + selected-lanes: common,onnx,openvino,torch_cpu,tools,executorch,wc_1,wc_2,wc_3,wc_4,wc_5,wc_6,fuzz,torch_cuda,gptqmodel_cuda - name: Merge duration reports if: ${{ always() }} From 43cf8d600b012c0a3660d6b2ff5a57d2e9b92020 Mon Sep 17 00:00:00 2001 From: "Kruszynski, MarcinX" Date: Mon, 25 May 2026 09:09:12 +0200 Subject: [PATCH 6/8] enable fuzz test (python 3.11) --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 8a6d1720c45..2eb937db0ef 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -301,7 +301,7 @@ jobs: - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: '3.11' - name: Install uv uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 From da9dc448b31167d9efb0fb91128bb5b2ea81e40b Mon Sep 17 00:00:00 2001 From: "Kruszynski, MarcinX" Date: Mon, 25 May 2026 11:13:28 +0200 Subject: [PATCH 7/8] remove fuzz test / use multiple flags --- .github/coverage/tests_python.yml | 14 ++--- .github/workflows/coverage.yml | 85 ++----------------------------- 2 files changed, 6 insertions(+), 93 deletions(-) diff --git a/.github/coverage/tests_python.yml b/.github/coverage/tests_python.yml index b940c9cbb52..60a80e0c7f4 100644 --- a/.github/coverage/tests_python.yml +++ b/.github/coverage/tests_python.yml @@ -12,11 +12,10 @@ reporting: - src: .tmp/python-coverage dest: python-coverage-debug uploads: - - name: NNCF Python - artifact_pattern: coverage-python-* + - name_template: NNCF Python, {lane} file: python-coverage.xml - key: nncf_python_xml - flag: nncf-python + key_template: nncf_python_{lane}_xml + flag_template: nncf-python-{lane} tests: - name: common @@ -68,13 +67,6 @@ tests: python3 -m pytest -ra --durations=30 ${PYTEST_COVERAGE_ARGS} tests/tools --junitxml nncf-tests.xml - - name: fuzz - kind: command - profiles: [cpu] - command: >- - python3 -m coverage run --rcfile=${PY_COV_CONFIG} --append - tests/cross_fw/sdl/fuzz/fuzz_target.py -runs=10 -seed=1 - - name: executorch kind: command profiles: [cpu] diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 2eb937db0ef..f4dfcabcc3b 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -281,85 +281,6 @@ jobs: compression-level: 0 overwrite: true - CoveragePythonFuzz: - name: Coverage Python Fuzz (CPU) - timeout-minutes: 30 - runs-on: ubuntu-latest - defaults: - run: - shell: bash - env: - TEST_PROFILE: cpu - OV_WORKSPACE: ${{ github.workspace }} - PY_TEST_NAMES: fuzz - COVERAGE_WRITE_STEP_SUMMARY: 'false' - steps: - - name: Checkout sources - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - lfs: true - - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: '3.11' - - - name: Install uv - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - - - name: Install coverage Python dependencies - uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage - with: - command: install-requirements - - - name: Install NNCF and fuzz requirements - run: uv pip install --system . -r tests/cross_fw/sdl/fuzz/requirements.txt - - - name: Run fuzz coverage test - uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage - with: - command: run-suite - workspace: ${{ github.workspace }} - config-dir: ${{ github.workspace }}/.github/coverage - suite: python - profile: cpu - lane: fuzz - test-names: fuzz - python-package: nncf - python-source-dir: src/nncf - python-path-alias: nncf - python-coverage-omit: | - */tests/* - */docs/* - */examples/* - */tools/* - */build/* - */.venv/* - */venv/* - - - name: Collect fuzz results - id: cpu_artifact - if: ${{ always() }} - uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage - with: - command: collect-results - workspace: ${{ github.workspace }} - config-dir: ${{ github.workspace }}/.github/coverage - suite: python - profile: cpu - lane: fuzz - test-names: fuzz - - - name: Upload fuzz artifact - if: ${{ always() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: ${{ steps.cpu_artifact.outputs['artifact-name'] }} - path: ${{ steps.cpu_artifact.outputs.dir }} - if-no-files-found: warn - retention-days: 7 - compression-level: 0 - overwrite: true - CoveragePythonCuda: name: Coverage Python ${{ matrix.test.name }} (CUDA) timeout-minutes: 120 @@ -477,7 +398,7 @@ jobs: CoverageSummary: name: Coverage Summary if: ${{ always() && !cancelled() }} - needs: [CoveragePythonCore, CoveragePythonExecuTorch, CoveragePythonWeightCompression, CoveragePythonFuzz, CoveragePythonCuda] + needs: [CoveragePythonCore, CoveragePythonExecuTorch, CoveragePythonWeightCompression, CoveragePythonCuda] outputs: upload_matrix_json: ${{ steps.coverage_files.outputs['upload-matrix-json'] || '[]' }} runs-on: ubuntu-latest @@ -505,8 +426,8 @@ jobs: command: render-summary workspace: ${{ github.workspace }} config-dir: ${{ github.workspace }}/.github/coverage - selection: common,onnx,openvino,torch_cpu,tools,executorch,wc_1,wc_2,wc_3,wc_4,wc_5,wc_6,fuzz,torch_cuda,gptqmodel_cuda - selected-lanes: common,onnx,openvino,torch_cpu,tools,executorch,wc_1,wc_2,wc_3,wc_4,wc_5,wc_6,fuzz,torch_cuda,gptqmodel_cuda + selection: common,onnx,openvino,torch_cpu,tools,executorch,wc_1,wc_2,wc_3,wc_4,wc_5,wc_6,torch_cuda,gptqmodel_cuda + selected-lanes: common,onnx,openvino,torch_cpu,tools,executorch,wc_1,wc_2,wc_3,wc_4,wc_5,wc_6,torch_cuda,gptqmodel_cuda - name: Merge duration reports if: ${{ always() }} From 81e69349e6aac27b12434cb1ade68b622ecb221d Mon Sep 17 00:00:00 2001 From: "Kruszynski, MarcinX" Date: Tue, 2 Jun 2026 18:11:45 +0200 Subject: [PATCH 8/8] switch composite-action to master --- .github/workflows/coverage.yml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index f4dfcabcc3b..990b4aa3cab 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -60,7 +60,7 @@ jobs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - name: Install coverage Python dependencies - uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master with: command: install-requirements @@ -68,7 +68,7 @@ jobs: run: uv pip install --system . -r ${{ matrix.test.requirements }} - name: Run Python coverage tests - uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master with: command: run-suite workspace: ${{ github.workspace }} @@ -92,7 +92,7 @@ jobs: - name: Collect Python results id: cpu_artifact if: ${{ always() }} - uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master with: command: collect-results workspace: ${{ github.workspace }} @@ -145,7 +145,7 @@ jobs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - name: Install coverage Python dependencies - uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master with: command: install-requirements @@ -153,7 +153,7 @@ jobs: run: pip install . -r tests/executorch/requirements.txt - name: Run ExecuTorch coverage tests - uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master with: command: run-suite workspace: ${{ github.workspace }} @@ -177,7 +177,7 @@ jobs: - name: Collect ExecuTorch results id: cpu_artifact if: ${{ always() }} - uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master with: command: collect-results workspace: ${{ github.workspace }} @@ -228,7 +228,7 @@ jobs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - name: Install coverage Python dependencies - uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master with: command: install-requirements @@ -236,7 +236,7 @@ jobs: run: uv pip install --system . -r tests/post_training/requirements.txt - name: Run weight-compression coverage tests - uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master with: command: run-suite workspace: ${{ github.workspace }} @@ -260,7 +260,7 @@ jobs: - name: Collect weight-compression results id: cpu_artifact if: ${{ always() }} - uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master with: command: collect-results workspace: ${{ github.workspace }} @@ -335,7 +335,7 @@ jobs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - name: Install coverage Python dependencies - uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master with: command: install-requirements @@ -350,7 +350,7 @@ jobs: run: python -c "import torch; print(torch.cuda.is_available())" - name: Run CUDA Python coverage tests - uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master with: command: run-suite workspace: ${{ github.workspace }} @@ -374,7 +374,7 @@ jobs: - name: Collect CUDA Python results id: gpu_artifact if: ${{ always() }} - uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master with: command: collect-results workspace: ${{ github.workspace }} @@ -409,7 +409,7 @@ jobs: lfs: false - name: Install coverage Python dependencies - uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master with: command: install-requirements @@ -421,7 +421,7 @@ jobs: - name: Generate final summary if: ${{ always() }} - uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master with: command: render-summary workspace: ${{ github.workspace }} @@ -431,7 +431,7 @@ jobs: - name: Merge duration reports if: ${{ always() }} - uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master with: command: merge-durations workspace: ${{ github.workspace }} @@ -452,7 +452,7 @@ jobs: - name: Resolve Codecov upload files id: coverage_files if: ${{ always() }} - uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master with: command: resolve-uploads workspace: ${{ github.workspace }} @@ -482,7 +482,7 @@ jobs: - name: Resolve Codecov upload files id: upload_files - uses: mkruszyx/openvino/.github/actions/coverage_toolkit@composite-coverage + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master with: command: resolve-upload-files workspace: ${{ github.workspace }}