diff --git a/.github/coverage/tests_python.yml b/.github/coverage/tests_python.yml new file mode 100644 index 00000000000..60a80e0c7f4 --- /dev/null +++ b/.github/coverage/tests_python.yml @@ -0,0 +1,147 @@ +schema_version: 1 +suite: python +reporting: + label: Python + artifact_group: python + artifact_name_template: coverage-{suite}-{lane} + selection_env: PY_TEST_NAMES + stats_file: python-coverage-stats.env + duration_file: python-test-durations.csv + coverage_file: python-coverage.xml + debug_dirs: + - src: .tmp/python-coverage + dest: python-coverage-debug + uploads: + - name_template: NNCF Python, {lane} + file: python-coverage.xml + key_template: nncf_python_{lane}_xml + flag_template: nncf-python-{lane} + +tests: + - name: common + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -ra --durations=30 -n2 ${PYTEST_COVERAGE_ARGS} + tests/common --junitxml nncf-tests.xml + + - name: onnx + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -ra --durations=30 -n4 ${PYTEST_COVERAGE_ARGS} + tests/onnx --junitxml nncf-tests.xml + + - name: openvino + kind: command + profiles: [cpu] + command: >- + ONEDNN_MAX_CPU_ISA=AVX2 python3 -m pytest -ra --durations=30 -n4 --dist loadscope + ${PYTEST_COVERAGE_ARGS} tests/openvino --junitxml nncf-tests.xml + + - name: torch_cpu + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -ra -n2 --durations=30 ${PYTEST_COVERAGE_ARGS} + tests/torch -m "not cuda" --junitxml nncf-tests.xml + + - name: torch_cuda + kind: command + profiles: [gpu] + command: >- + python3 -m pytest -ra --durations=30 ${PYTEST_COVERAGE_ARGS} + tests/torch -m cuda --junitxml nncf-tests.xml + + - name: gptqmodel_cuda + kind: command + profiles: [gpu] + command: >- + python3 -m pytest -ra --durations=30 ${PYTEST_COVERAGE_ARGS} + tests/integration/gptq_model/ --junitxml nncf-tests.xml + + - name: tools + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -ra --durations=30 ${PYTEST_COVERAGE_ARGS} + tests/tools --junitxml nncf-tests.xml + + - name: executorch + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -ra --durations=30 ${PYTEST_COVERAGE_ARGS} + tests/executorch --junitxml nncf-tests.xml + + - name: wc_1 + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -s -ra ${PYTEST_COVERAGE_ARGS} + tests/post_training/test_quantize_conformance.py::test_weight_compression + --junitxml=nncf-tests.xml + --durations-path=tests/post_training/data/wc_test_durations.json + --splitting-algorithm=least_duration + --splits 6 + --group 1 + + - name: wc_2 + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -s -ra ${PYTEST_COVERAGE_ARGS} + tests/post_training/test_quantize_conformance.py::test_weight_compression + --junitxml=nncf-tests.xml + --durations-path=tests/post_training/data/wc_test_durations.json + --splitting-algorithm=least_duration + --splits 6 + --group 2 + + - name: wc_3 + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -s -ra ${PYTEST_COVERAGE_ARGS} + tests/post_training/test_quantize_conformance.py::test_weight_compression + --junitxml=nncf-tests.xml + --durations-path=tests/post_training/data/wc_test_durations.json + --splitting-algorithm=least_duration + --splits 6 + --group 3 + + - name: wc_4 + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -s -ra ${PYTEST_COVERAGE_ARGS} + tests/post_training/test_quantize_conformance.py::test_weight_compression + --junitxml=nncf-tests.xml + --durations-path=tests/post_training/data/wc_test_durations.json + --splitting-algorithm=least_duration + --splits 6 + --group 4 + + - name: wc_5 + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -s -ra ${PYTEST_COVERAGE_ARGS} + tests/post_training/test_quantize_conformance.py::test_weight_compression + --junitxml=nncf-tests.xml + --durations-path=tests/post_training/data/wc_test_durations.json + --splitting-algorithm=least_duration + --splits 6 + --group 5 + + - name: wc_6 + kind: command + profiles: [cpu] + command: >- + python3 -m pytest -s -ra ${PYTEST_COVERAGE_ARGS} + tests/post_training/test_quantize_conformance.py::test_weight_compression + --junitxml=nncf-tests.xml + --durations-path=tests/post_training/data/wc_test_durations.json + --splitting-algorithm=least_duration + --splits 6 + --group 6 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 00000000000..990b4aa3cab --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,505 @@ +name: Code coverage + +on: + workflow_dispatch: + pull_request: + +permissions: read-all + +concurrency: + group: ${{ github.event_name == 'pull_request' && github.run_id || format('{0}-{1}', github.workflow, github.ref) }} + cancel-in-progress: false + +env: + PYTHON_VERSION: '3.12' + ENABLE_BRANCH_COVERAGE: 'false' + +jobs: + CoveragePythonCore: + name: Coverage Python ${{ matrix.test.name }} (CPU) + timeout-minutes: 120 + runs-on: ${{ matrix.test.runner }} + strategy: + fail-fast: false + matrix: + test: + - name: common + runner: ubuntu-latest + requirements: tests/common/requirements.txt + - name: onnx + runner: ubuntu-latest-8-cores + requirements: tests/onnx/requirements.txt + - name: openvino + runner: ubuntu-latest-8-cores + requirements: tests/openvino/requirements.txt + - name: torch_cpu + runner: ubuntu-latest + requirements: tests/torch/requirements.txt + - name: tools + runner: ubuntu-latest + requirements: tests/tools/requirements.txt + defaults: + run: + shell: bash + env: + TEST_PROFILE: cpu + OV_WORKSPACE: ${{ github.workspace }} + PY_TEST_NAMES: ${{ matrix.test.name }} + COVERAGE_WRITE_STEP_SUMMARY: 'false' + steps: + - name: Checkout sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + lfs: true + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + + - name: Install coverage Python dependencies + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master + with: + command: install-requirements + + - name: Install NNCF and test requirements + run: uv pip install --system . -r ${{ matrix.test.requirements }} + + - name: Run Python coverage tests + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master + with: + command: run-suite + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + suite: python + profile: cpu + lane: ${{ matrix.test.name }} + test-names: ${{ matrix.test.name }} + python-package: nncf + python-source-dir: src/nncf + python-path-alias: nncf + python-coverage-omit: | + */tests/* + */docs/* + */examples/* + */tools/* + */build/* + */.venv/* + */venv/* + + - name: Collect Python results + id: cpu_artifact + if: ${{ always() }} + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master + with: + command: collect-results + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + suite: python + profile: cpu + lane: ${{ matrix.test.name }} + test-names: ${{ matrix.test.name }} + + - name: Upload Python artifact + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ steps.cpu_artifact.outputs['artifact-name'] }} + path: ${{ steps.cpu_artifact.outputs.dir }} + if-no-files-found: warn + retention-days: 7 + compression-level: 0 + overwrite: true + + CoveragePythonExecuTorch: + name: Coverage Python ExecuTorch (CPU) + timeout-minutes: 60 + runs-on: ubuntu-latest-8-cores + defaults: + run: + shell: bash + env: + TEST_PROFILE: cpu + OV_WORKSPACE: ${{ github.workspace }} + PY_TEST_NAMES: executorch + COVERAGE_WRITE_STEP_SUMMARY: 'false' + DEBIAN_FRONTEND: noninteractive + steps: + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get --assume-yes install gcc g++ build-essential ninja-build libgl1-mesa-dev libglib2.0-0 + + - name: Checkout sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + lfs: true + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.10.14' + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + + - name: Install coverage Python dependencies + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master + with: + command: install-requirements + + - name: Install NNCF and test requirements + run: pip install . -r tests/executorch/requirements.txt + + - name: Run ExecuTorch coverage tests + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master + with: + command: run-suite + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + suite: python + profile: cpu + lane: executorch + test-names: executorch + python-package: nncf + python-source-dir: src/nncf + python-path-alias: nncf + python-coverage-omit: | + */tests/* + */docs/* + */examples/* + */tools/* + */build/* + */.venv/* + */venv/* + + - name: Collect ExecuTorch results + id: cpu_artifact + if: ${{ always() }} + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master + with: + command: collect-results + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + suite: python + profile: cpu + lane: executorch + test-names: executorch + + - name: Upload ExecuTorch artifact + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ steps.cpu_artifact.outputs['artifact-name'] }} + path: ${{ steps.cpu_artifact.outputs.dir }} + if-no-files-found: warn + retention-days: 7 + compression-level: 0 + overwrite: true + + CoveragePythonWeightCompression: + name: Coverage Python Weight Compression ${{ matrix.group }}/6 (CPU) + timeout-minutes: 120 + runs-on: ubuntu-latest-16-cores + strategy: + fail-fast: false + matrix: + group: [1, 2, 3, 4, 5, 6] + defaults: + run: + shell: bash + env: + TEST_PROFILE: cpu + OV_WORKSPACE: ${{ github.workspace }} + PY_TEST_NAMES: wc_${{ matrix.group }} + COVERAGE_WRITE_STEP_SUMMARY: 'false' + TQDM_DISABLE: 1 + HOME_HF: /home/runner/hf_home + steps: + - name: Checkout sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + + - name: Install coverage Python dependencies + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master + with: + command: install-requirements + + - name: Install NNCF and test requirements + run: uv pip install --system . -r tests/post_training/requirements.txt + + - name: Run weight-compression coverage tests + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master + with: + command: run-suite + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + suite: python + profile: cpu + lane: wc_${{ matrix.group }} + test-names: wc_${{ matrix.group }} + python-package: nncf + python-source-dir: src/nncf + python-path-alias: nncf + python-coverage-omit: | + */tests/* + */docs/* + */examples/* + */tools/* + */build/* + */.venv/* + */venv/* + + - name: Collect weight-compression results + id: cpu_artifact + if: ${{ always() }} + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master + with: + command: collect-results + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + suite: python + profile: cpu + lane: wc_${{ matrix.group }} + test-names: wc_${{ matrix.group }} + + - name: Upload weight-compression artifact + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ steps.cpu_artifact.outputs['artifact-name'] }} + path: ${{ steps.cpu_artifact.outputs.dir }} + if-no-files-found: warn + retention-days: 7 + compression-level: 0 + overwrite: true + + CoveragePythonCuda: + name: Coverage Python ${{ matrix.test.name }} (CUDA) + timeout-minutes: 120 + runs-on: aks-linux-6-cores-55gb-gpu-a10 + strategy: + fail-fast: false + matrix: + test: + - name: torch_cuda + requirements: tests/torch/requirements.txt + extra_requirements: '' + - name: gptqmodel_cuda + requirements: tests/integration/gptq_model/requirements.txt + extra_requirements: tests/integration/gptq_model/requirements_extra.txt + defaults: + run: + shell: bash + env: + TEST_PROFILE: gpu + OV_WORKSPACE: ${{ github.workspace }} + PY_TEST_NAMES: ${{ matrix.test.name }} + COVERAGE_WRITE_STEP_SUMMARY: 'false' + PYTHONDONTWRITEBYTECODE: '1' + DEBIAN_FRONTEND: noninteractive + steps: + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get --assume-yes install build-essential ninja-build libgl1-mesa-dev libglib2.0-0 wget make + + - name: Download CUDA + run: | + wget -q https://developer.download.nvidia.com/compute/cuda/12.6.3/local_installers/cuda_12.6.3_560.35.05_linux.run + sudo sh cuda_12.6.3_560.35.05_linux.run --toolkit --silent + + - name: Runner info + continue-on-error: true + run: | + nvidia-smi + cat /proc/cpuinfo + + - name: Checkout sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + lfs: true + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + + - name: Install coverage Python dependencies + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master + with: + command: install-requirements + + - name: Install NNCF and test requirements + run: uv pip install --system . -r ${{ matrix.test.requirements }} + + - name: Install extra requirements + if: ${{ matrix.test.extra_requirements != '' }} + run: uv pip install --system -r ${{ matrix.test.extra_requirements }} + + - name: Check CUDA + run: python -c "import torch; print(torch.cuda.is_available())" + + - name: Run CUDA Python coverage tests + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master + with: + command: run-suite + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + suite: python + profile: gpu + lane: ${{ matrix.test.name }} + test-names: ${{ matrix.test.name }} + python-package: nncf + python-source-dir: src/nncf + python-path-alias: nncf + python-coverage-omit: | + */tests/* + */docs/* + */examples/* + */tools/* + */build/* + */.venv/* + */venv/* + + - name: Collect CUDA Python results + id: gpu_artifact + if: ${{ always() }} + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master + with: + command: collect-results + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + suite: python + profile: gpu + lane: ${{ matrix.test.name }} + test-names: ${{ matrix.test.name }} + + - name: Upload CUDA Python artifact + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ steps.gpu_artifact.outputs['artifact-name'] }} + path: ${{ steps.gpu_artifact.outputs.dir }} + if-no-files-found: warn + retention-days: 7 + compression-level: 0 + overwrite: true + + CoverageSummary: + name: Coverage Summary + if: ${{ always() && !cancelled() }} + needs: [CoveragePythonCore, CoveragePythonExecuTorch, CoveragePythonWeightCompression, CoveragePythonCuda] + outputs: + upload_matrix_json: ${{ steps.coverage_files.outputs['upload-matrix-json'] || '[]' }} + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + lfs: false + + - name: Install coverage Python dependencies + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master + with: + command: install-requirements + + - name: Download Python artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: coverage-python-* + path: ${{ github.workspace }}/artifacts/python + + - name: Generate final summary + if: ${{ always() }} + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master + with: + command: render-summary + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + selection: common,onnx,openvino,torch_cpu,tools,executorch,wc_1,wc_2,wc_3,wc_4,wc_5,wc_6,torch_cuda,gptqmodel_cuda + selected-lanes: common,onnx,openvino,torch_cpu,tools,executorch,wc_1,wc_2,wc_3,wc_4,wc_5,wc_6,torch_cuda,gptqmodel_cuda + + - name: Merge duration reports + if: ${{ always() }} + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master + with: + command: merge-durations + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + duration-output: ${{ github.workspace }}/coverage-test-durations-all.csv + + - name: Upload merged duration artifact + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-test-durations + path: coverage-test-durations-all.csv + if-no-files-found: warn + retention-days: 7 + compression-level: 0 + overwrite: true + + - name: Resolve Codecov upload files + id: coverage_files + if: ${{ always() }} + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master + with: + command: resolve-uploads + workspace: ${{ github.workspace }} + config-dir: ${{ github.workspace }}/.github/coverage + + CoverageCodecovUploads: + name: Coverage Codecov Upload (${{ matrix.upload.name }}) + if: ${{ always() && needs.CoverageSummary.result == 'success' && needs.CoverageSummary.outputs.upload_matrix_json != '[]' }} + needs: CoverageSummary + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + upload: ${{ fromJSON(needs.CoverageSummary.outputs.upload_matrix_json || '[]') }} + steps: + - name: Checkout sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + lfs: false + + - name: Download coverage artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: ${{ matrix.upload.artifact_pattern }} + path: ${{ github.workspace }}/artifacts/${{ matrix.upload.artifact_group }} + merge-multiple: false + + - name: Resolve Codecov upload files + id: upload_files + uses: openvinotoolkit/openvino/.github/actions/coverage_toolkit@master + with: + command: resolve-upload-files + workspace: ${{ github.workspace }} + artifact-group: ${{ matrix.upload.artifact_group }} + artifact-names-json: ${{ toJSON(matrix.upload.artifact_names) }} + upload-key: ${{ matrix.upload.key }} + upload-file: ${{ matrix.upload.file }} + + - name: Upload Codecov (${{ matrix.upload.name }}) + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ${{ steps.upload_files.outputs.files }} + disable_search: true + plugins: noop + flags: ${{ matrix.upload.flag }} + override_commit: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + override_pr: ${{ github.event_name == 'pull_request' && github.event.pull_request.number || '' }} + fail_ci_if_error: true + verbose: true