CI-python.yml

name: CI Python

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  schedule:
    - cron:  '30 5 * * *'

jobs:
  ci-python:
    strategy:
      matrix:
        packageDirectory: ["ml_wrappers"]
        operatingSystem: [ubuntu-latest, macos-latest, windows-latest]
        pythonVersion: ['3.9', '3.10', '3.11']
        openaiVersion: ['0.28.1', 'openai-latest']
        exclude:
          - openaiVersion: '0.28.1'
            pythonVersion: '3.9'
          - openaiVersion: '0.28.1'
            pythonVersion: '3.10'
          - openaiVersion: '0.28.1'
            operatingSystem: 'macos-latest'
          - openaiVersion: '0.28.1'
            operatingSystem: 'windows-latest'
          # Exclude Windows + Python 3.9 + openai-latest due to blis/spacy/fastai build failures
          - openaiVersion: 'openai-latest'
            pythonVersion: '3.9'
            operatingSystem: 'windows-latest'
          # Exclude macOS + Python 3.9 due to gRPC mutex deadlock
          - openaiVersion: 'openai-latest'
            pythonVersion: '3.9'
            operatingSystem: 'macos-latest'

    runs-on: ${{ matrix.operatingSystem }}

    steps:
    - uses: actions/checkout@v4
    - uses: conda-incubator/setup-miniconda@v3
      with:
        auto-update-conda: true
        python-version: ${{ matrix.pythonVersion }}
        channels: conda-forge

    - if: ${{ matrix.operatingSystem == 'macos-latest' }}
      name: Use Homebrew to install libomp on MacOS
      shell: bash -l {0}
      run: |
        brew install libomp

    - if: ${{ matrix.operatingSystem == 'windows-latest' }}
      name: Install pytorch on windows for python 3.9 to 3.11
      shell: bash -l {0}
      run: |
        conda install --yes --quiet pytorch torchvision captum cpuonly "libtiff<4.5.0" -c pytorch -c conda-forge --strict-channel-priority

    - if: ${{ matrix.operatingSystem == 'ubuntu-latest' }}
      name: Install pytorch on ubuntu for python 3.9 to 3.11
      shell: bash -l {0}
      run: |
        conda install --yes --quiet pytorch torchvision captum cpuonly -c pytorch -c conda-forge --strict-channel-priority

    - if: ${{ matrix.operatingSystem == 'macos-latest' }}
      name: Install pytorch on MacOS for python 3.9 to 3.11
      shell: bash -l {0}
      run: |
        conda install --yes --quiet pytorch torchvision captum -c pytorch -c conda-forge

    - if: ${{ matrix.operatingSystem == 'macos-latest' }}
      name: Install lightgbm from conda on MacOS
      shell: bash -l {0}
      run: |
        conda install --yes "lightgbm>=4.5.0" -c conda-forge

    - name: Install backwards-compatible tf-keras for transformers
      shell: bash -l {0}
      run: |
        pip install tf-keras

    - name: Install package
      shell: bash -l {0}
      run: |
        pip install -e ./python

    - name: Install dev dependencies
      shell: bash -l {0}
      run: |
        pip install -r requirements-dev.txt

    - name: Install test dependencies
      shell: bash -l {0}
      run: |
        pip install -r requirements-test.txt

    - name: Upgrade matplotlib for NumPy 2.x compatibility
      shell: bash -l {0}
      run: |
        pip install --upgrade "matplotlib>=3.9.0"

    - if: ${{ matrix.openaiVersion != 'openai-latest' }}
      name: Install openai version ${{ matrix.openaiVersion }}
      shell: bash -l {0}
      run: |
        pip install openai==${{ matrix.openaiVersion }}

    - if: ${{ matrix.operatingSystem == 'macos-latest' }}
      name: Set macOS environment variables to prevent hangs
      shell: bash -l {0}
      run: |
        # GRPC_POLL_STRATEGY=poll - Forces gRPC to use the poll strategy instead of
        # platform-specific mechanisms that can deadlock on macOS
        echo "GRPC_POLL_STRATEGY=poll" >> $GITHUB_ENV
        # TF_ENABLE_ONEDNN_OPTS=0 - Disables oneDNN optimizations that can cause issues on macOS
        echo "TF_ENABLE_ONEDNN_OPTS=0" >> $GITHUB_ENV
        # OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES - Disables the Objective-C fork safety check
        # that can cause hangs when Python multiprocessing interacts with certain libraries
        echo "OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES" >> $GITHUB_ENV

    - name: Test with pytest
      shell: bash -l {0}
      run: |
        pytest ./tests/main -s -v --durations=10 --cov='ml_wrappers' --cov-report=xml --cov-report=html

    - name: Upload code coverage results
      uses: actions/upload-artifact@v4
      with:
        name: ${{ matrix.packageDirectory }}-${{ matrix.openaiVersion }}-${{ matrix.pythonVersion }}-${{ matrix.operatingSystem }}-code-coverage-results
        path: htmlcov
      # Use always() to always run this step to publish test results when there are test failures
      if: ${{ always() }}

    - if: ${{ (matrix.operatingSystem == 'windows-latest') && (matrix.pythonVersion == '3.9') }}
      name: Upload to codecov
      id: codecovupload1
      uses: codecov/codecov-action@v3
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        directory: .
        env_vars: OS,PYTHON
        fail_ci_if_error: false
        files: ./coverage.xml
        flags: unittests
        name: codecov-umbrella
        verbose: true

    - if: ${{ (steps.codecovupload1.outcome == 'failure') && (matrix.pythonVersion == '3.9') && (matrix.operatingSystem == 'windows-latest') }}
      name: Retry upload to codecov
      id: codecovupload2
      uses: codecov/codecov-action@v3
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        directory: .
        env_vars: OS,PYTHON
        fail_ci_if_error: false
        files: ./coverage.xml
        flags: unittests
        name: codecov-umbrella
        verbose: true

    - name: Set codecov status
      if: ${{ (matrix.pythonVersion == '3.9') && (matrix.operatingSystem == 'windows-latest') }}
      shell: bash
      run: |
        if ${{ (steps.codecovupload1.outcome == 'success') || (steps.codecovupload2.outcome == 'success') }} ; then
          echo fine
        else
          exit 1
        fi