ci.yml

name: CI

env:
  ACT: ""
  
on:
  push:
    branches: [ main ]
  pull_request:

jobs:
  # ---------- fast checks: lint, type, unit ----------
  checks:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        if: ${{ env.ACT != 'true' }}
        uses: actions/checkout@v4

      # uv + Python 3.12
      - name: Setup uv (and Python)
        uses: astral-sh/setup-uv@v5
        with:
          python-version: "3.12"
          enable-cache: true
      - name: Diagnose Working Directory
        run: |
          echo "PWD=$(pwd)"
          ls -la
          test -f pyproject.toml || (echo "pyproject.toml fehlt!" && exit 1)

      # Install Dev extras (ruff, mypy, pytest, …)
      - name: Sync deps (dev)
        run: uv sync --extra dev --frozen

      - name: Show tool versions
        run: |
          uv run python -V
          uv run ruff --version
          uv run mypy --version
          uv run pytest --version

      - name: "Debug: commit & tree"
        run: |
          echo "PWD=$(pwd)"
          git rev-parse --short HEAD
          git log -1 --stat

      - name: Ruff (lint)
        run: uv run ruff check src tests --no-cache --output-format=github

      - name: Mypy (type check)
        run: uv run mypy src --config-file=pyproject.toml

      - name: Unit tests (fast)
        env:
          PYTHONWARNINGS: default
        run: uv run pytest -q tests -m "unit and not (postgres or databricks_spark or bigquery or snowflake)" --maxfail=1

  # ---------- Engine-specific unit slices (require optional extras) ----------
  unit-matrix:
    runs-on: ubuntu-latest
    needs: checks
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: postgres
            extra: postgres
            marker: "unit and postgres"
          - name: databricks_spark
            extra: spark
            marker: "unit and databricks_spark"
            java: true
          - name: bigquery
            extra: bigquery_bf
            marker: "unit and bigquery"

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup uv (and Python)
        uses: astral-sh/setup-uv@v5
        with:
          python-version: "3.12"
          enable-cache: true

      - name: Sync deps (dev + extra)
        run: uv sync --extra dev --extra ${{ matrix.extra }} --frozen

      - name: Setup Java for Spark
        if: matrix.java == true
        uses: actions/setup-java@v4
        with:
          distribution: temurin
          java-version: '17'

      - name: Run unit tests for engine
        env:
          PYTHONWARNINGS: default
        run: uv run pytest -q tests -m "${{ matrix.marker }}" --maxfail=1

  # ---------- Examples: Integration Tests ----------
  examples-matrix:
    runs-on: ubuntu-latest
    needs: checks
    strategy:
      fail-fast: false
      matrix:
        include:
          # ---------- DuckDB examples ----------
          - engine: duckdb
            extra: ""
            example: api_demo
            env_file: examples/api_demo/.env.dev_duckdb
          - engine: duckdb
            extra: ""
            example: basic_demo
            env_file: examples/basic_demo/.env.dev_duckdb
          - engine: duckdb
            extra: ""
            example: cache_demo
            env_file: examples/cache_demo/.env.dev_duckdb
          - engine: duckdb
            extra: ""
            example: ci_demo
            env_file: examples/ci_demo/.env.dev_duckdb
          - engine: duckdb
            extra: ""
            example: dq_demo
            env_file: examples/dq_demo/.env.dev_duckdb
          - engine: duckdb
            extra: ""
            example: hooks_demo
            env_file: examples/hooks_demo/.env.dev_duckdb
          - engine: duckdb
            extra: ""
            example: incremental_demo
            env_file: examples/incremental_demo/.env.dev_duckdb
          - engine: duckdb
            extra: ""
            example: macros_demo
            env_file: examples/macros_demo/.env.dev_duckdb
          - engine: duckdb
            extra: ""
            example: materializations_demo
            env_file: examples/materializations_demo/.env.dev_duckdb
          # ---------- Postgres examples ----------
          - engine: postgres
            extra: "postgres"
            example: api_demo
            env_file: examples/api_demo/.env.dev_postgres
          - engine: postgres
            extra: "postgres"
            example: basic_demo
            env_file: examples/basic_demo/.env.dev_postgres
          - engine: postgres
            extra: "postgres"
            example: ci_demo
            env_file: examples/ci_demo/.env.dev_postgres
          - engine: postgres
            extra: "postgres"
            example: cache_demo
            env_file: examples/cache_demo/.env.dev_postgres
          - engine: postgres
            extra: "postgres"
            example: dq_demo
            env_file: examples/dq_demo/.env.dev_postgres
          - engine: postgres
            extra: "postgres"
            example: hooks_demo
            env_file: examples/hooks_demo/.env.dev_postgres
          - engine: postgres
            extra: "postgres"
            example: incremental_demo
            env_file: examples/incremental_demo/.env.dev_postgres
          - engine: postgres
            extra: "postgres"
            example: macros_demo
            env_file: examples/macros_demo/.env.dev_postgres
          - engine: postgres
            extra: "postgres"
            example: materializations_demo
            env_file: examples/materializations_demo/.env.dev_postgres
          # ---------- Spark examples ----------
          - engine: databricks_spark
            extra: "spark"
            example: api_demo
            java: true
            env_file: examples/api_demo/.env.dev_databricks
          - engine: databricks_spark
            extra: "spark"
            example: basic_demo
            java: true
            env_file: examples/basic_demo/.env.dev_databricks
          - engine: databricks_spark
            extra: "spark"
            example: cache_demo
            java: true
            env_file: examples/cache_demo/.env.dev_databricks
          - engine: databricks_spark
            extra: "spark"
            example: ci_demo
            java: true
            env_file: examples/ci_demo/.env.dev_databricks
          - engine: databricks_spark
            extra: "spark"
            example: dq_demo
            java: true
            env_file: examples/dq_demo/.env.dev_databricks
          - engine: databricks_spark
            extra: "spark"
            example: hooks_demo
            java: true
            env_file: examples/hooks_demo/.env.dev_databricks
          - engine: databricks_spark
            extra: "spark"
            example: incremental_demo
            java: true
            env_file: examples/incremental_demo/.env.dev_databricks
          - engine: databricks_spark
            extra: "spark"
            example: macros_demo
            java: true
            env_file: examples/macros_demo/.env.dev_databricks
          - engine: databricks_spark
            extra: "spark"
            example: materializations_demo
            java: true
            env_file: examples/materializations_demo/.env.dev_databricks

    services:
      postgres:
        image: postgres:16
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: fastflowtransform
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U postgres"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup uv (and Python)
        uses: astral-sh/setup-uv@v5
        with:
          python-version: "3.12"
          enable-cache: true

      - name: Sync deps (dev + extra)
        run: |
          extras="--extra dev"
          if [ -n "${{ matrix.extra }}" ]; then
            extras="$extras --extra ${{ matrix.extra }}"
          fi
          uv sync $extras --frozen

      - name: Setup Java for Spark
        if: matrix.engine == 'databricks_spark'
        uses: actions/setup-java@v4
        with:
          distribution: temurin
          java-version: '17'

      - name: Run example/integration tests for engine
        run: |
          echo "Running integration tests for engine=${{ matrix.engine }} example=${{ matrix.example }}"
          set -a
          unset FF_PG_DSN FF_PG_SCHEMA
          if [ -f "${{ matrix.env_file }}" ]; then
            source "${{ matrix.env_file }}"
          fi
          if [ "${{ matrix.engine }}" = "postgres" ]; then
            export FF_PG_SCHEMA="${{ matrix.example }}"
            export FF_PG_DSN="${FF_PG_DSN:-postgresql+psycopg://postgres:postgres@localhost:5432/fastflowtransform}"
          fi
          set +a
          uv run pytest -m "integration and ${{ matrix.engine }}" \
            -vv --show-capture=all --log-cli-level=INFO \
            -k "${{ matrix.example }} and ${{ matrix.engine }}" \
            --maxfail=1 -q tests/integration/examples/test_examples_matrix.py