PIPELINE-3363: Improve repository structure. Upgrade to Python 3.12 and Apache Beam 2.71

.github/workflows/main.yml

@@ -2,7 +2,7 @@ name: main
 
 on:
   push:
-    branches: [master, develop]
+    branches: [main]
   pull_request:
 
 jobs:
@@ -15,7 +15,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4.1.0
         with:
-          python-version: 3.8
+          python-version: 3.12
 
       - name: Install flake8
         run: pip --disable-pip-version-check install flake8
@@ -24,10 +24,16 @@ jobs:
         run: flake8 --count
 
   tests:
-    runs-on: ubuntu-latest
+    name: tests - Python ${{ matrix.python-version }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: true
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        os: ["ubuntu-latest"]
+        python-version: ["3.13"]
+    env:
+      VENV: .venv
+      PACKAGE: pipe_segment
 
     steps:
       - uses: actions/checkout@v4
@@ -39,37 +45,36 @@ jobs:
           # cache option make the step fail if you don´t have requirements.txt or pyproject.toml on root.
           # https://github.com/actions/setup-python/issues/807.
 
-      - name: Install dependencies
+      - name: Create virtual environment
+        run: python -m venv $VENV
+
+      - name: Install package with test dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements/test.txt
-          pip install -e .
-    
+          . $VENV/bin/activate
+          make install-test
+          pip install .
+
       - name: Test with pytest
         run: |
-          make testintegration
+          . $VENV/bin/activate
+          pytest --cov=$PACKAGE --cov-report=xml
 
-      # Remove this step when this is done in tests-in-docker job.
       - name: Upload coverage reports to Codecov
-        if: ${{ matrix.python-version == '3.8' }}
+        if: github.ref == 'refs/heads/main' && matrix.python-version == '3.13'
         uses: codecov/codecov-action@v4.0.1
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
-# Use this job when base docker image is not pulled from GFW gcr.
-# tests-in-docker:
-#   runs-on: ubuntu-latest
-#   steps:
-#     - uses: actions/checkout@v4
-#     - name: Set up Python ${{ matrix.python-version }}
-#       uses: actions/setup-python@v4
-
-#     - name: Test with pytest in docker
-#       run: |
-#         docker volume create --name=gcp
-#         docker compose run test
-
-#     - name: Upload coverage reports to Codecov
-#       uses: codecov/codecov-action@v4.0.1
-#       with:
-#         token: ${{ secrets.CODECOV_TOKEN }}
+  tests-in-docker:
+    name: tests - Python 3.12 on Docker
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4.1.0
+
+      - name: Build docker image
+        run: make docker-build
+
+      - name: Run tests
+        run: make docker-ci-test

Dockerfile

@@ -1,50 +1,67 @@
 # ---------------------------------------------------------------------------------------
-# BASE
+# BUILDER
 # ---------------------------------------------------------------------------------------
-FROM python:3.8 AS base
+FROM python:3.12-slim-bookworm AS builder
 
-# Configure the working directory
-RUN mkdir -p /opt/project
-WORKDIR /opt/project
+VOLUME ["/root/.config"]
 
-# Copy files from official SDK image, including script/dependencies.
-COPY --from=apache/beam_python3.8_sdk:2.56.0 /opt/apache/beam /opt/apache/beam
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        gcc g++ build-essential git && \
+    rm -rf /var/lib/apt/lists/*
 
-# Install SDK. (needed for Python SDK)
-RUN pip install --no-cache-dir apache-beam[gcp]==2.56.0
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
 
-# Install application dependencies
-COPY requirements.txt /opt/requirements.txt
-RUN pip install --no-cache-dir -r /opt/requirements.txt
+WORKDIR /install
 
-# Set the entrypoint to Apache Beam SDK launcher.
-ENTRYPOINT ["/opt/apache/beam/boot"]
+COPY requirements.txt .
+
+RUN uv pip install --system --upgrade pip && \
+    uv pip install --system build && \
+    uv pip install --system --prefix=/install -r requirements.txt
+
+COPY pyproject.toml README.md MANIFEST.in ./
+COPY src ./src
 
+RUN uv pip install --system --prefix=/install .
 # ---------------------------------------------------------------------------------------
-# PROD
+# PRODUCTION IMAGE
 # ---------------------------------------------------------------------------------------
-FROM base AS prod
+FROM python:3.12-slim-bookworm AS prod
+
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# COPY PYTHON PACKAGES
+COPY --from=builder /install /usr/local
+
+# APACHE BEAM INTEGRATION
+COPY --from=apache/beam_python3.12_sdk:2.71.0 /opt/apache/beam /opt/apache/beam
+ENTRYPOINT ["/opt/apache/beam/boot"]
 
-# Install app package
-COPY . /opt/project
-RUN pip install .
+WORKDIR /opt/project
 
 # ---------------------------------------------------------------------------------------
-# DEV
+# DEVELOPMENT IMAGE
 # ---------------------------------------------------------------------------------------
-FROM base AS dev
+FROM builder AS dev
+
+WORKDIR /opt/project
 
-COPY ./requirements/dev.txt ./
-COPY ./requirements/test.txt ./
+COPY . .
+RUN uv pip install --system -e .[lint,dev,build] && \
+    uv pip install --system -r requirements-test.txt
+
+# ---------------------------------------------------------------------------------------
+# TEST IMAGE
+# ---------------------------------------------------------------------------------------
+FROM prod AS test
 
-RUN pip install --no-cache-dir -r dev.txt
-RUN pip install --no-cache-dir -r test.txt
+COPY ./requirements-test.txt .
+RUN pip install -r requirements-test.txt
 
-# Install app package
-COPY . /opt/project
-ENV PYTHONPATH /opt/project
-RUN cd /usr/local/lib/python3.8/site-packages && \
-    python /opt/project/setup.py develop
+COPY ./tests ./tests
 
-# Set the entrypoint to Apache Beam SDK launcher.
-ENTRYPOINT ["pipe"]
+# Suppress all warnings during tests
+# To see/address warnings, run tests in your development environment.
+ENV PYTHONWARNINGS=ignore

MANIFEST.in

@@ -1,7 +1,6 @@
 include LICENSE
-include CHANGES.md
-include MANIFEST.in
 include README.md
-include setup.py
-recursive-include assets *
-recursive-include pipe_segment *
+
+recursive-include src/**/assets *
+recursive-exclude * __pycache__
+recursive-exclude * *.py[cod]