feat(helm): add API component and Gateway API HTTPRoute support

.github/workflows/ci.yml

@@ -87,6 +87,9 @@ jobs:
       - name: Install dependencies
         run: uv sync --all-extras
 
+      - name: Set up providers
+        run: uv run ref providers setup
+
       - name: Fetch test data
         run: |
           mkdir -p "$REF_TEST_DATA_DIR"

.github/workflows/packaging.yaml

@@ -0,0 +1,172 @@
+name: Packaging
+
+on:
+  pull_request:
+  workflow_dispatch:
+  push:
+    branches:
+      - "main"
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+  packages: write
+  attestations: write
+  id-token: write
+
+jobs:
+  changes:
+    name: Detect Changes
+    runs-on: ubuntu-latest
+    outputs:
+      helm: ${{ steps.filter.outputs.helm }}
+    steps:
+      - uses: actions/checkout@v6
+      # - uses: dorny/paths-filter@v3
+      #   id: filter
+      #   with:
+      #     filters: |
+      #       helm:
+      #         - 'helm/**'
+
+
+  helm:
+    name: Helm Chart
+    runs-on: ubuntu-latest
+    needs: [changes]
+    if: needs.changes.outputs.helm == 'true' || github.event_name != 'pull_request'
+    permissions:
+      packages: write
+    outputs:
+      generated-semver: ${{ steps.semantic-version.outputs.generated-semver }}
+    steps:
+    - uses: actions/checkout@v6
+    - uses: actions/setup-python@v6
+    - name: Install jq
+      run: |
+        sudo apt-get install --yes jq
+    - name: Install yq
+      run: |
+        pip install yq
+    - name: Generate SemVer
+      id: semantic-version
+      run: |
+        CHART_VERSION=$(yq -r '.version' helm/Chart.yaml)
+        if [ "${{ github.event_name }}" = "pull_request" ]; then
+          LOCAL_SEGMENT=+pr-${{ github.event.pull_request.number }}
+        elif [ "${{ github.ref_type }}" = "tag" ]; then
+          LOCAL_SEGMENT=""
+        else
+          SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
+          LOCAL_SEGMENT=+${SHORT_SHA}
+        fi
+        GENERATED_VERSION=${CHART_VERSION}${LOCAL_SEGMENT}
+        yq -Y -i ".version = \"$GENERATED_VERSION\"" helm/Chart.yaml
+        echo "generated-semver=$GENERATED_VERSION" >> $GITHUB_OUTPUT
+    - name: Chart | Push
+      uses: appany/helm-oci-chart-releaser@v0.5.0
+      with:
+        name: climate-ref-aft
+        repository: climate-ref/charts
+        tag: ${{ steps.semantic-version.outputs.generated-semver }}
+        path: helm
+        registry: ghcr.io
+        registry_username: ${{ github.actor }}
+        registry_password: ${{ secrets.GITHUB_TOKEN }}
+        update_dependencies: 'true'
+
+  test:
+    name: Test Helm Deployment
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    if: github.event_name == 'pull_request' && needs.changes.outputs.helm == 'true'
+    needs: [changes, helm]
+    steps:
+    - uses: actions/checkout@v6
+    - name: Cache Sample Data (Restore)
+      id: cache-sample-data-restore
+      uses: actions/cache/restore@v5
+      with:
+        path: ${{ github.workspace }}/cache/ref-config
+        key: ${{ runner.os }}-sample-data
+        enableCrossOsArchive: true
+    - name: Set permissions for cached data
+      run: |
+        sudo install -d --owner=1000 --group=1000 ${GITHUB_WORKSPACE}/cache/ref-config
+    - name: Start minikube
+      uses: medyagh/setup-minikube@latest
+      with:
+        mount-path: '${{ github.workspace }}/cache/ref-config:/cache/ref-config'
+    - name: Set up Helm
+      uses: azure/setup-helm@v4.3.1
+    - name: Install Chart
+      run: |
+        helm install test oci://ghcr.io/climate-ref/charts/climate-ref-aft \
+          --version=${{ needs.helm.outputs.generated-semver }} \
+          --set defaults.image.tag=pr-${{ github.event.pull_request.number }} \
+          -f helm/ci/gh-actions-values.yaml
+
+        sleep 60
+        kubectl get pods
+        echo ""
+        kubectl describe pod -l app.kubernetes.io/component=pmp
+        echo ""
+        kubectl logs -l app.kubernetes.io/component=pmp
+    - name: Run Migrations
+      run: |
+        kubectl exec deployment/test-climate-ref-aft-orchestrator -- ref config list
+    - name: Initialize Providers
+      run: |
+        # Imports ilamb3 which tries to create /home/app/.config/ilamb3 on import, no way to tell it to live somewhere else
+        # First, set up all providers without fetching data (handles conda envs)
+        kubectl exec deployment/test-climate-ref-aft-orchestrator -- ref providers setup --skip-data --skip-validate
+        # Fetch data for providers except esmvaltool (ERA5 data is too large for CI)
+        kubectl exec deployment/test-climate-ref-aft-orchestrator -- ref providers setup --provider pmp
+        kubectl exec deployment/test-climate-ref-aft-orchestrator -- ref providers setup --provider ilamb
+    - name: Fetch Test Data
+      run: |
+        kubectl exec deployment/test-climate-ref-aft-orchestrator -- ref datasets fetch-data --registry sample-data --output-directory /ref/sample-data
+
+    - name: Cache Sample Data (Save)
+      uses: actions/cache/save@v5
+      with:
+        path: ${{ github.workspace }}/cache/ref-config
+        key: ${{ runner.os }}-sample-data
+
+    - name: Ingest Test Data (CMIP6)
+      run: |
+        kubectl exec deployment/test-climate-ref-aft-orchestrator -- ref -v datasets ingest --source-type cmip6 /ref/sample-data/CMIP6
+    - name: Ingest Test Data (obs4mips)
+      run: |
+        kubectl exec deployment/test-climate-ref-aft-orchestrator -- ref -v datasets ingest --source-type obs4mips /ref/sample-data/obs4REF
+    - name: Simple Solve
+      run: |
+        # Use a fixed set of fast diagnostics to keep CI times predictable
+        kubectl exec deployment/test-climate-ref-aft-orchestrator -- ref -v solve --timeout 720 --one-per-provider \
+          --diagnostic global-mean-timeseries \
+          --diagnostic annual-cycle \
+          --diagnostic gpp-wecann
+    - name: Capture Worker Logs on Failure
+      if: failure()
+      run: |
+        echo "=== PMP Worker Logs ==="
+        kubectl logs -l app.kubernetes.io/component=pmp --tail=500 || true
+        echo ""
+        echo "=== ESMValTool Worker Logs ==="
+        kubectl logs -l app.kubernetes.io/component=esmvaltool --tail=500 || true
+        echo ""
+        echo "=== ILAMB Worker Logs ==="
+        kubectl logs -l app.kubernetes.io/component=ilamb --tail=500 || true
+        echo ""
+        echo "=== Example Worker Logs ==="
+        kubectl logs -l app.kubernetes.io/component=example --tail=500 || true
+        echo ""
+        echo "=== Orchestrator Worker Logs ==="
+        kubectl logs -l app.kubernetes.io/component=orchestrator --tail=500 || true
+        echo ""
+        echo "=== Flower Logs ==="
+        kubectl logs -l app.kubernetes.io/component=flower --tail=500 || true
+        echo ""
+        echo "=== Dragonfly Logs ==="
+        kubectl logs -l app.kubernetes.io/name=dragonfly --tail=500 || true

.github/workflows/release.yml

@@ -34,7 +34,7 @@ jobs:
       - name: Push chart
         run: |
           CHART_VERSION=$(grep '^version:' helm/Chart.yaml | awk '{print $2}')
-          helm push ref-${CHART_VERSION}.tgz oci://ghcr.io/climate-ref/charts
+          helm push climate-ref-aft-${CHART_VERSION}.tgz oci://ghcr.io/climate-ref/charts
 
   release:
     name: GitHub Release

README.md

@@ -23,9 +23,11 @@ The AFT deployment brings together independently versioned packages:
 | `climate-ref-core` | [Climate-REF/climate-ref](https://github.com/Climate-REF/climate-ref) | Core library with base classes and interfaces |
 | `climate-ref` | [Climate-REF/climate-ref](https://github.com/Climate-REF/climate-ref) | Main application, CLI, database, solver |
 | `climate-ref-celery` | [Climate-REF/climate-ref](https://github.com/Climate-REF/climate-ref) | Celery executor for distributed execution |
-| `climate-ref-esmvaltool` | [Climate-REF/climate-ref-esmvaltool](https://github.com/Climate-REF/climate-ref-esmvaltool) | ESMValTool diagnostic provider |
-| `climate-ref-pmp` | [Climate-REF/climate-ref-pmp](https://github.com/Climate-REF/climate-ref-pmp) | PCMDI Metrics Package diagnostic provider |
-| `climate-ref-ilamb` | [Climate-REF/climate-ref-ilamb](https://github.com/Climate-REF/climate-ref-ilamb) | ILAMB diagnostic provider |
+| `climate-ref-esmvaltool` | [Climate-REF/climate-ref](https://github.com/Climate-REF/climate-ref) | ESMValTool diagnostic provider |
+| `climate-ref-pmp` | [Climate-REF/climate-ref](https://github.com/Climate-REF/climate-ref) | PCMDI Metrics Package diagnostic provider |
+| `climate-ref-ilamb` | [Climate-REF/climate-ref](https://github.com/Climate-REF/climate-ref) | ILAMB diagnostic provider |
+
+Note: we intend to split the providers out into their own repositories in the coming weeks.
 
 ## Versioning
 
@@ -53,7 +55,7 @@ bash scripts/smoke-test.sh
 helm install ref ./helm -f helm/local-test-values.yaml
 
 # Or from the OCI registry
-helm install ref oci://ghcr.io/climate-ref/charts/ref --version 0.9.1
+helm install ref oci://ghcr.io/climate-ref/charts/climate-ref-aft --version 0.9.1
 ```
 
 ### Integration Tests
@@ -74,6 +76,7 @@ uv run pytest tests/ -v --slow
 | Workflow | Trigger | What It Does |
 |----------|---------|--------------|
 | `ci.yml` | Push, PR | Lint, install pinned versions, run integration tests |
+| `packaging.yaml` | Push, PR | Helm chart OCI publish and minikube deployment test |
 | `nightly.yml` | Scheduled (daily) | Test against latest versions of all components |
 | `release.yml` | Tag push | Publish Helm chart, create GitHub release |
 

changelog/3.feature.md

@@ -0,0 +1 @@
+Add ref-app API component and Gateway API HTTPRoute support to the Helm chart.

docker/docker-compose.yaml

@@ -26,6 +26,21 @@ services:
     ports:
       - "5432:5432"
 
+  ref-app:
+    image: ghcr.io/climate-ref/climate-ref-frontend:main
+    platform: linux/amd64
+    restart: always
+    depends_on:
+      - postgres
+      - redis
+    environment:
+      - REF_DATABASE_URL=postgresql://postgres:example@postgres:5432/postgres
+      - ENVIRONMENT=local
+      - SECRET_KEY=local-dev-secret
+      - REF_CONFIGURATION=/app/.ref
+    ports:
+      - "8000:8000"
+
   flower:
     image: mher/flower:2.0.1
     restart: always
@@ -40,7 +55,8 @@ services:
 
   # Base worker to track the executions of async tasks
   climate-ref:
-    image: ghcr.io/climate-ref/climate-ref:latest
+    image: ghcr.io/climate-ref/climate-ref:main
+    platform: linux/amd64
     restart: always
     depends_on:
       - postgres

helm/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
-name: ref
-description: A Helm chart for Kubernetes
+name: climate-ref-aft
+description: A Helm chart for deploying the Climate REF application for the CMIP7 Assessment Fasttrack (AFT)
 type: application
 
 # This is the chart version. This version number should be incremented each time you make changes