loculus-project · chaoran-chen · Jun 18, 2026
diff --git a/.github/workflows/build-arm-images.yaml b/.github/workflows/build-arm-images.yaml
@@ -75,3 +75,15 @@ jobs:
     uses: ./.github/workflows/preprocessing-nextclade-image.yml
     with:
       build_arm: true
+  trigger-config-loader:
+    needs: should-build
+    if: needs.should-build.outputs.should_run == 'true'
+    uses: ./.github/workflows/config-loader-image.yml
+    with:
+      build_arm: true
+  trigger-config-adapter:
+    needs: should-build
+    if: needs.should-build.outputs.should_run == 'true'
+    uses: ./.github/workflows/config-adapter-image.yml
+    with:
+      build_arm: true
diff --git a/.github/workflows/config-adapter-image.yml b/.github/workflows/config-adapter-image.yml
@@ -0,0 +1,93 @@
+name: config-adapter-image
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+    inputs:
+      build_arm:
+        type: boolean
+        description: "Build for ARM as well"
+        default: false
+        required: false
+  workflow_call:
+    inputs:
+      build_arm:
+        type: string
+        description: "Build for ARM as well"
+        default: "false"
+        required: true
+env:
+  DOCKER_IMAGE_NAME: ghcr.io/loculus-project/config-adapter
+  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+  BUILD_ARM: ${{ github.event.inputs.build_arm || inputs.build_arm || github.ref == 'refs/heads/main' }}
+  sha: ${{ github.event.pull_request.head.sha || github.sha }}
+concurrency:
+  group: ci-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}-config-adapter
+  cancel-in-progress: true
+jobs:
+  config-adapter-image:
+    name: Build Config Adapter Docker Image
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      packages: write
+      contents: read
+      checks: read
+    steps:
+      - name: Shorten sha
+        run: echo "sha=${sha::7}" >> $GITHUB_ENV
+      - uses: actions/checkout@v6
+      - name: Add filename hash to environment
+        run: |
+          find config-tools -type f -not -path 'config-tools/node_modules/*' -print | sort | sha256sum > config-tools/filename_hash
+          cat config-tools/filename_hash
+      - name: Generate files hash
+        id: files-hash
+        run: |
+          DIR_HASH=$(echo -n ${{ hashFiles('config-tools/**', '.github/workflows/config-adapter-image.yml') }})
+          echo "DIR_HASH=$DIR_HASH${{ env.BUILD_ARM == 'true' && '-arm' || '' }}" >> $GITHUB_ENV
+          rm config-tools/filename_hash
+      - name: Setup Docker metadata
+        id: dockerMetadata
+        uses: docker/metadata-action@v6
+        with:
+          images: ${{ env.DOCKER_IMAGE_NAME }}
+          tags: |
+            type=raw,value=${{ env.DIR_HASH }}
+            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
+            type=raw,value=commit-${{ env.sha }}
+            type=raw,value=${{ env.BRANCH_NAME }}
+            type=raw,value=${{ env.BRANCH_NAME }}-arm,enable=${{ env.BUILD_ARM }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Check if image exists
+        id: check-image
+        run: |
+          EXISTS=$(docker manifest inspect ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} > /dev/null 2>&1 && echo "true" || echo "false")
+          echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+      - name: Build and push image if input files changed
+        if: env.CACHE_HIT == 'false'
+        uses: docker/build-push-action@v7
+        with:
+          context: ./config-tools
+          file: ./config-tools/Dockerfile.adapter
+          push: true
+          tags: ${{ steps.dockerMetadata.outputs.tags }}
+          cache-from: type=gha,scope=config-adapter-${{ github.ref }}
+          cache-to: type=gha,mode=max,scope=config-adapter-${{ github.ref }}
+          platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
+      - name: Retag and push existing image if cache hit
+        if: env.CACHE_HIT == 'true'
+        run: |
+          TAGS=(${{ steps.dockerMetadata.outputs.tags }})
+          for TAG in "${TAGS[@]}"; do
+            docker buildx imagetools create --tag $TAG ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }}
+          done
diff --git a/.github/workflows/config-loader-image.yml b/.github/workflows/config-loader-image.yml
@@ -0,0 +1,93 @@
+name: config-loader-image
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+    inputs:
+      build_arm:
+        type: boolean
+        description: "Build for ARM as well"
+        default: false
+        required: false
+  workflow_call:
+    inputs:
+      build_arm:
+        type: string
+        description: "Build for ARM as well"
+        default: "false"
+        required: true
+env:
+  DOCKER_IMAGE_NAME: ghcr.io/loculus-project/config-loader
+  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+  BUILD_ARM: ${{ github.event.inputs.build_arm || inputs.build_arm || github.ref == 'refs/heads/main' }}
+  sha: ${{ github.event.pull_request.head.sha || github.sha }}
+concurrency:
+  group: ci-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}-config-loader
+  cancel-in-progress: true
+jobs:
+  config-loader-image:
+    name: Build Config Loader Docker Image
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      packages: write
+      contents: read
+      checks: read
+    steps:
+      - name: Shorten sha
+        run: echo "sha=${sha::7}" >> $GITHUB_ENV
+      - uses: actions/checkout@v6
+      - name: Add filename hash to environment
+        run: |
+          find config-tools -type f -not -path 'config-tools/node_modules/*' -print | sort | sha256sum > config-tools/filename_hash
+          cat config-tools/filename_hash
+      - name: Generate files hash
+        id: files-hash
+        run: |
+          DIR_HASH=$(echo -n ${{ hashFiles('config-tools/**', '.github/workflows/config-loader-image.yml') }})
+          echo "DIR_HASH=$DIR_HASH${{ env.BUILD_ARM == 'true' && '-arm' || '' }}" >> $GITHUB_ENV
+          rm config-tools/filename_hash
+      - name: Setup Docker metadata
+        id: dockerMetadata
+        uses: docker/metadata-action@v6
+        with:
+          images: ${{ env.DOCKER_IMAGE_NAME }}
+          tags: |
+            type=raw,value=${{ env.DIR_HASH }}
+            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
+            type=raw,value=commit-${{ env.sha }}
+            type=raw,value=${{ env.BRANCH_NAME }}
+            type=raw,value=${{ env.BRANCH_NAME }}-arm,enable=${{ env.BUILD_ARM }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Check if image exists
+        id: check-image
+        run: |
+          EXISTS=$(docker manifest inspect ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }} > /dev/null 2>&1 && echo "true" || echo "false")
+          echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+      - name: Build and push image if input files changed
+        if: env.CACHE_HIT == 'false'
+        uses: docker/build-push-action@v7
+        with:
+          context: ./config-tools
+          file: ./config-tools/Dockerfile.loader
+          push: true
+          tags: ${{ steps.dockerMetadata.outputs.tags }}
+          cache-from: type=gha,scope=config-loader-${{ github.ref }}
+          cache-to: type=gha,mode=max,scope=config-loader-${{ github.ref }}
+          platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
+      - name: Retag and push existing image if cache hit
+        if: env.CACHE_HIT == 'true'
+        run: |
+          TAGS=(${{ steps.dockerMetadata.outputs.tags }})
+          for TAG in "${TAGS[@]}"; do
+            docker buildx imagetools create --tag $TAG ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DIR_HASH }}
+          done
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -110,6 +110,17 @@ jobs:
         run: cd integration-tests && npx playwright install --with-deps --only-shell chromium ${{ matrix.browser == 'firefox' && 'firefox' || '' }}
       - name: Wait for the pods to be ready
         run: ./.github/scripts/wait_for_pods_to_be_ready.py --timeout ${{ env.wait_timeout }}
+      # Phase 3.6: the config-loader Job posts fixture YAMLs to the backend admin API
+      # as a Helm post-install hook. SILO + LAPIS adapter init containers depend on
+      # those organisms existing in the backend, so wait for the Job to complete
+      # before continuing.
+      - name: Wait for config-loader Job to complete
+        run: |
+          kubectl wait --for=condition=complete --timeout=300s job/loculus-config-loader || {
+            echo '--- config-loader Job logs ---'
+            kubectl logs job/loculus-config-loader --all-containers --tail=200 || true
+            exit 1
+          }
       - name: Sleep for 10 secs
         run: sleep 10
       - name: Run Integration test

diff --git a/.github/workflows/update-argocd-metadata.yml b/.github/workflows/update-argocd-metadata.yml
@@ -99,6 +99,20 @@ jobs:
           max_attempts: 200
           retry_wait_seconds: 5
           command: docker manifest inspect ghcr.io/loculus-project/website:commit-${{ steps.get_sha.outputs.sha }}
+      - name: Wait for Config Loader Docker Image
+        uses: lewagon/wait-on-check-action@v1.7.0
+        with:
+          ref: ${{ github.sha }}
+          check-name: Build Config Loader Docker Image
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          wait-interval: 2
+      - name: Wait for Config Adapter Docker Image
+        uses: lewagon/wait-on-check-action@v1.7.0
+        with:
+          ref: ${{ github.sha }}
+          check-name: Build Config Adapter Docker Image
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          wait-interval: 2
       # End of wait block
       - name: Checkout External Repository
         uses: actions/checkout@v6

diff --git a/.github/workflows/website-image.yml b/.github/workflows/website-image.yml
@@ -30,7 +30,10 @@ jobs:
       - name: Generate files hash
         id: prepare
         run: |
-          DIR_HASH=$(echo -n ${{ hashFiles('website/**', '.github/workflows/website-image.yml') }})
+          # config-tools/src/schema/ is the source of truth for the canonical Zod
+          # schemas; website re-exports them. Changes there must invalidate the
+          # website image cache too.
+          DIR_HASH=$(echo -n ${{ hashFiles('website/**', 'config-tools/src/schema/**', '.github/workflows/website-image.yml') }})
           echo "dir-hash=$DIR_HASH" >> $GITHUB_OUTPUT
           echo "sha-short=${sha::7}" >> $GITHUB_OUTPUT
       - name: Login to GitHub Container Registry
@@ -89,6 +92,12 @@ jobs:
         uses: docker/build-push-action@v7
         with:
           context: ./website
+          # The website re-exports canonical Zod schemas from `config-tools/`,
+          # which lives outside its build context. Bring it in as a named build
+          # context so `../../../config-tools/...` relative imports resolve at
+          # build time (the Dockerfile does `COPY --from=config-tools .`).
+          build-contexts: |
+            config-tools=./config-tools
           platforms: ${{ matrix.platform }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha,scope=website-${{ github.ref }}-${{ env.PLATFORM_PAIR }}

diff --git a/.github/workflows/website-tests.yml b/.github/workflows/website-tests.yml
@@ -28,7 +28,9 @@ jobs:
       - uses: actions/cache@v5
         with:
           path: ~/.npm
-          key: ${{ runner.os }}-node-${{ hashFiles('website/**/package-lock.json') }}
+          key: ${{ runner.os }}-node-${{ hashFiles('website/**/package-lock.json', 'config-tools/**/package-lock.json') }}
+      - run: npm ci
+        working-directory: ./config-tools
       - run: npm ci
       - run: npm run check-format
       - run: npm run check-types
@@ -49,7 +51,10 @@ jobs:
         uses: actions/cache@v5
         with:
           path: ~/.npm
-          key: ${{ runner.os }}-node-${{ hashFiles('website/**/package-lock.json') }}
+          key: ${{ runner.os }}-node-${{ hashFiles('website/**/package-lock.json', 'config-tools/**/package-lock.json') }}
+      - name: Install config-tools dependencies
+        run: npm ci
+        working-directory: ./config-tools
       - name: Install dependencies
         run: npm ci
       - name: Run tests

diff --git a/AGENTS.md b/AGENTS.md
@@ -5,6 +5,10 @@ Components include: website, backend, deployment, preprocessing, ingest, deposit
 
 Write detailed PR summaries, not just short bullet points. When creating PRs, you should generally create them as a draft PR. If you have access to https://github.com/loculus-project/agent_store/ you can upload screenshots (which you might take using `playwright-cli` there to be able to use in PR descriptions, and maybe issue descriptions).
 
+## Markdown formatting
+
+When writing or editing Markdown files, keep each paragraph on a single line — do not insert manual line breaks (hard wraps) within a paragraph. Readers use editors with soft line wrapping. Hard-wrapping is only acceptable where the syntax requires a new line (list items, table rows, headings, code blocks, etc.).
+
 ## Preventing flaky Playwright tests (website)
 
 When adding or modifying interactive components in the website, ensure they are disabled until React hydration completes to prevent race conditions in Playwright tests:
@@ -19,4 +23,3 @@ These wrappers automatically disable components until client-side hydration is c
 
 Conda dependencies in `environment.yml` files are not automatically updated by dependabot.
 The `maintenance-scripts/` folder contains utilities to help update conda environment versions.
-
diff --git a/backend/README.md b/backend/README.md
@@ -24,16 +24,26 @@ All commands mentioned in this section are run from the `backend` directory unle
    postgres:latest
    ```
 
-2. Start the backend (including test config):
+2. Start the backend once to run Flyway migrations:
+
+   ```sh
+   ./start_dev.sh
+   ```
+
+   Stop it again after startup has completed.
+
+3. Import the preview organisms and restart the backend:
 
    ```sh
    ../generate_local_test_config.sh
+   ./import_local_test_config.py
    ./start_dev.sh
    ```
 
-The service listens, by default, to **port 8079**: <http://localhost:8079/swagger-ui/index.html>. The test config will be written to `loculus/website/tests/config`.
+The service listens, by default, to **port 8079**: <http://localhost:8079/swagger-ui/index.html>. The test config will be written to `loculus/website/tests/config`; `import_local_test_config.py` imports the generated preview organisms into the DB-backed config tables and merges website-facing schema fields from the generated website config.
+The importer uses a local `psql` binary when available, then falls back to the k3d/Helm database via `kubectl`, the `loculus_postgres` container created above, or a temporary Docker PostgreSQL client against the exposed host port.
 
-3. Clean up the database when done:
+4. Clean up the database when done:
 
    ```sh
    docker stop loculus_postgres
@@ -54,10 +64,11 @@ You need to set:
 --spring.datasource.password=unsecure
 ```
 
-- the path to the config file (use `../generate_local_test_config.sh` to generate this file):
+- the technical backend URLs:
 
 ```sh
---loculus.config.path=../website/tests/config/backend_config.json
+--loculus.backend.website-url=http://localhost:3000
+--loculus.backend.backend-url=http://localhost:8079
 ```
 
 - the url to fetch the public key for JWT verification