Develop => main

.github/pull_request_template.md

@@ -11,6 +11,7 @@
 # Dev Checklist:
 
 - [ ] My code follows the guidelines at https://sites.google.com/lbl.gov/trussresources/home?authuser=0
+- [ ] My submission follows the [AI Covenant](/AI_COVENANT.md) principles
 - [ ] I have performed a self-review of my own code
 - [ ] I have commented my code, particularly in hard-to-understand areas
 - [ ] I have made corresponding changes to the documentation

.github/workflows/integration_tests.yaml

@@ -0,0 +1,54 @@
+name: CEPH Integration tests
+
+on:
+    workflow_call:
+
+    push:
+        branches:
+            - main
+    pull_request:
+        types:
+            - opened
+            - reopened
+            - synchronize
+            - ready_for_review
+
+permissions:
+    contents: read
+
+jobs:
+    ceph_integration_tests:
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v6
+
+            - name: Run CEPH-backed integration tests
+              run: |
+                  docker compose up \
+                    --build \
+                    --abort-on-container-exit \
+                    --exit-code-from integration-tests
+
+            - name: Capture compose diagnostics on failure
+              if: failure()
+              run: |
+                  echo "=== docker compose ps ==="
+                  docker compose ps -a || true
+
+                  echo "=== ceph logs ==="
+                  docker logs cdm-data-loaders-ceph-1 || true
+
+                  echo "=== ceph integration test logs ==="
+                  docker logs cdm-data-loaders-integration-tests-1 || true
+
+                  echo "=== ceph inspect state ==="
+                  docker inspect cdm-data-loaders-ceph-1 --format '{{json .State}}' || true
+
+                  echo "=== ceph health state ==="
+                  docker inspect cdm-data-loaders-ceph-1 --format '{{json .State.Health}}' || true
+
+            - name: Tear down
+              if: always()
+              run: docker compose down --volumes

.github/workflows/tests.yml

@@ -91,4 +91,4 @@ jobs:
       - name: Run local tests
         shell: bash
         continue-on-error: false
-        run: uv run pytest -m "not requires_spark"
+        run: uv run pytest -m "not requires_spark and not requires_ceph"

Dockerfile

@@ -3,21 +3,29 @@
 # Dockerfile is based heavily on the example uv dockerfile:
 # https://github.com/astral-sh/uv-docker-example
 
-# Pull the pre-built Rust app from ghcr.io
-FROM ghcr.io/ialarmedalien/xml_file_splitter:latest AS rust-app
-
 # Use a Python image with uv pre-installed
 FROM ghcr.io/astral-sh/uv:python3.13-trixie-slim
 
+ARG QSV_VERSION="20.1.0"
+ARG XML_FILE_SPLITTER_VERSION="v0.1.2"
+
 # Set environment variable to noninteractive to prevent prompts during apt operations
 ENV DEBIAN_FRONTEND=noninteractive
 
 # add tini and git
-RUN apt-get update -y && apt-get install -y --no-install-recommends tini git ca-certificates && \
-    rm -rf /var/lib/apt/lists/*
-
-# copy only the compiled xml-file-splitter binary from the Rust image
-COPY --from=rust-app /usr/local/bin/xml_file_splitter /usr/local/bin/xml_file_splitter
+RUN apt-get update -y && apt-get install -y --no-install-recommends tini git ca-certificates wget unzip && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /tmp
+
+# download and install the xml_file_splitter and qsv binaries, and copy them to /usr/local/bin
+RUN wget https://github.com/ialarmedalien/xml_file_splitter/releases/download/${XML_FILE_SPLITTER_VERSION}/xml_file_splitter-aarch64-unknown-linux-gnu.tar.gz && \
+    tar -xvf xml_file_splitter-aarch64-unknown-linux-gnu.tar.gz && \
+    mv xml_file_splitter-aarch64-unknown-linux-gnu/xml_file_splitter /usr/local/bin/ && \
+    # qsv release -- only need the `qsv` binary from it
+    wget https://github.com/dathere/qsv/releases/download/${QSV_VERSION}/qsv-${QSV_VERSION}-aarch64-unknown-linux-gnu.zip && \
+    unzip qsv-${QSV_VERSION}-aarch64-unknown-linux-gnu.zip -d /tmp/qsv && \
+    mv /tmp/qsv/qsv /usr/local/bin/ && \
+    rm -rf /tmp/*
 
 # Setup a non-root user
 RUN groupadd --system --gid 999 nonroot \

README.md

@@ -5,10 +5,12 @@ Repo for CDM input data loading and wrangling
 - [cdm-data-loaders](#cdm-data-loaders)
   - [Environment and python management](#environment-and-python-management)
   - [Installation](#installation)
+    - [Lakehouse and Use with Jupyter notebooks](#lakehouse-and-use-with-jupyter-notebooks)
   - [Running import pipelines](#running-import-pipelines)
   - [Development](#development)
     - [Spark and other non-python dependencies](#spark-and-other-non-python-dependencies)
     - [Tests](#tests)
+      - [CEPH-Backed integration tests (CEPH + NCBI FTP)](#ceph-backed-integration-tests-ceph--ncbi-ftp)
   - [Loading genomes, contigs, and features](#loading-genomes-contigs-and-features)
   - [Running bbmap stats and checkm2 on genome or contigset files](#running-bbmap-stats-and-checkm2-on-genome-or-contigset-files)
 
@@ -48,6 +50,43 @@ To activate a virtual environment with these dependencies installed, run
 If you are using IDEs like VSCode, they should pick up the creation of the new environment and offer it for executing python code.
 
 
+### Lakehouse and Use with Jupyter notebooks
+
+`cdm-data-loaders` can be installed on platforms like the KBase Lakehouse using the same installation steps:
+
+```sh
+cd cdm-data-loaders
+uv sync
+source .venv/bin/activate
+```
+
+To use the library in a Jupyter notebook, it must be registered as a Jupyter kernel. After performing the three steps above,
+run the following commands:
+
+```sh
+uv pip install -e .
+uv pip install ipykernel
+uv run python -m ipykernel install --user --name cdm-data-loaders --display-name "cdm-data-loaders"
+```
+
+The `cdm-data-loaders` kernel should now be available from the dropdown list of kernels in the Jupyter notebook interface.
+
+#### Jupyter Kernel Environment Variables
+
+If you would like to include environment variables for your kernel that are not present in your default environment, you can add them to the `kernel.json` for your new kernel (e.g., in `cdm-data-loaders/.venv/share/jupyter/kernels/python3/kernel.json`):
+```json
+{
+  "argv": ["..."],
+  "display_name": "cdm-data-loaders",
+  "language": "python",
+  "env": {
+    "MY_CUSTOM_VAR": "...",
+    ...
+  }
+}
+```
+
+
 ## Running import pipelines
 
 The repo provides a Docker container that can be used to run several import pipelines or to run unit tests for the repo. The [entrypoint script](scripts/entrypoint.sh) parses the container `run` arguments and launches the appropriate functions.
@@ -103,16 +142,16 @@ See the [BERDataLakehouse/spark_notebook](https://github.com/BERDataLakehouse/sp
 
 Tests are categorised using pytest markers to allow developers to execute some or all the tests. See [pyproject.toml](pyproject.toml) for the markers used.
 
-To run all tests (requires a running Spark instance), execute the command:
+To run all tests (requires a running Spark instance and a running CEPH test container), execute the command:
 
 ```sh
 > uv run pytest
 ```
 
-To run only tests that do not require Spark, run
+To run only tests that do not require Spark or CEPH, run
 
 ```sh
-> uv run pytest -m "not requires_spark"
+> uv run pytest -m "not requires_spark and not requires_ceph"
 ```
 
 To generate coverage for the tests, run
@@ -123,6 +162,64 @@ To generate coverage for the tests, run
 The standard python `coverage` package is used and coverage can be generated as html or other formats by changing the parameters.
 
 
+#### CEPH-Backed integration tests (CEPH + NCBI FTP)
+
+End-to-end integration tests for the NCBI assembly pipeline live in `tests/integration/`. They exercise the full flow — manifest diffing, FTP download, S3 promote/archive — against a locally running [CEPH](https://ceph.io/) container and the real NCBI FTP server.
+
+**Requirements:**
+- Docker or Podman (for CEPH)
+- Network access to `ftp.ncbi.nlm.nih.gov`
+
+**Running with Docker Compose (easiest)**
+
+The [docker-compose.yml](docker-compose.yml) at the repo root defines both a CEPH service and the integration test runner. To build the image, start CEPH, and run the integration tests in one command:
+
+```sh
+docker compose up --build --abort-on-container-exit
+```
+
+Compose will stream test output to the terminal and exit with the pytest exit code. To clean up afterwards:
+
+```sh
+docker compose down --volumes
+```
+
+**Running manually**
+
+If you prefer to run the tests directly against a local CEPH instance (e.g. for faster iteration during development), follow the steps below.
+
+**1. Start CEPH locally:**
+
+```sh
+docker run -d \
+  --name ceph \
+  -p 9000:8080 \
+  -p 9001:8443 \
+  -e RGW_PORT=8080 \
+  -e RGW_ACCESS_KEY=test_access_key \
+  -e RGW_SECRET_KEY=test_access_secret \
+  ghcr.io/kbasetest/ceph-rgw-test-image:0.1.5
+```
+
+**2. Run the integration tests:**
+
+```sh
+> uv run pytest tests/integration/ -m requires_ceph -v
+```
+
+**3. Inspect results:**
+
+Buckets are **not** cleaned up after tests. The CEPH dashboard at [http://localhost:9001](http://localhost:9001) (login: `admin` / `admin`) does not support inspecting bucket contents; use the CLI below (e.g. `scripts/s3_local.py ls/head`) to inspect the final state of each test bucket. Each test method creates its own bucket (e.g. `integ-test-promote-dry-run`).
+
+**4. Stop CEPH when done:**
+
+```sh
+docker stop ceph && docker rm ceph
+```
+
+> **Note:** These tests download real assemblies from NCBI FTP and are inherently slow (~30–60s per assembly). They are also marked `slow_test` so you can exclude them independently: `uv run pytest -m "not slow_test"`.
+
+
 ## Loading genomes, contigs, and features
 
 The [genome loader](src/cdm_data_loaders/parsers/genome_loader.py) can be used to load and integrate data from related GFF and FASTA files. Currently, the loader requires a GFF file and two FASTA files (one for amino acid seqs, one for nucleic acid seqs) for each genome. The list of files to be processed should be specified in the genome paths file, which has the following format:

docker-compose.yml

@@ -0,0 +1,31 @@
+services:
+  ceph:
+    image: ghcr.io/kbasetest/ceph-rgw-test-image:0.1.5
+    environment:
+      RGW_PORT: "8080"
+      RGW_ACCESS_KEY: test_access_key
+      RGW_SECRET_KEY: test_access_secret
+    ports:
+      - "9000:8080"
+      - "9001:8443"
+    healthcheck:
+      # Mark healthy only when the RGW/S3 HTTP endpoint is reachable.
+      test: [ "CMD-SHELL", "curl -fsS http://127.0.0.1:8080/ >/dev/null || wget -q -O- http://127.0.0.1:8080/ >/dev/null" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+
+  integration-tests:
+    image: cdm-data-loaders-integration-tests:latest
+    build:
+      context: .
+    depends_on:
+      ceph:
+        condition: service_healthy
+    environment:
+      AWS_ENDPOINT_URL: http://ceph:8080
+      AWS_ACCESS_KEY_ID: test_access_key
+      AWS_SECRET_ACCESS_KEY: test_access_secret
+    entrypoint: [ "/app/scripts/entrypoint.sh", "ceph_integration_test" ]
+    command: []