Develop => main

.github/workflows/docker_build.yaml

@@ -41,16 +41,16 @@ jobs:
               uses: actions/checkout@v6
 
             - name: Set up Docker Buildx
-              uses: docker/setup-buildx-action@v3
+              uses: docker/setup-buildx-action@v4
 
             - name: Extract Docker metadata
-              uses: docker/metadata-action@v5
+              uses: docker/metadata-action@v6
               id: meta
               with:
                   images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
             - name: Build test Docker image
-              uses: docker/build-push-action@v6
+              uses: docker/build-push-action@v7
               id: build-test-image
               with:
                   context: .
@@ -66,7 +66,7 @@ jobs:
                   docker run --rm ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.TEST_TAG }} test
 
             - name: Build and push
-              uses: docker/build-push-action@v6
+              uses: docker/build-push-action@v7
               id: build-push-image
               if: ${{ github.event_name == 'release' || github.event_name == 'push' }}
               with:

.github/workflows/tests.yml

@@ -49,11 +49,11 @@ jobs:
         shell: bash
         continue-on-error: false
         run: |
-          docker pull ghcr.io/berdatalakehouse/spark_notebook:main
+          docker pull ghcr.io/kbasedatalakehouse/spark_notebook:main
           echo "Running tests inside container..."
           # slightly nasty hack
           chmod 777 .
-          docker run --rm -e NB_USER=runner -v .:/tmp/cdm ghcr.io/berdatalakehouse/spark_notebook:main /bin/bash /tmp/cdm/scripts/run_tests.sh
+          docker run --rm -e NB_USER=runner -v .:/tmp/cdm ghcr.io/kbasedatalakehouse/spark_notebook:main /bin/bash /tmp/cdm/scripts/run_tests.sh
 
       - name: Send to Codecov
         id: send_to_codecov

.github/workflows/trivy.yaml

@@ -35,7 +35,7 @@ jobs:
         runs-on: ubuntu-latest
         steps:
             - name: Checkout code
-              uses: actions/checkout@v4
+              uses: actions/checkout@v6
 
             - name: Build an image from Dockerfile
               run: |
@@ -52,6 +52,6 @@ jobs:
                   timeout: 15m
 
             - name: Upload Trivy scan results to GitHub Security tab
-              uses: github/codeql-action/upload-sarif@v3
+              uses: github/codeql-action/upload-sarif@v4
               with:
                   sarif_file: "trivy-results.sarif"

.gitignore

@@ -1,3 +1,5 @@
+data/
+
 .DS_Store
 
 # Byte-compiled / optimized / DLL files

CHANGELOG.md

@@ -1,6 +1,7 @@
 # CDM Data Loaders Changelog
 
 - [CDM Data Loaders Changelog](#cdm-data-loaders-changelog)
+    - [v0.1.10](#v0110)
     - [v0.1.9](#v019)
     - [v0.1.8](#v018)
     - [v0.1.7](#v017)
@@ -13,6 +14,11 @@
     - [v0.1.0](#v010)
 
 
+### v0.1.10
+
+- Add in the ability to configure loggers using a file
+- Bugfix for UniProt pipeline
+
 ### v0.1.9
 
 - Splitting out NCBI REST API pipeline into annotation and dataset components.

Dockerfile

@@ -55,14 +55,26 @@ RUN --mount=type=cache,target=/root/.cache/uv \
 
 # Then, add the rest of the project source code and install it
 # Installing separately from its dependencies allows optimal layer caching
-COPY --chown=nonroot:nonroot . /app
+COPY --chown=nonroot:nonroot .dlt /app/.dlt
+COPY --chown=nonroot:nonroot docs /app/docs
+COPY --chown=nonroot:nonroot scripts /app/scripts
+COPY --chown=nonroot:nonroot src /app/src
+COPY --chown=nonroot:nonroot tests /app/tests
+COPY --chown=nonroot:nonroot README.md /app/README.md
+COPY --chown=nonroot:nonroot pyproject.toml /app/pyproject.toml
+COPY --chown=nonroot:nonroot uv.lock /app/uv.lock
+
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv sync --locked --no-editable
 
 # Place executables in the environment at the front of the path
 ENV PATH="/app/.venv/bin:$PATH"
 
 RUN chmod +x ./scripts/entrypoint.sh
+
+# make sure that the nonroot user owns the app directory
+RUN chown nonroot:nonroot /app
+
 # Use the non-root user to run our application
 USER nonroot
 ENTRYPOINT ["./scripts/entrypoint.sh"]

README.md

@@ -6,6 +6,7 @@ Repo for CDM input data loading and wrangling
   - [Environment and python management](#environment-and-python-management)
   - [Installation](#installation)
     - [Lakehouse and Use with Jupyter notebooks](#lakehouse-and-use-with-jupyter-notebooks)
+      - [Jupyter Kernel Environment Variables](#jupyter-kernel-environment-variables)
   - [Running import pipelines](#running-import-pipelines)
   - [Development](#development)
     - [Spark and other non-python dependencies](#spark-and-other-non-python-dependencies)
@@ -103,24 +104,24 @@ Current endpoints include:
 
 ### Spark and other non-python dependencies
 
-Some parts of this codebase rely on having a Spark instance available. Spark dependencies are pulled in by the `berdl-notebook-utils` package from [BERDataLakehouse/spark_notebook](https://github.com/BERDataLakehouse/spark_notebook), and the [Docker container](https://github.com/BERDataLakehouse/spark_notebook/pkgs/container/spark_notebook) generated by the same repo should be used for development and testing to mimic the container where code will be run.
+Some parts of this codebase rely on having a Spark instance available. Spark dependencies are pulled in by the `berdl-notebook-utils` package from [KBaseDataLakehouse/spark_notebook](https://github.com/KBaseDataLakehouse/spark_notebook), and the [Docker container](https://github.com/KBaseDataLakehouse/spark_notebook/pkgs/container/spark_notebook) generated by the same repo should be used for development and testing to mimic the container where code will be run.
 
 Pull the docker image:
 
 ```sh
-> docker pull ghcr.io/berdatalakehouse/spark_notebook:main
+> docker pull ghcr.io/kbasedatalakehouse/spark_notebook:main
 ```
 
 Mount the current directory at `/tmp/cdm` and run the tests:
 
 ```sh
-> docker run --rm -e NB_USER=runner -v .:/tmp/cdm ghcr.io/berdatalakehouse/spark_notebook:main /bin/bash /tmp/cdm/scripts/run_tests.sh
+> docker run --rm -e NB_USER=runner -v .:/tmp/cdm ghcr.io/kbasedatalakehouse/spark_notebook:main /bin/bash /tmp/cdm/scripts/run_tests.sh
 ```
 
 Run the container interactively as the user `runner`; current directory is mounted at `/tmp/cdm`:
 
 ```sh
-> docker run --rm -e NB_USER=runner -it -v .:/tmp/cdm ghcr.io/berdatalakehouse/spark_notebook:main
+> docker run --rm -e NB_USER=runner -it -v .:/tmp/cdm ghcr.io/kbasedatalakehouse/spark_notebook:main
 ```
 
 This will launch a bash shell; the contents of the `cdm-data-loaders` directory are mounted at `/tmp/cdm`.
@@ -129,13 +130,13 @@ This will launch a bash shell; the contents of the `cdm-data-loaders` directory
 Run the container and sleep:
 
 ```sh
-> docker run --rm -e NB_USER=runner -it -v .:/tmp/cdm ghcr.io/berdatalakehouse/spark_notebook:main sleep 100000000
+> docker run --rm -e NB_USER=runner -it -v .:/tmp/cdm ghcr.io/kbasedatalakehouse/spark_notebook:main sleep 100000000
 ```
 
 The `sleep` command will run the container for long enough that you can then connect to it via Docker Desktop or the [VSCode Containers extension](https://code.visualstudio.com/docs/containers/overview).
 
 
-See the [BERDataLakehouse/spark_notebook](https://github.com/BERDataLakehouse/spark_notebook) repo for more information on the container and for a full docker-compose set up to mimic the BER Data Lakehouse container infrastructure.
+See the [KBaseDataLakehouse/spark_notebook](https://github.com/KBaseDataLakehouse/spark_notebook) repo for more information on the container and for a full docker-compose set up to mimic the BER Data Lakehouse container infrastructure.
 
 
 ### Tests

logger_config.json

@@ -0,0 +1,24 @@
+{
+    "version": 1,
+    "disable_existing_loggers": false,
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "level": "INFO",
+            "stream": "ext://sys.stdout"
+        },
+        "file": {
+            "class": "logging.handlers.RotatingFileHandler",
+            "filename": "/output_dir/log_file.txt"
+        }
+    },
+    "loggers": {
+        "root": {
+            "level": "INFO",
+            "handlers": [
+                "console",
+                "file"
+            ]
+        }
+    }
+}

notebooks/pangenome_refseq.py

@@ -11,18 +11,15 @@
    - missing_refseq_ids
 """
 
-from __future__ import annotations
-
 import logging
 import tempfile
 import urllib.request
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 import click
-from pyspark.sql.functions import regexp_replace
-
 from berdl_notebook_utils.setup_spark_session import get_spark_session
+from pyspark.sql.functions import regexp_replace
 
 if TYPE_CHECKING:
     from pyspark.sql import SparkSession

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cdm-data-loaders"
-version = "0.1.9"
+version = "0.1.10"
 description = "Data loaders and wranglers for the CDM."
 requires-python = ">= 3.13"
 readme = "README.md"
@@ -14,7 +14,7 @@ dependencies = [
     "click>=8.4.1",
     "defusedxml>=0.7.1",
     "delta-spark>=4.2.0",
-    "dlt[deltalake,duckdb,filesystem,parquet,pyiceberg]>=1.27.2",
+    "dlt[deltalake,duckdb,filesystem,hub,parquet,pyiceberg]>=1.27.2",
     "frictionless[aws]>=5.19.0",
     "frozendict>=2.4.7",
     "ipykernel>=7.2.0",
@@ -170,10 +170,17 @@ ignore = [
 
 [tool.ruff.lint.per-file-ignores]
 "*.ipynb" = ["T201"] # ignore printing in notebooks
-"tests/**/*.py" = ["S101", "T201", "FBT001", "FBT002", "ARG002"]  # use of assert, booleans, unused mock args
+"**/__init__.py" = ["D104"] # no docstrings for __init__ files
+"scripts/**/*.py" = ["T201"] # ignore printing in scripts
+# S101: use of assert
+# T201: print statements
+# FBT001, FBT002: booleans as args
+# ANN401: type annotations
+# ARG002: unused mock args
+# PLR0913: too many arguments
+"tests/**/*.py" = ["S101", "T201", "FBT001", "FBT002", "ARG002", "PLR0913"]
 "tests/integration/**/*.py" = ["S101", "T201", "FBT001", "FBT002", "ARG002", "ANN401"]
 "tests/utils/test_s3.py" = ["ANN401"]
-"**/__init__.py" = ["D104"]
 
 [tool.ruff.lint.mccabe]
 # Flag errors (`C901`) whenever the complexity level exceeds 15.
@@ -217,5 +224,5 @@ AWS_SECRET_ACCESS_KEY = "test_access_secret"
 
 [tool.uv.sources]
 cdm-schema = { git = "https://github.com/kbase/cdm-schema.git" }
-berdl-notebook-utils = { git = "https://github.com/BERDataLakehouse/spark_notebook.git", subdirectory = "notebook_utils" }
+berdl-notebook-utils = { git = "https://github.com/KBaseDataLakehouse/spark_notebook.git", subdirectory = "notebook_utils" }
 cdm-task-service-client = { git = "https://github.com/kbase/cdm-task-service-client.git" }

scripts/parse_index.py

@@ -1,8 +1,11 @@
 """Script to parse a file containing paths to genome data."""
 
 import json
 import sys
 from pathlib import Path
+from typing import Final
+
+N_EXPECTED_ARGS: Final[int] = 3
 
 
 def main() -> None:
@@ -19,8 +22,8 @@
         Prints the values associated with the specified key, one per line.
     """
     # Check if the correct number of arguments is provided
-    if len(sys.argv) < 3:
-        print("Usage: parse_json.py <input_json_file> <key>")  # noqa: T201
+    if len(sys.argv) < N_EXPECTED_ARGS:
+        print("Usage: parse_json.py <input_json_file> <key>")
         sys.exit(1)
 
     input_file = sys.argv[1]  # Path to the JSON file
@@ -30,21 +33,21 @@
         # Attempt to open and read the JSON file
         with Path(input_file).open() as file:
             data = json.load(file)
-    except Exception as e:
-        print(f"Error reading JSON file: {e}")  # noqa: T201
+    except Exception as e:  # noqa: BLE001
+        print(f"Error reading JSON file: {e}")
         sys.exit(1)
 
     # throw error if top level data structure is incorrect
     if not isinstance(data, dict):
-        print("Error: expected JSON file to be a dictionary")  # noqa: T201
+        print("Error: expected JSON file to be a dictionary")
         sys.exit(1)
 
     # iterate over dictionary values and extract the values from dicts containing the target key
     extracted_paths = [entry[target_key] for entry in data.values() if isinstance(entry, dict) and target_key in entry]
 
     # Print each extracted path on a new line
     for path in extracted_paths:
-        print(path)  # noqa: T201
+        print(path)
 
 
 if __name__ == "__main__":

scripts/s3_local.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# ruff: noqa: T201, EM101, EM102, TRY003, D103
+# ruff: noqa: D103
 """Thin S3 CLI for local CEPH testing (no aws-cli install required).
 
 Usage (all commands assume ``uv run`` from the repo root):
@@ -19,7 +19,7 @@
 import os
 import sys
 
-import cdm_data_loaders.utils.s3 as s3
+from cdm_data_loaders.utils import s3
 
 
 def _client() -> None:
@@ -41,7 +41,8 @@ def _client() -> None:
 def main() -> None:
     if len(sys.argv) < 2 or sys.argv[1] not in COMMANDS:  # noqa: PLR2004
         cmds = ", ".join(COMMANDS)
-        raise SystemExit(f"Usage: s3_local.py <{cmds}> [args ...]\n\n{__doc__}")
+        err_msg = f"Usage: s3_local.py <{cmds}> [args ...]\n\n{__doc__}"
+        raise SystemExit(err_msg)
     _client()
     COMMANDS[sys.argv[1]](sys.argv[2:])
 

src/cdm_data_loaders/audit/checkpoint.py

@@ -1,11 +1,12 @@
 """Checkpoint audit table functions: adding and updating information on data import pipeline execution."""
 
+from logging import Logger, getLogger
+
 from delta.tables import DeltaTable
 from pyspark.sql import SparkSession
 from pyspark.sql import functions as sf
 
 from cdm_data_loaders.audit.schema import (
-    AUDIT_SCHEMA,
     CHECKPOINT,
     LAST_ENTRY_ID,
     PIPELINE,
@@ -18,7 +19,8 @@
     current_run_expr,
 )
 from cdm_data_loaders.core.pipeline_run import PipelineRun
-from cdm_data_loaders.utils.cdm_logger import get_cdm_logger
+
+logger: Logger = getLogger(__name__)
 
 
 # Checkpoint table-related functions
@@ -58,7 +60,7 @@ def upsert_checkpoint(
         .whenNotMatchedInsertAll()
         .execute()
     )
-    get_cdm_logger().info("%s %s: checkpoint created/updated", run.pipeline, run.run_id)
+    logger.info("%s %s: checkpoint created/updated", run.pipeline, run.run_id)
 
 
 def update_checkpoint_status(spark: SparkSession, run: PipelineRun, status: str) -> None:
@@ -80,15 +82,15 @@ def update_checkpoint_status(spark: SparkSession, run: PipelineRun, status: str)
     # N.b. this may not work correctly if another process updates the table in the interim
     metrics = delta.history(1).select("operationMetrics").collect()[0][0]
     if int(metrics.get("numUpdatedRows", 0)) == 0:
-        get_cdm_logger().warning(
+        logger.warning(
             "%s %s: cannot update '%s' to status %s because no record exists.",
             run.pipeline,
             run.run_id,
             CHECKPOINT,
             status,
         )
     else:
-        get_cdm_logger().info("%s %s: checkpoint successfully updated to status %s", run.pipeline, run.run_id, status)
+        logger.info("%s %s: checkpoint successfully updated to status %s", run.pipeline, run.run_id, status)
 
 
 def load_checkpoint(spark: SparkSession, run: PipelineRun) -> str | None: