From f7fc9d57c6459e0f679b4eebbdc5e96cee0b35d9 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 16 Apr 2026 15:18:29 -0700 Subject: [PATCH 001/128] draft ncbi scripts --- cdm_ncbi_fto.prompt.md | 292 ++++++++++ notebooks/ncbi_ftp_manifest.ipynb | 210 +++++++ notebooks/ncbi_ftp_promote.ipynb | 172 ++++++ pyproject.toml | 3 +- scripts/entrypoint.sh | 8 +- src/cdm_data_loaders/ncbi_ftp/__init__.py | 0 src/cdm_data_loaders/ncbi_ftp/assembly.py | 206 +++++++ src/cdm_data_loaders/ncbi_ftp/manifest.py | 471 ++++++++++++++++ src/cdm_data_loaders/ncbi_ftp/promote.py | 281 ++++++++++ .../pipelines/ncbi_ftp_download.py | 191 +++++++ src/cdm_data_loaders/utils/checksums.py | 55 ++ src/cdm_data_loaders/utils/ftp_client.py | 162 ++++++ src/cdm_data_loaders/utils/s3.py | 114 ++++ tests/ncbi_ftp/__init__.py | 0 tests/ncbi_ftp/conftest.py | 80 +++ tests/ncbi_ftp/test_assembly.py | 114 ++++ tests/ncbi_ftp/test_manifest.py | 516 ++++++++++++++++++ tests/ncbi_ftp/test_notebooks.py | 89 +++ tests/ncbi_ftp/test_promote.py | 278 ++++++++++ tests/pipelines/test_ncbi_ftp_download.py | 230 ++++++++ tests/utils/test_checksums.py | 76 +++ tests/utils/test_ftp_client.py | 199 +++++++ tests/utils/test_s3.py | 128 +++++ 23 files changed, 3872 insertions(+), 3 deletions(-) create mode 100644 cdm_ncbi_fto.prompt.md create mode 100644 notebooks/ncbi_ftp_manifest.ipynb create mode 100644 notebooks/ncbi_ftp_promote.ipynb create mode 100644 src/cdm_data_loaders/ncbi_ftp/__init__.py create mode 100644 src/cdm_data_loaders/ncbi_ftp/assembly.py create mode 100644 src/cdm_data_loaders/ncbi_ftp/manifest.py create mode 100644 src/cdm_data_loaders/ncbi_ftp/promote.py create mode 100644 src/cdm_data_loaders/pipelines/ncbi_ftp_download.py create mode 100644 src/cdm_data_loaders/utils/checksums.py create mode 100644 src/cdm_data_loaders/utils/ftp_client.py create mode 100644 tests/ncbi_ftp/__init__.py create mode 100644 tests/ncbi_ftp/conftest.py create mode 100644 tests/ncbi_ftp/test_assembly.py create mode 100644 tests/ncbi_ftp/test_manifest.py create mode 100644 tests/ncbi_ftp/test_notebooks.py create mode 100644 tests/ncbi_ftp/test_promote.py create mode 100644 tests/pipelines/test_ncbi_ftp_download.py create mode 100644 tests/utils/test_checksums.py create mode 100644 tests/utils/test_ftp_client.py diff --git a/cdm_ncbi_fto.prompt.md b/cdm_ncbi_fto.prompt.md new file mode 100644 index 00000000..a1e52592 --- /dev/null +++ b/cdm_ncbi_fto.prompt.md @@ -0,0 +1,292 @@ +# Plan: Port NCBI Assembly Sync to cdm-data-loaders + +Port the 3-phase NCBI assembly transfer pipeline from this repo +([kbase-transfers](https://github.com/kbase/kbase-transfers)) on the `develop-ncbi-automation` branchinto +[kbase/cdm-data-loaders](https://github.com/kbase/cdm-data-loaders/tree/develop) +(develop branch). + +- **Phase 2** (container download) becomes a new CTS entrypoint command. +- **Phases 1 and 3** become Jupyter notebooks in `notebooks/`. +- The existing cdm-data-loaders `utils/s3.py` gets new functions for metadata support + (existing functions are not modified). +- Tests use **moto** (matching cdm-data-loaders conventions). +- FTP logic stays as **ftplib**. +- New code lives in a dedicated `src/cdm_data_loaders/ncbi_ftp/` module, + separate from existing NCBI REST / refseq code. + +### Phase responsibilities + +Each phase has a deliberately narrow scope: + +| Phase | Input | Output | Responsibility | +|-------|-------|--------|----------------| +| 1 — Manifest | NCBI assembly summary + previous snapshot from S3 | `transfer_manifest.txt`, `removed_manifest.txt`, `diff_summary.json` | **All** filtering logic (prefix ranges, limits, diffing). Produces a final list of what to transfer and what to archive. | +| 2 — Download | `transfer_manifest.txt` (from input mount) | Downloaded files in output mount, preserving FTP directory structure (`GCF/000/001/.../assembly_dir/`) | Reads the manifest; downloads exactly those assemblies from NCBI FTP; verifies MD5; writes `.md5` sidecars. No filtering, no S3 access. | +| 3 — Promote | Downloaded files in S3 staging prefix + `removed_manifest.txt` | Files at final Lakehouse paths; archived replaced assemblies | Syncs staging → final location. Archives replaced/suppressed assemblies. **Removes successfully-promoted entries from `transfer_manifest.txt`** so an interrupted Phase 2 can resume from where it left off. | + +--- + +## Background: the 3-phase pipeline + +The pipeline is documented in this repo's +[scripts/ncbi/README.md](README.md#semi-automated-transfer-pipeline): + +| Phase | Script (this repo) | Purpose | +|-------|-------------------|---------| +| 1 — Manifest | [`generate_transfer_manifest.py`](generate_transfer_manifest.py) | Diff NCBI assembly summary against previous snapshot; produce download + removal manifests | +| 2 — Download | [`container_download.py`](container_download.py) | Download assemblies from NCBI FTP, verify MD5, write `.md5` sidecars (runs in CTS container) | +| 3 — Promote | [`promote_staged_files.py`](promote_staged_files.py) | Copy staged files to final Lakehouse paths; archive replaced/suppressed assemblies | + +Supporting code in this repo: + +| File | What to port | +|------|-------------| +| [`download_genomes.py`](download_genomes.py) | FTP resilience patterns (TCP keepalive, NOOP pings, thread-local connections), file filters, accession path construction | +| [`../../kbase_transfers/minio_client.py`](../../kbase_transfers/minio_client.py) | Metadata-aware upload pattern (MD5 as user metadata, CRC64/NVME checksums) | +| [`../../tests/test_sync.py`](../../tests/test_sync.py) | Unit tests for parsing, diffing — port to moto-based tests | +| [`../../tests/test_minio_client.py`](../../tests/test_minio_client.py) | Integration test patterns for S3 operations | + +--- + +## Phase A: Extend cdm-data-loaders `utils/s3.py` + +The promote step needs to attach user-metadata (MD5) to uploads and read +checksums via HEAD. The existing +[`s3.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/utils/s3.py) +doesn't support custom metadata on upload or `head_object` with checksum +retrieval. + +**Prefer adding new functions over modifying existing ones to minimise +impact on other scripts.** + +### Steps + +1. **Add `upload_file_with_metadata()`** — new function that accepts + `local_file_path`, `destination_dir`, `metadata: dict[str, str]`, + optional `object_name`. Passes `Metadata` in `ExtraArgs` alongside the + existing `ChecksumAlgorithm: CRC64NVME`. Same upload logic as + `upload_file()` but with metadata support. + +2. **Add `head_object(s3_path)`** — new function returning dict with `size`, + `metadata`, `checksum_crc64nvme` (from `ChecksumCRC64NVME` header), or + `None` if 404. Uses `ChecksumMode='ENABLED'`. + +3. **Add `copy_object_with_metadata()`** — new function wrapping + `s3.copy_object()` that accepts `metadata` + `MetadataDirective='REPLACE'` + for archiving replaced assemblies with tags (`archive_reason`, `archive_date`, + `ncbi_last_release`). + +4. **Add moto-based tests** following the existing `mock_s3_client` fixture pattern + in [`tests/utils/test_s3.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/tests/utils/test_s3.py). + Use the existing `strip_checksum_algorithm` workaround for moto's CRC64NVME limitation. + +**Files modified:** +- `src/cdm_data_loaders/utils/s3.py` — add 3 new functions (no changes to existing functions) +- `tests/utils/test_s3.py` — add corresponding tests + +--- + +## Phase B: Create the `ncbi_ftp` module + +New module at `src/cdm_data_loaders/ncbi_ftp/`, separate from the +existing `ncbi_rest_api.py` pipeline and `refseq_pipeline/`. + +``` +src/cdm_data_loaders/ncbi_ftp/ +├── __init__.py +├── ftp_client.py # FTP: connect, keepalive, list, download, retry +├── manifest.py # Phase 1: summary diffing, manifest generation, ALL filtering +├── download.py # Phase 2: CTS container download + MD5 verification +├── promote.py # Phase 3: sync staging → final, archive, trim manifest +├── checksums.py # MD5 verification, CRC64/NVME computation +└── settings.py # Pydantic settings (extends CtsDefaultSettings) +``` + +### Steps + +5. **`ftp_client.py`** — Port from [`download_genomes.py`](download_genomes.py) + and [`container_download.py`](container_download.py): + - `connect_ftp(host, timeout)` with TCP keepalive (`SO_KEEPALIVE`, `TCP_KEEPIDLE`, `TCP_KEEPINTVL`) + - `ftp_noop_keepalive(ftp, interval)` — NOOP sender for idle connections + - `ftp_list_dir(ftp, path)` — NLST wrapper with retry on `error_temp` + - `ftp_download_file(ftp, remote_path, local_path)` — `RETR` with retry + - Thread-local FTP connection management for parallel downloads + - Use `get_cdm_logger()` instead of print statements + +6. **`checksums.py`** — Port from [`download_genomes.py`](download_genomes.py): + - `compute_crc64nvme(file_path) -> str` — reads in 1MB chunks, returns base64-encoded 8-byte big-endian (uses `awscrt.checksums.crc64nvme`) + - `verify_md5(file_path, expected_md5) -> bool` + - `parse_md5_checksums_file(text) -> dict[str, str]` — parses NCBI `md5checksums.txt` + +7. **`settings.py`** — Pydantic settings following + [`cts_defaults.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/pipelines/cts_defaults.py) pattern: + - `DownloadSettings(CtsDefaultSettings)` — adds `manifest`, `threads`, `ftp_host` + - CLI-parseable with `AliasChoices`, `field_validator` for constraints + +8. **`manifest.py`** — Port from [`generate_transfer_manifest.py`](generate_transfer_manifest.py): + - `download_assembly_summary(ftp, database) -> str` + - `parse_assembly_summary(text) -> dict[str, AssemblyRecord]` + - `compute_diff(current, previous) -> DiffResult` — new/updated/replaced/suppressed/withdrawn + - `write_manifest(diff, output_dir)` — writes `transfer_manifest.txt`, `removed_manifest.txt`, `diff_summary.json` + - **All filtering logic lives here**: prefix-range filtering (`--prefix-from`, `--prefix-to`), `--limit`, any other subsetting + - The output `transfer_manifest.txt` is the final, filtered list — Phase 2 downloads exactly what's in it + +9. **`download.py`** — Port from [`container_download.py`](container_download.py). + **This phase is deliberately simple**: read manifest, download, verify. + - `run_download(settings: DownloadSettings)` — main CTS entry point + - Reads `transfer_manifest.txt` from input mount; each line is an FTP path + - `download_assembly(ftp, ftp_path, output_dir) -> DownloadResult` + - File filters: `_genomic.fna.gz`, `_genomic.gff.gz`, `_protein.faa.gz`, `_gene_ontology.gaf.gz`, `_assembly_report.txt`, `_assembly_stats.txt`, etc. + - MD5 verification (3 retries), `.md5` sidecar writing + - `ThreadPoolExecutor` for parallel downloads + - Output preserves FTP directory structure: `{GCF|GCA}/{000}/{001}/{215}/{assembly_dir}/` (same subfolder hierarchy as on the FTP server) + - Writes `download_report.json` summary + - `cli()` function using `run_cli(DownloadSettings, run_download)` from + [`core.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/pipelines/core.py) + - **No filtering or subsetting logic** — downloads exactly what's in the manifest + +10. **`promote.py`** — Port from [`promote_staged_files.py`](promote_staged_files.py): + - `run_promote(staging_prefix, removed_manifest, release_tag, manifest_path)` + - Walk staged files in S3 staging prefix, upload each to final Lakehouse path via `upload_file_with_metadata()` with MD5 from `.md5` sidecars + - Skip `.md5` and `.crc64nvme` sidecar files themselves + - Archive replaced/suppressed assemblies (from `removed_manifest.txt`): `copy_object_with_metadata()` to `archive/{release}/...` with metadata tags, then `delete_object()` + - **Manifest trimming for resumability**: after successfully promoting an assembly's files, remove that entry from `transfer_manifest.txt`. If Phase 2 is re-run after a partial failure, it only downloads the remaining entries. + - `--dry-run` support + +--- + +## Phase C: Notebooks for Phases 1 and 3 + +11. **`notebooks/ncbi_ftp_manifest.ipynb`** — Phase 1: + - Cell 1: imports + S3 client init (`get_s3_client()`) + - Cell 2: configure parameters (database, prefix-from/to, limit, dry-run) + - Cell 3: download current assembly summary from FTP + - Cell 4: load previous summary from S3 (or scan existing prefixes) + - Cell 5: compute diff, display summary + - Cell 6: apply filters (prefix range, limit), write manifest files, upload new summary to S3 + +12. **`notebooks/ncbi_ftp_promote.ipynb`** — Phase 3: + - Cell 1: imports + S3 client init + - Cell 2: configure parameters (staging prefix, removed manifest path, release tag, manifest path for trimming, dry-run) + - Cell 3: scan staged files, display summary + - Cell 4: promote files to final paths + - Cell 5: archive replaced/suppressed assemblies + - Cell 6: trim manifest (remove promoted entries), display promotion report + +--- + +## Phase D: Container integration (Phase 2) + +13. Register CLI entry point in `pyproject.toml`: + ```toml + [project.scripts] + ncbi_ftp_sync = "cdm_data_loaders.ncbi_ftp.download:cli" + ``` + +14. Add command to `scripts/entrypoint.sh`: + ```bash + ncbi_ftp_sync) + exec /usr/bin/tini -- uv run --no-sync ncbi_ftp_sync "$@" + ;; + ``` + +15. No Dockerfile changes needed (package installed via `uv sync`; entrypoint dispatches). + +--- + +## Phase E: Tests + +All tests use **moto** for S3 mocking. No live MinIO dependency in CI. + +``` +tests/ncbi_ftp/ +├── __init__.py +├── conftest.py # Mock FTP, sample manifests, assembly records +├── test_ftp_client.py # Mock ftplib: keepalive, retry, thread-local +├── test_checksums.py # MD5 verify, CRC64/NVME, md5checksums.txt parsing +├── test_manifest.py # Summary parsing, diff logic, filtering (port from test_sync.py) +├── test_download.py # Mock FTP + fs: filters, MD5 verify, sidecars, layout +├── test_promote.py # moto S3: upload with metadata, archive, manifest trimming, dry-run +└── test_settings.py # Pydantic validation (follow test_ncbi_rest_api.py) +``` + +### Steps + +16. **`test_checksums.py`** — `verify_md5` correct/incorrect, `parse_md5_checksums_file`, + `compute_crc64nvme` (skip if `awscrt` unavailable) + +17. **`test_manifest.py`** — Port relevant tests from this repo's + [`tests/test_sync.py`](../../tests/test_sync.py): `parse_assembly_summary`, + `compute_diff` (new/updated/replaced/suppressed/withdrawn), `write_manifest`, + prefix-range filtering, limit filtering + +18. **`test_ftp_client.py`** — Mock `ftplib.FTP`: keepalive options, retry on + `error_temp`, thread-local connections + +19. **`test_download.py`** — Mock FTP + filesystem: file filter logic, MD5 + verification, sidecar writing, directory layout preserves FTP structure, + `download_report.json` + +20. **`test_promote.py`** — moto `mock_s3_client` fixture: upload with metadata, + archive copy with tags, deletion of originals, **manifest trimming** (verify + promoted entries removed, remaining entries preserved), dry-run no side effects. + Use `strip_checksum_algorithm` workaround for CRC64NVME. + +21. **`test_settings.py`** — Follow + [`test_ncbi_rest_api.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/tests/pipelines/test_ncbi_rest_api.py) + pattern: defaults, all params, CLI variants, invalid values, boolean parsing + +--- + +## Phase F: Dependencies and CI + +22. Add `awscrt` to `pyproject.toml` if not already covered by `boto3[crt]`. + +23. All new tests run automatically in CI — no `requires_spark` marks needed. + ruff checks apply (120 char lines, py313 target). + +--- + +## Key reference patterns in cdm-data-loaders + +| Pattern | Where to find it | +|---------|-----------------| +| S3 utility functions + moto tests | [`utils/s3.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/utils/s3.py), [`tests/utils/test_s3.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/tests/utils/test_s3.py) | +| CTS settings base class | [`pipelines/cts_defaults.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/pipelines/cts_defaults.py) | +| `run_cli()` entry point | [`pipelines/core.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/pipelines/core.py) | +| Logger | [`utils/cdm_logger.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/utils/cdm_logger.py) | +| Settings test pattern | [`tests/pipelines/test_ncbi_rest_api.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/tests/pipelines/test_ncbi_rest_api.py) | +| Entrypoint dispatch | [`scripts/entrypoint.sh`](https://github.com/kbase/cdm-data-loaders/blob/develop/scripts/entrypoint.sh) | +| moto CRC64NVME workaround | `strip_checksum_algorithm()` in [`tests/utils/test_s3.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/tests/utils/test_s3.py) | + +--- + +## Verification + +1. `ruff check src/cdm_data_loaders/ncbi_ftp/ tests/ncbi_ftp/` — lint passes +2. `ruff format --check src/cdm_data_loaders/ncbi_ftp/ tests/ncbi_ftp/` — formatting passes +3. `uv run pytest tests/ncbi_ftp/ -v` — all unit tests pass +4. `uv run pytest tests/utils/test_s3.py -v` — new S3 function tests pass +5. Manual: build Docker image, run `ncbi_ftp_sync` with small manifest against local MinIO +6. Manual: run both notebooks against local MinIO for end-to-end verification + +--- + +## Decisions + +- **`ncbi_ftp` naming** — distinguishes bulk FTP file transfer from the existing NCBI REST API pipeline (`ncbi_rest_api.py`) and Spark-based refseq processing (`refseq_pipeline/`) +- **New functions in `s3.py`, not modified existing ones** — minimises impact on other scripts; avoids signature changes that could break callers +- **All filtering in Phase 1** — Phase 2 is a pure download-what's-in-the-manifest step; Phase 3 is a pure sync-and-archive step. Clean separation of concerns. +- **Manifest trimming in Phase 3** — enables resumable Phase 2 runs. After promoting files, Phase 3 removes those entries from `transfer_manifest.txt`. Re-running Phase 2 only downloads what's left. +- **Output preserves FTP directory structure** — Phase 2 writes files under the same `GCF/000/001/.../assembly_dir/` path used on the FTP server, making it trivial to correlate staged files with their NCBI source +- **moto for tests** — matches cdm-data-loaders conventions; fast, no Docker in CI. The `strip_checksum_algorithm` workaround handles the CRC64NVME gap. +- **ftplib over httpx** — NCBI FTP is the established bulk download protocol; existing keepalive/NOOP/retry patterns are proven +- **Notebooks for Phases 1 & 3** — interactive, judgement-requiring steps; natural fit for JupyterLab +- **Phase 2 as CTS command** — matches the entrypoint dispatch pattern and CTS mount contract + +## Excluded from scope + +- Frictionless `datapackage.json` descriptors (only in old monolithic `download_genomes.py`) +- `backfill_checksums.py` (legacy utility, not part of ongoing pipeline) +- `download_genomes.py` monolith (superseded by the 3-phase pipeline) +- Spark/Delta Lake integration (assembly sync is file-level, not data transformation) diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb new file mode 100644 index 00000000..0cbb5295 --- /dev/null +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -0,0 +1,210 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "7a05af26", + "metadata": {}, + "source": [ + "# NCBI Assembly Manifest Generation (Phase 1)\n", + "\n", + "Downloads the current NCBI assembly summary from FTP, compares it against a\n", + "previous snapshot, and produces:\n", + "\n", + "- `transfer_manifest.txt` — assemblies to download in Phase 2\n", + "- `removed_manifest.txt` — assemblies to archive in Phase 3\n", + "- `diff_summary.json` — human-readable summary of changes\n", + "\n", + "All filtering (prefix range, limit) is applied here so downstream phases\n", + "receive a final, pre-filtered manifest." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "319383dc", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Imports and S3 client initialisation.\"\"\"\n", + "\n", + "from __future__ import annotations\n", + "\n", + "import json\n", + "from pathlib import Path\n", + "\n", + "from cdm_data_loaders.ncbi_ftp.assembly import FTP_HOST\n", + "from cdm_data_loaders.ncbi_ftp.manifest import (\n", + " AssemblyRecord,\n", + " compute_diff,\n", + " download_assembly_summary,\n", + " filter_by_prefix_range,\n", + " parse_assembly_summary,\n", + " write_diff_summary,\n", + " write_removed_manifest,\n", + " write_transfer_manifest,\n", + " write_updated_manifest,\n", + ")\n", + "from cdm_data_loaders.utils.s3 import get_s3_client, split_s3_path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4d8cdb6f", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Configure parameters.\"\"\"\n", + "\n", + "# Which NCBI database to sync: \"refseq\" or \"genbank\"\n", + "DATABASE = \"refseq\"\n", + "\n", + "# Accession prefix filtering (3-digit, inclusive). Set to None to skip.\n", + "PREFIX_FROM: str | None = None # e.g. \"000\"\n", + "PREFIX_TO: str | None = None # e.g. \"003\"\n", + "\n", + "# Maximum number of new/updated assemblies to include (None = unlimited)\n", + "LIMIT: int | None = None\n", + "\n", + "# S3 path to the previous assembly summary snapshot (set to None on first run)\n", + "PREVIOUS_SUMMARY_S3: str | None = None # e.g. \"s3://cdm-lake/tenant-general-warehouse/kbase/datasets/ncbi/metadata/assembly_summary_refseq_prev.txt\"\n", + "\n", + "# S3 path where the new snapshot will be uploaded after diffing\n", + "SNAPSHOT_UPLOAD_S3: str | None = None # e.g. \"s3://cdm-lake/tenant-general-warehouse/kbase/datasets/ncbi/metadata/assembly_summary_refseq_curr.txt\"\n", + "\n", + "# Local output directory for manifest files\n", + "OUTPUT_DIR = Path(\"output\")\n", + "OUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n", + "\n", + "# FTP hostname (default is the standard NCBI FTP server)\n", + "FTP_HOSTNAME = FTP_HOST\n", + "\n", + "print(f\"Database: {DATABASE}\")\n", + "print(f\"Prefix range: {PREFIX_FROM} -> {PREFIX_TO}\")\n", + "print(f\"Limit: {LIMIT}\")\n", + "print(f\"Output dir: {OUTPUT_DIR}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b10c3aaf", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Download current assembly summary from NCBI FTP.\"\"\"\n", + "\n", + "raw_summary = download_assembly_summary(database=DATABASE, ftp_host=FTP_HOSTNAME)\n", + "current = parse_assembly_summary(raw_summary)\n", + "print(f\"Parsed {len(current)} assemblies from current {DATABASE} summary\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "88954378", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Load previous summary from S3 (or start fresh).\"\"\"\n", + "\n", + "previous: dict[str, AssemblyRecord] | None = None\n", + "\n", + "if PREVIOUS_SUMMARY_S3:\n", + " s3 = get_s3_client()\n", + " bucket, key = split_s3_path(PREVIOUS_SUMMARY_S3)\n", + " resp = s3.get_object(Bucket=bucket, Key=key)\n", + " prev_text = resp[\"Body\"].read().decode(\"utf-8\")\n", + " previous = parse_assembly_summary(prev_text)\n", + " print(f\"Loaded {len(previous)} assemblies from previous snapshot\")\n", + "else:\n", + " print(\"No previous snapshot — all current 'latest' assemblies will be marked as new\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18482b3c", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Compute diff and apply filters.\"\"\"\n", + "\n", + "# Filter current assemblies by prefix range\n", + "filtered = filter_by_prefix_range(current, prefix_from=PREFIX_FROM, prefix_to=PREFIX_TO)\n", + "print(f\"After prefix filter: {len(filtered)} assemblies\")\n", + "\n", + "# Also filter previous if present\n", + "filtered_prev = filter_by_prefix_range(previous, prefix_from=PREFIX_FROM, prefix_to=PREFIX_TO) if previous else None\n", + "\n", + "# Compute diff\n", + "diff = compute_diff(filtered, previous_assemblies=filtered_prev)\n", + "\n", + "print(f\"New: {len(diff.new)}\")\n", + "print(f\"Updated: {len(diff.updated)}\")\n", + "print(f\"Replaced: {len(diff.replaced)}\")\n", + "print(f\"Suppressed: {len(diff.suppressed)}\")\n", + "print(f\"Total to transfer: {len(diff.new) + len(diff.updated)}\")\n", + "print(f\"Total to remove: {len(diff.replaced) + len(diff.suppressed)}\")\n", + "\n", + "# Apply limit if set\n", + "if LIMIT is not None:\n", + " original_new = len(diff.new)\n", + " original_updated = len(diff.updated)\n", + " combined = diff.new + diff.updated\n", + " limited = combined[:LIMIT]\n", + " diff.new = [a for a in diff.new if a in set(limited)]\n", + " diff.updated = [a for a in diff.updated if a in set(limited)]\n", + " print(f\"After limit ({LIMIT}): {len(diff.new)} new, {len(diff.updated)} updated\")\n", + " print(f\" (was {original_new} new, {original_updated} updated)\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d9e2b631", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Write manifest files and upload snapshot to S3.\"\"\"\n", + "\n", + "# Write transfer manifest\n", + "transfer_path = OUTPUT_DIR / \"transfer_manifest.txt\"\n", + "paths = write_transfer_manifest(diff, filtered, transfer_path, ftp_host=FTP_HOSTNAME)\n", + "print(f\"Transfer manifest: {len(paths)} entries -> {transfer_path}\")\n", + "\n", + "# Write removed manifest\n", + "removed_path = OUTPUT_DIR / \"removed_manifest.txt\"\n", + "removed = write_removed_manifest(diff, removed_path)\n", + "print(f\"Removed manifest: {len(removed)} entries -> {removed_path}\")\n", + "\n", + "# Write updated manifest (for Phase 3 pre-overwrite archiving)\n", + "updated_path = OUTPUT_DIR / \"updated_manifest.txt\"\n", + "updated = write_updated_manifest(diff, updated_path)\n", + "print(f\"Updated manifest: {len(updated)} entries -> {updated_path}\")\n", + "\n", + "# Write diff summary\n", + "summary_path = OUTPUT_DIR / \"diff_summary.json\"\n", + "summary = write_diff_summary(diff, summary_path, DATABASE, PREFIX_FROM, PREFIX_TO)\n", + "print(f\"Diff summary -> {summary_path}\")\n", + "print(json.dumps(summary[\"counts\"], indent=2))\n", + "\n", + "# Upload new snapshot to S3 for future diffing\n", + "if SNAPSHOT_UPLOAD_S3:\n", + " s3 = get_s3_client()\n", + " bucket, key = split_s3_path(SNAPSHOT_UPLOAD_S3)\n", + " s3.put_object(Bucket=bucket, Key=key, Body=raw_summary.encode(\"utf-8\"))\n", + " print(f\"Uploaded new snapshot to {SNAPSHOT_UPLOAD_S3}\")\n", + "else:\n", + " print(\"Skipping S3 snapshot upload (SNAPSHOT_UPLOAD_S3 not set)\")" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/ncbi_ftp_promote.ipynb b/notebooks/ncbi_ftp_promote.ipynb new file mode 100644 index 00000000..de19c0e6 --- /dev/null +++ b/notebooks/ncbi_ftp_promote.ipynb @@ -0,0 +1,172 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "2eda19a9", + "metadata": {}, + "source": [ + "# NCBI Assembly Promote & Archive (Phase 3)\n", + "\n", + "Promotes staged assembly files from S3 staging (written by CTS Phase 2)\n", + "to their final Lakehouse paths, archives replaced/suppressed assemblies,\n", + "and trims the transfer manifest for resumability.\n", + "\n", + "Steps:\n", + "1. Configure staging prefix, removed manifest, updated manifest, and release tag\n", + "2. Scan staged files and display summary\n", + "3. Archive existing versions of updated assemblies (pre-overwrite)\n", + "4. Promote files to final paths with MD5 metadata\n", + "5. Archive replaced/suppressed assemblies\n", + "6. Trim manifest (remove promoted entries)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9b736665", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Imports and S3 client initialisation.\"\"\"\n", + "\n", + "from __future__ import annotations\n", + "\n", + "import json\n", + "\n", + "from cdm_data_loaders.ncbi_ftp.promote import (\n", + " DEFAULT_PATH_PREFIX,\n", + " promote_from_s3,\n", + ")\n", + "from cdm_data_loaders.utils.s3 import get_s3_client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b36a556c", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Configure parameters.\"\"\"\n", + "\n", + "# S3 bucket where staged files and final Lakehouse data live\n", + "BUCKET = \"cdm-lake\" # e.g. \"cdm-lake\"\n", + "\n", + "# Staging prefix written by CTS Phase 2 (from the CTS output mount)\n", + "STAGING_PREFIX = \"staging/run1/\" # e.g. \"staging/run1/\"\n", + "\n", + "# Local path to removed_manifest.txt from Phase 1 (or None to skip archiving)\n", + "REMOVED_MANIFEST: str | None = None # e.g. \"output/removed_manifest.txt\"\n", + "\n", + "# Local path to updated_manifest.txt from Phase 1 (or None to skip pre-overwrite archiving)\n", + "UPDATED_MANIFEST: str | None = None # e.g. \"output/updated_manifest.txt\"\n", + "\n", + "# NCBI release tag for archive metadata (e.g. \"2024-01\")\n", + "NCBI_RELEASE: str | None = None\n", + "\n", + "# S3 key of transfer_manifest.txt for trimming (or None to skip)\n", + "MANIFEST_S3_KEY: str | None = None # e.g. \"ncbi/transfer_manifest.txt\"\n", + "\n", + "# Final Lakehouse path prefix\n", + "PATH_PREFIX = DEFAULT_PATH_PREFIX\n", + "\n", + "# Dry-run mode — log actions without making changes\n", + "DRY_RUN = True\n", + "\n", + "print(f\"Updated manifest: {UPDATED_MANIFEST}\")\n", + "print(f\"NCBI release: {NCBI_RELEASE}\")\n", + "print(f\"Manifest S3 key: {MANIFEST_S3_KEY}\")\n", + "print(f\"Path prefix: {PATH_PREFIX}\")\n", + "\n", + "print(f\"Dry-run: {DRY_RUN}\")\n", + "print(f\"Path prefix: {PATH_PREFIX}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e521fd45", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Scan staged files and display summary.\"\"\"\n", + "\n", + "s3 = get_s3_client()\n", + "paginator = s3.get_paginator(\"list_objects_v2\")\n", + "\n", + "staged: list[str] = []\n", + "for page in paginator.paginate(Bucket=BUCKET, Prefix=STAGING_PREFIX):\n", + " staged.extend(obj[\"Key\"] for obj in page.get(\"Contents\", []))\n", + "\n", + "sidecars = [k for k in staged if k.endswith((\".md5\", \".crc64nvme\"))]\n", + "data_files = [k for k in staged if k not in set(sidecars)]\n", + "\n", + "print(f\"Staged objects: {len(staged)}\")\n", + "print(f\" Data files: {len(data_files)}\")\n", + "print(f\" Sidecars: {len(sidecars)}\")\n", + "\n", + "# Show first few data files\n", + "PREVIEW_COUNT = 10\n", + "for key in data_files[:PREVIEW_COUNT]:\n", + " print(f\" {key}\")\n", + "if len(data_files) > PREVIEW_COUNT:\n", + " print(f\" ... and {len(data_files) - PREVIEW_COUNT} more\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10a46367", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Promote staged files to final Lakehouse paths.\"\"\"\n", + "\n", + "report = promote_from_s3(\n", + " staging_prefix=STAGING_PREFIX,\n", + " bucket=BUCKET,\n", + " removed_manifest=REMOVED_MANIFEST,\n", + " updated_manifest=UPDATED_MANIFEST,\n", + " ncbi_release=NCBI_RELEASE,\n", + " manifest_path=MANIFEST_S3_KEY,\n", + " path_prefix=PATH_PREFIX,\n", + " dry_run=DRY_RUN,\n", + ")\n", + "\n", + "print(json.dumps(report, indent=2))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9d18a1e0", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Display promotion report.\"\"\"\n", + "\n", + "print(\"=\" * 50)\n", + "print(\"PROMOTION REPORT\")\n", + "print(\"=\" * 50)\n", + "print(f\"Promoted: {report['promoted']}\")\n", + "print(f\"Archived: {report['archived']}\")\n", + "print(f\"Failed: {report['failed']}\")\n", + "print(f\"Dry-run: {report['dry_run']}\")\n", + "print(f\"Timestamp: {report['timestamp']}\")\n", + "\n", + "if report['failed'] > 0:\n", + " print(\"\\n⚠️ Some operations failed — check logs above for details.\")\n", + "\n", + "if report['dry_run']:\n", + " print(\"\\n📋 This was a dry-run. Set DRY_RUN = False and re-run to apply changes.\")" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyproject.toml b/pyproject.toml index c90538a0..5898efde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ dependencies = [ uniprot = "cdm_data_loaders.pipelines.uniprot_kb:cli" uniref = "cdm_data_loaders.pipelines.uniref:cli" ncbi_rest_api = "cdm_data_loaders.pipelines.ncbi_rest_api:cli" +ncbi_ftp_sync = "cdm_data_loaders.pipelines.ncbi_ftp_download:cli" [dependency-groups] dev = [ @@ -170,7 +171,7 @@ ignore = [ [tool.ruff.lint.per-file-ignores] "*.ipynb" = ["T201"] # ignore printing in notebooks -"tests/**/*.py" = ["S101", "T201", "FBT001", "FBT002"] # use of assert, booleans +"tests/**/*.py" = ["S101", "T201", "FBT001", "FBT002", "ARG002"] # use of assert, booleans, unused mock args "tests/utils/test_s3.py" = ["ANN401"] "**/__init__.py" = ["D104"] diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index 74d90521..ee087591 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -3,7 +3,7 @@ set -euo pipefail # Ensure at least one argument is provided if [ "$#" -eq 0 ]; then - echo "Usage: $0 {uniref|uniprot|ncbi_rest_api|xml_split|test} [args...]" + echo "Usage: $0 {uniref|uniprot|ncbi_rest_api|ncbi_ftp_sync|xml_split|test} [args...]" exit 1 fi @@ -27,6 +27,10 @@ case "$cmd" in # Run the NCBI datasets API importer exec /usr/bin/tini -- uv run --no-sync ncbi_rest_api "$@" ;; + ncbi_ftp_sync) + # Run the NCBI FTP assembly download pipeline (Phase 2) + exec /usr/bin/tini -- uv run --no-sync ncbi_ftp_sync "$@" + ;; test) # run the tests exec /usr/bin/tini -- uv run --no-sync pytest -m "not requires_spark" @@ -35,7 +39,7 @@ case "$cmd" in exec /usr/bin/tini -- /bin/bash ;; *) - echo "Error: unknown command '$cmd'; valid commands are 'uniref', 'uniprot', 'ncbi_rest_api', or 'xml_split'." >&2 + echo "Error: unknown command '$cmd'; valid commands are 'uniref', 'uniprot', 'ncbi_rest_api', 'ncbi_ftp_sync', or 'xml_split'." >&2 exit 1 ;; esac diff --git a/src/cdm_data_loaders/ncbi_ftp/__init__.py b/src/cdm_data_loaders/ncbi_ftp/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/cdm_data_loaders/ncbi_ftp/assembly.py b/src/cdm_data_loaders/ncbi_ftp/assembly.py new file mode 100644 index 00000000..cd2981ef --- /dev/null +++ b/src/cdm_data_loaders/ncbi_ftp/assembly.py @@ -0,0 +1,206 @@ +"""NCBI FTP assembly-specific domain logic. + +Provides path helpers, file filters, MD5 checksum parsing, and single-assembly +download logic for NCBI GenBank/RefSeq assemblies. Orchestration (batching, +threading, CLI) lives in :mod:`cdm_data_loaders.pipelines.ncbi_ftp_download`. +""" + +import contextlib +import re +import time +from ftplib import FTP +from pathlib import Path +from typing import Any + +from cdm_data_loaders.utils.cdm_logger import get_cdm_logger +from cdm_data_loaders.utils.checksums import compute_md5 +from cdm_data_loaders.utils.ftp_client import connect_ftp, ftp_noop_keepalive, ftp_retrieve_text + +logger = get_cdm_logger() + +FTP_HOST = "ftp.ncbi.nlm.nih.gov" + +FILE_FILTERS = [ + "_gene_ontology.gaf.gz", + "_genomic.fna.gz", + "_genomic.gff.gz", + "_protein.faa.gz", + "_ani_contam_ranges.tsv", + "_assembly_regions.txt", + "_assembly_report.txt", + "_assembly_stats.txt", + "_gene_expression_counts.txt.gz", + "_normalized_gene_expression_counts.txt.gz", +] + + +def parse_md5_checksums_file(text: str) -> dict[str, str]: + """Parse an NCBI ``md5checksums.txt`` file into a filename-to-hash mapping. + + Each line has the format `` ./`` (two-space separator). + + :param text: raw text of the md5checksums.txt file + :return: dict mapping filename to MD5 hex digest + """ + checksums: dict[str, str] = {} + for raw_line in text.strip().splitlines(): + stripped = raw_line.strip() + if not stripped: + continue + parts = stripped.split(" ", maxsplit=1) + if len(parts) == 2: # noqa: PLR2004 + md5_hash, filename = parts + checksums[filename.removeprefix("./")] = md5_hash.strip() + return checksums + + +# ── Path helpers ───────────────────────────────────────────────────────── + + +def build_accession_path(assembly_dir: str) -> str: + """Build the relative output path for an assembly directory. + + Produces ``raw_data/{GCF|GCA}/{000}/{001}/{215}/{assembly_dir}/``. + + :param assembly_dir: full assembly directory name (e.g. ``GCF_000001215.4_Release_6...``) + :return: relative path string + :raises ValueError: if the assembly directory name cannot be parsed + """ + m = re.match(r"GC[AF]_(\d{3})(\d{3})(\d{3})\.\d+.*", assembly_dir) + if not m: + msg = f"Cannot parse accession: {assembly_dir}" + raise ValueError(msg) + p1, p2, p3 = m.groups() + return f"raw_data/{assembly_dir[:3]}/{p1}/{p2}/{p3}/{assembly_dir}/" + + +def parse_assembly_path(assembly_path: str) -> tuple[str, str, str]: + """Extract database, assembly_dir, and accession from an FTP assembly path. + + :param assembly_path: FTP directory path (e.g. ``/genomes/all/GCF/000/.../GCF_000001215.4_Rel.../``) + :return: tuple of ``(database, assembly_dir, accession)`` + :raises ValueError: if the path cannot be parsed + """ + m = re.search( + r"/(GC[AF])/\d{3}/\d{3}/\d{3}/((GC[AF]_\d{9}\.\d+)_[^/]+)/?$", + assembly_path.rstrip("/"), + ) + if not m: + msg = f"Cannot parse assembly path: {assembly_path}" + raise ValueError(msg) + return m.group(1), m.group(2), m.group(3) + + +# ── Single assembly download ──────────────────────────────────────────── + + +def _download_and_verify( # noqa: PLR0913 + ftp: FTP, + filename: str, + dest_dir: Path, + md5_checksums: dict[str, str], + stats: dict[str, Any], + last_activity: float, +) -> float: + """Download one file, verify its MD5, and write a sidecar if valid.""" + last_activity = ftp_noop_keepalive(ftp, last_activity) + local_file = dest_dir / filename + expected_md5 = md5_checksums.get(filename) + + for attempt in range(1, 4): + logger.debug(" Downloading %s (attempt %d/3)", filename, attempt) + with local_file.open("wb") as f: + ftp.retrbinary(f"RETR {filename}", f.write) + last_activity = time.monotonic() + + if expected_md5: + actual_md5 = compute_md5(str(local_file)) + if actual_md5 != expected_md5: + logger.warning( + " MD5 mismatch for %s: expected %s, got %s", + filename, + expected_md5, + actual_md5, + ) + if attempt < 3: # noqa: PLR2004 + continue + stats["files_skipped_checksum_mismatch"] += 1 + local_file.unlink(missing_ok=True) + return last_activity + logger.debug(" MD5 verified: %s", filename) + else: + stats["files_without_checksum"] += 1 + + if expected_md5: + (dest_dir / f"{filename}.md5").write_text(expected_md5) + + stats["files_downloaded"] += 1 + return last_activity + + return last_activity + + +def download_assembly_to_local( + assembly_path: str, + output_dir: str | Path, + ftp_host: str = FTP_HOST, + ftp: FTP | None = None, +) -> dict[str, Any]: + """Download one assembly from NCBI FTP to a local directory. + + Creates a directory structure under *output_dir* matching the S3 layout, + downloads filtered files, verifies MD5 checksums, and writes ``.md5`` + sidecar files for downstream metadata. + + :param assembly_path: FTP directory path for the assembly + :param output_dir: base output directory + :param ftp_host: FTP hostname + :param ftp: optional existing FTP connection (caller manages lifecycle) + :return: dict with download statistics + """ + _database, assembly_dir, accession = parse_assembly_path(assembly_path) + rel_path = build_accession_path(assembly_dir) + dest_dir = Path(output_dir) / rel_path + dest_dir.mkdir(parents=True, exist_ok=True) + + logger.info("Downloading %s -> %s", accession, dest_dir) + + owns_ftp = ftp is None + if owns_ftp: + ftp = connect_ftp(ftp_host) + stats: dict[str, Any] = { + "accession": accession, + "assembly_dir": assembly_dir, + "files_downloaded": 0, + "files_skipped_checksum_mismatch": 0, + "files_without_checksum": 0, + } + + try: + ftp.cwd(assembly_path.rstrip("/")) + + files: list[str] = [] + ftp.retrlines("NLST", files.append) + + # Download and parse md5checksums.txt + md5_checksums: dict[str, str] = {} + if "md5checksums.txt" in files: + md5_text = ftp_retrieve_text(ftp, "md5checksums.txt") + md5_checksums = parse_md5_checksums_file(md5_text) + (dest_dir / "md5checksums.txt").write_text(md5_text) + stats["files_downloaded"] += 1 + + target_files = [f for f in files if any(f.endswith(s) for s in FILE_FILTERS)] + last_activity = time.monotonic() + + for filename in target_files: + last_activity = _download_and_verify(ftp, filename, dest_dir, md5_checksums, stats, last_activity) + + logger.info(" %s: %d files downloaded", accession, stats["files_downloaded"]) + + finally: + if owns_ftp: + with contextlib.suppress(Exception): + ftp.quit() + + return stats diff --git a/src/cdm_data_loaders/ncbi_ftp/manifest.py b/src/cdm_data_loaders/ncbi_ftp/manifest.py new file mode 100644 index 00000000..21aa084d --- /dev/null +++ b/src/cdm_data_loaders/ncbi_ftp/manifest.py @@ -0,0 +1,471 @@ +"""Phase 1: Assembly summary diffing and manifest generation. + +Downloads the current NCBI assembly summary from FTP, compares it against a +previous snapshot, and produces ``transfer_manifest.txt`` (assemblies to +download), ``removed_manifest.txt`` (assemblies to archive), and a JSON diff +summary. All filtering logic (prefix range, limit) lives here so that +downstream phases receive a final, pre-filtered manifest. +""" + +import contextlib +import csv +import json +import re +import time +from collections.abc import Iterable +from dataclasses import dataclass, field +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from cdm_data_loaders.ncbi_ftp.assembly import ( + FILE_FILTERS, + FTP_HOST, + build_accession_path, + parse_md5_checksums_file, +) +from cdm_data_loaders.utils.cdm_logger import get_cdm_logger +from cdm_data_loaders.utils.ftp_client import connect_ftp, ftp_noop_keepalive, ftp_retrieve_text +from cdm_data_loaders.utils.s3 import head_object + +logger = get_cdm_logger() + +SUMMARY_FTP_PATHS: dict[str, str] = { + "refseq": "/genomes/ASSEMBLY_REPORTS/assembly_summary_refseq.txt", + "genbank": "/genomes/ASSEMBLY_REPORTS/assembly_summary_genbank.txt", +} + + +# ── Data structures ───────────────────────────────────────────────────── + + +@dataclass +class AssemblyRecord: + """Parsed row from an NCBI assembly summary file.""" + + accession: str + status: str + seq_rel_date: str + ftp_path: str + assembly_dir: str + + +@dataclass +class DiffResult: + """Result of comparing current and previous assembly summaries.""" + + new: list[str] = field(default_factory=list) + updated: list[str] = field(default_factory=list) + replaced: list[str] = field(default_factory=list) + suppressed: list[str] = field(default_factory=list) + + +# ── Assembly summary download & parsing ────────────────────────────────── + + +def download_assembly_summary(database: str = "refseq", ftp_host: str = FTP_HOST) -> str: + """Download the assembly summary file from NCBI FTP. + + :param database: ``"refseq"`` or ``"genbank"`` + :param ftp_host: FTP hostname + :return: raw text content of the summary file + """ + ftp_path = SUMMARY_FTP_PATHS.get(database) + if not ftp_path: + msg = f"Unknown database: {database}" + raise ValueError(msg) + + logger.info("Downloading assembly_summary_%s.txt from NCBI FTP ...", database) + ftp = connect_ftp(ftp_host) + try: + content = ftp_retrieve_text(ftp, ftp_path) + finally: + with contextlib.suppress(Exception): + ftp.quit() + + logger.info("Downloaded assembly summary (%d bytes)", len(content)) + return content + + +def parse_assembly_summary(source: str | Path | list[str]) -> dict[str, AssemblyRecord]: + """Parse an NCBI assembly summary into a dict of assembly records. + + Accepts a file path, raw text string, or list of lines. + + Columns of interest (0-indexed): + 0: assembly_accession (e.g. GCF_000001215.4) + 10: version_status ("latest", "replaced", "suppressed") + 14: seq_rel_date + 19: ftp_path (full FTP URL or "na") + + :param source: file path, raw text, or list of lines + :return: dict mapping accession to :class:`AssemblyRecord` + """ + assemblies: dict[str, AssemblyRecord] = {} + + def _parse_lines(lines: Iterable[str]) -> None: + reader = csv.reader( + (line.rstrip("\n") for line in lines if not line.startswith("#")), + delimiter="\t", + ) + for row in reader: + if len(row) < 20: # noqa: PLR2004 + continue + accession = row[0] + ftp_path = row[19] + if ftp_path == "na": + continue + assemblies[accession] = AssemblyRecord( + accession=accession, + status=row[10], + seq_rel_date=row[14], + ftp_path=ftp_path, + assembly_dir=ftp_path.rstrip("/").split("/")[-1], + ) + + if isinstance(source, Path) or (isinstance(source, str) and "\n" not in source and Path(source).is_file()): + with Path(source).open() as f: + _parse_lines(f) + elif isinstance(source, list): + _parse_lines(source) + else: + _parse_lines(source.splitlines(keepends=True)) + + logger.info("Parsed %d assemblies from summary", len(assemblies)) + return assemblies + + +def get_latest_assembly_paths(assemblies: dict[str, AssemblyRecord], ftp_host: str = FTP_HOST) -> list[tuple[str, str]]: + """Extract FTP directory paths for all assemblies with ``latest`` status. + + :param assemblies: parsed assembly records + :param ftp_host: FTP hostname for URL stripping + :return: list of ``(accession, ftp_dir_path)`` tuples + """ + paths: list[tuple[str, str]] = [] + for accession, rec in assemblies.items(): + if rec.status != "latest": + continue + ftp_path = _ftp_dir_from_url(rec.ftp_path, ftp_host) + paths.append((accession, ftp_path.rstrip("/") + "/")) + return paths + + +# ── Prefix filtering ──────────────────────────────────────────────────── + + +def accession_prefix(accession: str) -> str | None: + """Extract the 3-digit prefix from an accession (e.g. ``GCF_000005845.2`` → ``"000"``).""" + m = re.match(r"GC[AF]_(\d{3})\d{6}\.\d+", accession) + return m.group(1) if m else None + + +def filter_by_prefix_range( + assemblies: dict[str, AssemblyRecord], + prefix_from: str | None = None, + prefix_to: str | None = None, +) -> dict[str, AssemblyRecord]: + """Filter assemblies to those whose 3-digit accession prefix is in range. + + Both bounds are inclusive. If neither is set, returns all assemblies. + + :param assemblies: dict of parsed assembly records + :param prefix_from: lower bound (inclusive), e.g. ``"000"`` + :param prefix_to: upper bound (inclusive), e.g. ``"003"`` + :return: filtered dict + """ + if prefix_from is None and prefix_to is None: + return assemblies + filtered: dict[str, AssemblyRecord] = {} + for acc, rec in assemblies.items(): + pfx = accession_prefix(acc) + if pfx is None: + continue + if prefix_from is not None and pfx < prefix_from: + continue + if prefix_to is not None and pfx > prefix_to: + continue + filtered[acc] = rec + return filtered + + +# ── Diff computation ──────────────────────────────────────────────────── + + +def compute_diff( # noqa: PLR0912 + current: dict[str, AssemblyRecord], + previous_assemblies: dict[str, AssemblyRecord] | None = None, + previous_accessions: set[str] | None = None, +) -> DiffResult: + """Compute the diff between current and previous assembly state. + + :param current: the new NCBI summary (parsed) + :param previous_assemblies: full parsed previous summary, or None if using fallback + :param previous_accessions: set of known accessions (store-scan fallback) + :return: diff result with new/updated/replaced/suppressed lists + """ + diff = DiffResult() + + if previous_assemblies is not None: + known = set(previous_assemblies.keys()) + elif previous_accessions is not None: + known = previous_accessions + else: + known = set() + + for acc, rec in current.items(): + if rec.status == "replaced": + if acc in known: + diff.replaced.append(acc) + continue + if rec.status == "suppressed": + if acc in known: + diff.suppressed.append(acc) + continue + if rec.status != "latest": + continue + + if acc not in known: + diff.new.append(acc) + elif previous_assemblies is not None: + prev = previous_assemblies.get(acc) + if prev and (rec.seq_rel_date != prev.seq_rel_date or rec.assembly_dir != prev.assembly_dir): + diff.updated.append(acc) + + # Accessions in previous but entirely absent from current (withdrawn) + current_accs = set(current.keys()) + for acc in known: + if acc not in current_accs and acc not in diff.suppressed: + diff.suppressed.append(acc) + + diff.new.sort() + diff.updated.sort() + diff.replaced.sort() + diff.suppressed.sort() + return diff + + +# ── FTP URL helpers ────────────────────────────────────────────────────── + + +def _ftp_dir_from_url(ftp_url: str, ftp_host: str = FTP_HOST) -> str: + """Convert an FTP URL from the assembly summary to an FTP directory path.""" + if ftp_url.startswith("https://"): + return ftp_url.replace("https://ftp.ncbi.nlm.nih.gov", "") + if ftp_url.startswith("ftp://"): + return ftp_url.replace(f"ftp://{ftp_host}", "") + return ftp_url + + +# ── Checksum verification against S3 store ─────────────────────────────── + + +def verify_transfer_candidates( + accessions: list[str], + current_assemblies: dict[str, AssemblyRecord], + bucket: str, + path_prefix: str, + ftp_host: str = FTP_HOST, +) -> list[str]: + """Verify which transfer candidates actually need downloading. + + For each accession, downloads ``md5checksums.txt`` from NCBI FTP and + compares the checksums of filtered files against the ``md5`` user metadata + on corresponding S3 objects. Only accessions where at least one file + differs or is missing from S3 are returned. + + This acts as a final gate before Phase 2: even if the summary diff flags an + assembly, we skip it if every file in the store already matches. + + :param accessions: list of candidate accessions (new + updated from diff) + :param current_assemblies: parsed current assembly summary + :param bucket: S3 bucket name + :param path_prefix: Lakehouse path prefix (e.g. ``"tenant-general-warehouse/kbase/datasets/ncbi/"``) + :param ftp_host: NCBI FTP hostname + :return: filtered list of accessions that actually need downloading + """ + if not accessions: + return [] + + ftp = connect_ftp(ftp_host) + confirmed: list[str] = [] + pruned = 0 + last_activity = time.monotonic() + + try: + for acc in accessions: + rec = current_assemblies.get(acc) + if not rec: + confirmed.append(acc) + continue + + # Keep FTP alive between assemblies + last_activity = ftp_noop_keepalive(ftp, last_activity) + + # Download md5checksums.txt from FTP + ftp_dir = _ftp_dir_from_url(rec.ftp_path, ftp_host) + try: + md5_text = ftp_retrieve_text(ftp, ftp_dir.rstrip("/") + "/md5checksums.txt") + last_activity = time.monotonic() + ftp_checksums = parse_md5_checksums_file(md5_text) + except Exception: # noqa: BLE001 + logger.warning("Cannot fetch md5checksums.txt for %s, keeping in transfer list", acc) + confirmed.append(acc) + continue + + # Filter to files we'd actually download + target_checksums = { + fname: md5 + for fname, md5 in ftp_checksums.items() + if any(fname.endswith(suffix) for suffix in FILE_FILTERS) + } + + if not target_checksums: + confirmed.append(acc) + continue + + # Build S3 prefix for this assembly + s3_rel = build_accession_path(rec.assembly_dir) + + # Short-circuit: if any file differs or is missing, keep the assembly + needs_update = False + for fname, expected_md5 in target_checksums.items(): + s3_path = f"{bucket}/{path_prefix}{s3_rel}{fname}" + obj_info = head_object(s3_path) + + if obj_info is None: + needs_update = True + break + + s3_md5 = obj_info["metadata"].get("md5", "") + if s3_md5 != expected_md5: + logger.debug("MD5 mismatch for %s/%s: S3=%s FTP=%s", acc, fname, s3_md5, expected_md5) + needs_update = True + break + + if needs_update: + confirmed.append(acc) + else: + pruned += 1 + logger.debug("Pruned %s — all files match S3 checksums", acc) + finally: + with contextlib.suppress(Exception): + ftp.quit() + + logger.info( + "Checksum verification: %d confirmed, %d pruned (of %d candidates)", + len(confirmed), + pruned, + len(accessions), + ) + return confirmed + + +# ── Manifest writing ──────────────────────────────────────────────────── + + +def write_transfer_manifest( + diff: DiffResult, + current_assemblies: dict[str, AssemblyRecord], + output_path: str | Path, + ftp_host: str = FTP_HOST, +) -> list[str]: + """Write the transfer manifest (new + updated assemblies). + + Each line is an FTP directory path suitable for Phase 2 download. + + :param diff: computed diff result + :param current_assemblies: parsed current assembly summary + :param output_path: path to write the manifest file + :param ftp_host: FTP hostname for URL stripping + :return: list of FTP paths written + """ + to_transfer = diff.new + diff.updated + paths: list[str] = [] + for acc in sorted(to_transfer): + rec = current_assemblies.get(acc) + if not rec: + continue + ftp_path = _ftp_dir_from_url(rec.ftp_path, ftp_host) + paths.append(ftp_path.rstrip("/") + "/") + + with Path(output_path).open("w") as f: + f.writelines(p + "\n" for p in paths) + + logger.info("Wrote %d entries to transfer manifest: %s", len(paths), output_path) + return paths + + +def write_removed_manifest(diff: DiffResult, output_path: str | Path) -> list[str]: + """Write the removed manifest (replaced + suppressed accessions). + + :param diff: computed diff result + :param output_path: path to write the manifest file + :return: list of accessions written + """ + removed = sorted(diff.replaced + diff.suppressed) + with Path(output_path).open("w") as f: + f.writelines(acc + "\n" for acc in removed) + logger.info("Wrote %d entries to removed manifest: %s", len(removed), output_path) + return removed + + +def write_updated_manifest(diff: DiffResult, output_path: str | Path) -> list[str]: + """Write the updated manifest (accessions whose content changed). + + This file is consumed by Phase 3 to archive existing S3 objects + before they are overwritten by the new versions. + + :param diff: computed diff result + :param output_path: path to write the manifest file + :return: list of accessions written + """ + updated = sorted(diff.updated) + with Path(output_path).open("w") as f: + f.writelines(acc + "\n" for acc in updated) + logger.info("Wrote %d entries to updated manifest: %s", len(updated), output_path) + return updated + + +def write_diff_summary( + diff: DiffResult, + output_path: str | Path, + database: str, + prefix_from: str | None = None, + prefix_to: str | None = None, +) -> dict[str, Any]: + """Write a JSON diff summary file. + + :param diff: computed diff result + :param output_path: path to write the JSON file + :param database: database name (``"refseq"`` or ``"genbank"``) + :param prefix_from: lower bound of prefix filter (if any) + :param prefix_to: upper bound of prefix filter (if any) + :return: the summary dict that was written + """ + summary: dict[str, Any] = { + "database": database, + "timestamp": datetime.now(UTC).isoformat(), + "prefix_range": { + "from": prefix_from, + "to": prefix_to, + }, + "counts": { + "new": len(diff.new), + "updated": len(diff.updated), + "replaced": len(diff.replaced), + "suppressed": len(diff.suppressed), + "total_to_transfer": len(diff.new) + len(diff.updated), + "total_to_remove": len(diff.replaced) + len(diff.suppressed), + }, + "accessions": { + "new": diff.new, + "updated": diff.updated, + "replaced": diff.replaced, + "suppressed": diff.suppressed, + }, + } + with Path(output_path).open("w") as f: + json.dump(summary, f, indent=2) + logger.info("Wrote diff summary to: %s", output_path) + return summary diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py new file mode 100644 index 00000000..37f1fa82 --- /dev/null +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -0,0 +1,281 @@ +"""Phase 3: Promote staged files to final Lakehouse paths in S3. + +Walks staged files in an S3 staging prefix (written by CTS after Phase 2), +uploads each to the final Lakehouse path with MD5 metadata from sidecar files, +archives replaced/suppressed and updated assemblies, and trims the transfer +manifest so that a re-run of Phase 2 only downloads remaining entries. +""" + +import re +import tempfile +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from cdm_data_loaders.utils.cdm_logger import get_cdm_logger +from cdm_data_loaders.utils.s3 import ( + copy_object_with_metadata, + delete_object, + get_s3_client, + upload_file_with_metadata, +) + +logger = get_cdm_logger() + +DEFAULT_PATH_PREFIX = "tenant-general-warehouse/kbase/datasets/ncbi/" + + +# ── Promote from S3 staging prefix ────────────────────────────────────── + + +def promote_from_s3( # noqa: PLR0913 + staging_prefix: str, + bucket: str, + removed_manifest: str | Path | None = None, + updated_manifest: str | Path | None = None, + ncbi_release: str | None = None, + manifest_path: str | None = None, + path_prefix: str = DEFAULT_PATH_PREFIX, + *, + dry_run: bool = False, +) -> dict[str, Any]: + """Promote files from an S3 staging prefix to the final Lakehouse path. + + Downloads each file to a temp location and re-uploads to the final path + with MD5 metadata from ``.md5`` sidecar files. + + :param staging_prefix: S3 key prefix where CTS output was written + :param bucket: S3 bucket name + :param removed_manifest: local path to the removed_manifest file + :param updated_manifest: local path to the updated_manifest file + :param ncbi_release: NCBI release version tag for archiving + :param manifest_path: S3 path to transfer_manifest.txt for trimming + :param path_prefix: Lakehouse path prefix for final locations + :param dry_run: if True, log actions without side effects + :return: report dict with counts + """ + s3 = get_s3_client() + paginator = s3.get_paginator("list_objects_v2") + + promoted = 0 + failed = 0 + + # Collect all objects under the staging prefix + staged_objects: list[str] = [] + for page in paginator.paginate(Bucket=bucket, Prefix=staging_prefix): + staged_objects.extend(obj["Key"] for obj in page.get("Contents", [])) + + # Separate data files from sidecars + sidecars = {k for k in staged_objects if k.endswith((".crc64nvme", ".md5"))} + data_files = [k for k in staged_objects if k not in sidecars] + + logger.info("Found %d data files and %d sidecars in staging", len(data_files), len(sidecars)) + + # Archive all affected assemblies BEFORE promoting or deleting + archived = 0 + for manifest_file, reason, delete in [ + (updated_manifest, "updated", False), + (removed_manifest, "replaced_or_suppressed", True), + ]: + if manifest_file and Path(str(manifest_file)).is_file(): + archived += _archive_assemblies( + str(manifest_file), + bucket=bucket, + ncbi_release=ncbi_release, + path_prefix=path_prefix, + archive_reason=reason, + delete_source=delete, + dry_run=dry_run, + ) + + promoted_accessions: set[str] = set() + + for staged_key in data_files: + if staged_key.endswith("download_report.json"): + continue + + rel_path = staged_key[len(staging_prefix) :] + if not rel_path.startswith("raw_data/"): + continue + final_key = path_prefix + rel_path + + if dry_run: + logger.info("[dry-run] would promote: %s -> %s", staged_key, final_key) + promoted += 1 + continue + + try: + with tempfile.NamedTemporaryFile(delete=False) as tmp: + tmp_path = tmp.name + try: + s3.download_file(Bucket=bucket, Key=staged_key, Filename=tmp_path) + + # Read MD5 from sidecar + metadata: dict[str, str] = {} + md5_key = staged_key + ".md5" + if md5_key in sidecars: + md5_obj = s3.get_object(Bucket=bucket, Key=md5_key) + metadata["md5"] = md5_obj["Body"].read().decode().strip() + + upload_file_with_metadata( + tmp_path, + f"{bucket}/{Path(final_key).parent}", + metadata=metadata, + object_name=Path(final_key).name, + ) + promoted += 1 + + # Track promoted accession for manifest trimming + acc_match = re.search(r"(GC[AF]_\d{9}\.\d+)", staged_key) + if acc_match: + promoted_accessions.add(acc_match.group(1)) + + finally: + Path(tmp_path).unlink() + except Exception: + logger.exception("Failed to promote %s", staged_key) + failed += 1 + + # Trim manifest for resumability + if manifest_path and promoted_accessions and not dry_run: + _trim_manifest(manifest_path, bucket, promoted_accessions) + + report: dict[str, Any] = { + "timestamp": datetime.now(UTC).isoformat(), + "promoted": promoted, + "archived": archived, + "failed": failed, + "dry_run": dry_run, + } + + logger.info( + "PROMOTE SUMMARY: %d promoted, %d archived, %d failed%s", + promoted, + archived, + failed, + " (dry-run)" if dry_run else "", + ) + return report + + +# ── Archive assemblies ────────────────────────────────────────────────── + + +def _archive_assemblies( # noqa: PLR0913 + manifest_path: str, + bucket: str, + ncbi_release: str | None = None, + path_prefix: str = DEFAULT_PATH_PREFIX, + archive_reason: str = "unknown", + *, + delete_source: bool = False, + dry_run: bool = False, +) -> int: + """Archive assembly objects to ``archive/{release_tag}/``. + + Copies S3 objects matching each accession to the archive prefix. + When *delete_source* is True (replaced/suppressed), the original + objects are deleted after copying. When False (updated), the + originals remain in place to be overwritten by the promote step. + + :param manifest_path: local path to a manifest file (one accession per line) + :param bucket: S3 bucket name + :param ncbi_release: release tag used in the archive path + :param path_prefix: Lakehouse path prefix + :param archive_reason: metadata value describing why the object was archived + :param delete_source: if True, delete the source object after copying + :param dry_run: if True, log without making changes + :return: number of objects archived + """ + s3 = get_s3_client() + release_tag = ncbi_release or "unknown" + datestamp = datetime.now(UTC).strftime("%Y-%m-%d") + archived = 0 + + with Path(manifest_path).open() as f: + accessions = [line.strip() for line in f if line.strip()] + + for accession in accessions: + m = re.match(r"(GC[AF])_(\d{3})(\d{3})(\d{3})\.\d+", accession) + if not m: + logger.warning("Cannot parse accession for archival: %s", accession) + continue + + db = m.group(1) + p1, p2, p3 = m.group(2), m.group(3), m.group(4) + source_prefix = f"{path_prefix}raw_data/{db}/{p1}/{p2}/{p3}/" + + paginator = s3.get_paginator("list_objects_v2") + matching_keys: list[str] = [] + for page in paginator.paginate(Bucket=bucket, Prefix=source_prefix): + matching_keys.extend(obj["Key"] for obj in page.get("Contents", []) if accession in obj["Key"]) + + if not matching_keys: + logger.debug("No objects found for %s, skipping archive", accession) + continue + + for source_key in matching_keys: + rel = source_key[len(path_prefix) :] + archive_key = f"{path_prefix}archive/{release_tag}/{rel}" + + if dry_run: + logger.info("[dry-run] would archive: %s -> %s", source_key, archive_key) + archived += 1 + continue + + try: + copy_object_with_metadata( + f"{bucket}/{source_key}", + f"{bucket}/{archive_key}", + metadata={ + "ncbi_last_release": release_tag, + "archive_reason": archive_reason, + "archive_date": datestamp, + }, + ) + if delete_source: + delete_object(f"{bucket}/{source_key}") + archived += 1 + logger.debug(" Archived: %s -> %s", source_key, archive_key) + except Exception: + logger.exception("Failed to archive %s", source_key) + + logger.info("Archived %d objects for %d accessions (%s)", archived, len(accessions), archive_reason) + return archived + + +# ── Manifest trimming ─────────────────────────────────────────────────── + + +def _trim_manifest(manifest_s3_path: str, bucket: str, promoted_accessions: set[str]) -> None: + """Remove promoted accessions from the transfer manifest in S3. + + :param manifest_s3_path: S3 key of the transfer_manifest.txt + :param bucket: S3 bucket name + :param promoted_accessions: set of accessions that were successfully promoted + """ + s3 = get_s3_client() + + with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".txt") as tmp: + tmp_path = tmp.name + + try: + s3.download_file(Bucket=bucket, Key=manifest_s3_path, Filename=tmp_path) + + with Path(tmp_path).open() as f: + lines = f.readlines() + + remaining = [line for line in lines if line.strip() and not any(acc in line for acc in promoted_accessions)] + + with Path(tmp_path).open("w") as f: + f.writelines(remaining) + + s3.upload_file(Filename=tmp_path, Bucket=bucket, Key=manifest_s3_path) + logger.info( + "Trimmed manifest: %d -> %d entries (%d promoted)", + len(lines), + len(remaining), + len(lines) - len(remaining), + ) + finally: + Path(tmp_path).unlink() diff --git a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py new file mode 100644 index 00000000..6508fc1f --- /dev/null +++ b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py @@ -0,0 +1,191 @@ +"""NCBI FTP assembly download pipeline (Phase 2). + +Orchestrates parallel downloading of NCBI assemblies listed in a transfer +manifest. Settings, batching, CLI entry point, and CTS integration live here; +domain-specific download logic is in :mod:`cdm_data_loaders.ncbi_ftp.assembly`. +""" + +import json +import threading +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import UTC, datetime +from ftplib import error_temp +from pathlib import Path +from typing import Any + +from pydantic import AliasChoices, Field, field_validator +from pydantic_settings import CliSuppress + +from cdm_data_loaders.ncbi_ftp.assembly import FTP_HOST, download_assembly_to_local +from cdm_data_loaders.pipelines.core import run_cli +from cdm_data_loaders.pipelines.cts_defaults import INPUT_MOUNT, OUTPUT_MOUNT, CtsDefaultSettings +from cdm_data_loaders.utils.cdm_logger import get_cdm_logger +from cdm_data_loaders.utils.ftp_client import ThreadLocalFTP + +logger = get_cdm_logger() + + +# ── Settings ───────────────────────────────────────────────────────────── + + +class DownloadSettings(CtsDefaultSettings): + """Configuration for the NCBI FTP assembly download pipeline.""" + + manifest: str = Field( + default=f"{INPUT_MOUNT}/transfer_manifest.txt", + description="Path to the transfer manifest file listing FTP paths to download", + validation_alias=AliasChoices("m", "manifest"), + ) + output_dir: str = Field( + default=OUTPUT_MOUNT, + description="Output directory for downloaded assembly files", + validation_alias=AliasChoices("o", "output-dir", "output_dir"), + ) + threads: int = Field( + default=4, + ge=1, + le=32, + description="Number of parallel download threads", + validation_alias=AliasChoices("t", "threads"), + ) + ftp_host: str = Field( + default=FTP_HOST, + description="NCBI FTP hostname", + validation_alias=AliasChoices("ftp-host", "ftp_host"), + ) + limit: CliSuppress[int | None] = Field( + default=None, + ge=1, + description="Limit to first N assemblies (for testing)", + validation_alias=AliasChoices("l", "limit"), + ) + + @field_validator("threads") + @classmethod + def validate_threads(cls, v: int) -> int: + """Validate threads is within range. + + :param v: number of threads + :raises ValueError: if out of range + :return: validated thread count + """ + if v < 1 or v > 32: # noqa: PLR2004 + msg = f"threads must be between 1 and 32, got {v}" + raise ValueError(msg) + return v + + +# ── Batch download ─────────────────────────────────────────────────────── + + +def download_batch( + manifest_path: str | Path, + output_dir: str | Path, + threads: int = 4, + ftp_host: str = FTP_HOST, + limit: int | None = None, +) -> dict[str, Any]: + """Download all assemblies listed in the manifest. + + :param manifest_path: path to the transfer manifest file + :param output_dir: base output directory + :param threads: number of parallel download threads + :param ftp_host: FTP hostname + :param limit: optional limit for testing + :return: report dict with overall stats + """ + with Path(manifest_path).open() as f: + assembly_paths = [line.strip() for line in f if line.strip() and not line.startswith("#")] + + if limit: + assembly_paths = assembly_paths[:limit] + + logger.info("Starting download of %d assemblies with %d threads", len(assembly_paths), threads) + + pool = ThreadLocalFTP(ftp_host) + lock = threading.Lock() + success_count = 0 + failed: list[dict[str, str]] = [] + all_stats: list[dict[str, Any]] = [] + + def _download_one(path: str) -> tuple[str, Exception | None]: + nonlocal success_count + last_error: Exception | None = None + for attempt in range(1, 4): + try: + stats = download_assembly_to_local(path, output_dir, ftp_host=ftp_host, ftp=pool.get()) + except error_temp as e: + last_error = e + if attempt < 3: # noqa: PLR2004 + logger.warning("Transient FTP error for %s, retry %d/3: %s", path, attempt, e) + time.sleep(5) + except Exception as e: # noqa: BLE001 + return path, e + else: + with lock: + success_count += 1 + all_stats.append(stats) + return path, None + return path, last_error + + try: + with ThreadPoolExecutor(max_workers=threads) as executor: + futures = {executor.submit(_download_one, p): p for p in assembly_paths} + for future in as_completed(futures): + path, error = future.result() + if error: + logger.error("FAILED: %s: %s", path, error) + with lock: + failed.append({"path": path, "error": str(error)}) + finally: + pool.close_all() + + report: dict[str, Any] = { + "timestamp": datetime.now(UTC).isoformat(), + "total_attempted": len(assembly_paths), + "succeeded": success_count, + "failed": len(failed), + "failures": failed, + "assembly_stats": all_stats, + } + + report_path = Path(output_dir) / "download_report.json" + report_path.parent.mkdir(parents=True, exist_ok=True) + with report_path.open("w") as f: + json.dump(report, f, indent=2) + logger.info("Download report written to: %s", report_path) + + logger.info( + "SUMMARY: %d attempted, %d succeeded, %d failed", + len(assembly_paths), + success_count, + len(failed), + ) + + return report + + +# ── CTS entry point ───────────────────────────────────────────────────── + + +def run_download(config: DownloadSettings) -> None: + """Main CTS entry point for Phase 2 download. + + :param config: validated download settings + """ + report = download_batch( + manifest_path=config.manifest, + output_dir=config.output_dir, + threads=config.threads, + ftp_host=config.ftp_host, + limit=config.limit, + ) + if report["failed"] > 0: + msg = f"Download completed with {report['failed']} failures" + raise RuntimeError(msg) + + +def cli() -> None: + """CLI entry point for ``ncbi_ftp_sync``.""" + run_cli(DownloadSettings, run_download) diff --git a/src/cdm_data_loaders/utils/checksums.py b/src/cdm_data_loaders/utils/checksums.py new file mode 100644 index 00000000..021098a6 --- /dev/null +++ b/src/cdm_data_loaders/utils/checksums.py @@ -0,0 +1,55 @@ +"""General-purpose file checksum utilities. + +Provides MD5 and CRC64/NVME checksum computation and verification for local +files. These are protocol-agnostic primitives used by download pipelines +and S3 metadata workflows. +""" + +import base64 +import hashlib +from pathlib import Path + +from cdm_data_loaders.utils.cdm_logger import get_cdm_logger + +logger = get_cdm_logger() + + +def compute_md5(file_path: str | Path) -> str: + """Compute the MD5 hex digest of a file. + + :param file_path: path to the file + :return: lowercase hex MD5 string + """ + md5_hash = hashlib.md5() # noqa: S324 + with Path(file_path).open("rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + md5_hash.update(chunk) + return md5_hash.hexdigest() + + +def verify_md5(file_path: str | Path, expected_md5: str) -> bool: + """Verify a file's MD5 checksum against an expected value. + + :param file_path: path to the file + :param expected_md5: expected lowercase hex MD5 string + :return: True if the checksum matches + """ + return compute_md5(file_path) == expected_md5 + + +def compute_crc64nvme(file_path: str | Path) -> str: + """Compute the CRC64/NVME checksum of a file. + + Returns the base64-encoded string matching the format used by S3-native + checksums (``ChecksumCRC64NVME``). + + :param file_path: path to the file + :return: base64-encoded CRC64/NVME checksum + """ + from awscrt.checksums import crc64nvme as _crc64nvme # noqa: PLC0415 + + crc = 0 + with Path(file_path).open("rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + crc = _crc64nvme(chunk, crc) + return base64.b64encode(crc.to_bytes(8, byteorder="big")).decode() diff --git a/src/cdm_data_loaders/utils/ftp_client.py b/src/cdm_data_loaders/utils/ftp_client.py new file mode 100644 index 00000000..0f8409e7 --- /dev/null +++ b/src/cdm_data_loaders/utils/ftp_client.py @@ -0,0 +1,162 @@ +"""General-purpose FTP client utilities. + +Provides resilient FTP connections with TCP keepalive, NOOP pings, retry +on transient errors, and thread-local connection management for parallel +downloads. Protocol-agnostic — callers supply the FTP hostname. +""" + +import contextlib +import socket +import threading +import time +from ftplib import FTP, error_temp +from pathlib import Path + +from cdm_data_loaders.utils.cdm_logger import get_cdm_logger + +logger = get_cdm_logger() + +DEFAULT_TIMEOUT = 60 + + +def connect_ftp(host: str, timeout: int = DEFAULT_TIMEOUT) -> FTP: + """Connect and log in to an FTP server with TCP keepalive enabled. + + :param host: FTP hostname + :param timeout: connection timeout in seconds + :return: logged-in FTP connection + """ + ftp = FTP(host, timeout=timeout) # noqa: S321 + ftp.login() + _set_keepalive(ftp) + return ftp + + +def _set_keepalive(ftp: FTP, idle: int = 30, interval: int = 10, count: int = 3) -> None: + """Enable TCP keepalive on the FTP control socket. + + Prevents idle-timeout disconnects (e.g. '421 No transfer timeout') when + the control connection sits idle during data transfers or checksum + verification. + """ + sock = ftp.sock + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + if hasattr(socket, "TCP_KEEPIDLE"): + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, idle) + if hasattr(socket, "TCP_KEEPINTVL"): + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, interval) + if hasattr(socket, "TCP_KEEPCNT"): + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, count) + + +def ftp_noop_keepalive(ftp: FTP, last_activity: float, interval: int = 25) -> float: + """Send NOOP if the connection has been idle longer than *interval* seconds. + + :param ftp: active FTP connection + :param last_activity: monotonic timestamp of last FTP activity + :param interval: seconds of idle time before sending NOOP + :return: updated last-activity timestamp + """ + if time.monotonic() - last_activity > interval: + with contextlib.suppress(Exception): + ftp.sendcmd("NOOP") + return time.monotonic() + return last_activity + + +def ftp_list_dir(ftp: FTP, path: str, retries: int = 3) -> list[str]: + """List files in an FTP directory with retry on transient errors. + + :param ftp: active FTP connection + :param path: remote directory path + :param retries: number of retry attempts + :return: list of filenames + """ + ftp.cwd(path) + for attempt in range(1, retries + 1): + try: + files: list[str] = [] + ftp.retrlines("NLST", files.append) + except error_temp as e: + if attempt < retries: + logger.warning("Transient FTP error listing %s (attempt %d/%d): %s", path, attempt, retries, e) + time.sleep(2) + else: + raise + else: + return files + return [] # unreachable, but keeps type checkers happy + + +def ftp_download_file(ftp: FTP, remote_path: str, local_path: str, retries: int = 3) -> None: + """Download a single file from FTP with retry on transient errors. + + :param ftp: active FTP connection + :param remote_path: full remote file path + :param local_path: local destination path + :param retries: number of retry attempts + """ + for attempt in range(1, retries + 1): + try: + with Path(local_path).open("wb") as f: + ftp.retrbinary(f"RETR {remote_path}", f.write) + except error_temp as e: + if attempt < retries: + logger.warning( + "Transient FTP error downloading %s (attempt %d/%d): %s", remote_path, attempt, retries, e + ) + time.sleep(2) + else: + raise + else: + return + + +def ftp_retrieve_text(ftp: FTP, remote_path: str) -> str: + """Retrieve a text file from FTP, returning its content as a string. + + :param ftp: active FTP connection + :param remote_path: full remote file path + :return: file content + """ + lines: list[str] = [] + ftp.retrlines(f"RETR {remote_path}", lines.append) + return "\n".join(lines) + + +class ThreadLocalFTP: + """Manage thread-local FTP connections for parallel downloads. + + Each thread gets its own FTP connection, created on first access. + Call :meth:`close_all` when done to cleanly shut down all connections. + """ + + def __init__(self, host: str, timeout: int = DEFAULT_TIMEOUT) -> None: + """Initialise with FTP host and timeout. + + :param host: FTP hostname (required — no default) + :param timeout: connection timeout in seconds + """ + self._host = host + self._timeout = timeout + self._local = threading.local() + self._lock = threading.Lock() + self._connections: list[FTP] = [] + + def get(self) -> FTP: + """Return the FTP connection for the current thread, creating one if needed.""" + ftp = getattr(self._local, "ftp", None) + if ftp is None: + ftp = connect_ftp(self._host, self._timeout) + self._local.ftp = ftp + with self._lock: + self._connections.append(ftp) + return ftp + + def close_all(self) -> None: + """Close all thread-local FTP connections.""" + with self._lock: + for ftp in self._connections: + with contextlib.suppress(Exception): + ftp.quit() + self._connections.clear() diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index 8253e2cd..6fc1696a 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -368,3 +368,117 @@ def delete_object(s3_path: str) -> dict[str, Any]: s3 = get_s3_client() (bucket, key) = split_s3_path(s3_path) return s3.delete_object(Bucket=bucket, Key=key) + + +def upload_file_with_metadata( + local_file_path: Path | str, + destination_dir: str, + metadata: dict[str, str], + object_name: str | None = None, +) -> bool: + """Upload a file to S3 with user-defined metadata and CRC64NVME checksum. + + Unlike :func:`upload_file`, this function always uploads (no existence check) + and attaches the supplied *metadata* dict as S3 user metadata. + + :param local_file_path: file to upload + :type local_file_path: Path | str + :param destination_dir: path to the destination directory on s3, INCLUDING the bucket name + :type destination_dir: str + :param metadata: user metadata key/value pairs to attach to the object + :type metadata: dict[str, str] + :param object_name: S3 object name; defaults to the local filename + :type object_name: str | None + :return: True if the upload succeeded + :rtype: bool + """ + if isinstance(local_file_path, str): + local_file_path = Path(local_file_path) + + if not destination_dir: + msg = "No destination directory supplied for the file" + raise ValueError(msg) + + if not object_name: + object_name = local_file_path.name + + s3_path = f"{destination_dir.removesuffix('/')}/{object_name}" + s3 = get_s3_client() + (bucket, key) = split_s3_path(s3_path) + + extra_args = {**DEFAULT_EXTRA_ARGS, "Metadata": metadata} + + file_size = local_file_path.stat().st_size + with tqdm.tqdm(total=file_size, unit="B", unit_scale=True, desc=str(local_file_path)) as pbar: + s3.upload_file( + Filename=str(local_file_path), + Bucket=bucket, + Key=key, + Callback=pbar.update, + ExtraArgs=extra_args, + ) + return True + + +def head_object(s3_path: str) -> dict[str, Any] | None: + """Return metadata for an S3 object, or None if it does not exist. + + The returned dict contains: + - ``size``: content length in bytes + - ``metadata``: user metadata dict + - ``checksum_crc64nvme``: CRC64NVME checksum string (if available) + + :param s3_path: path to the object on s3, INCLUDING the bucket name + :type s3_path: str + :return: dict with object info, or None if the object does not exist + :rtype: dict[str, Any] | None + """ + s3 = get_s3_client() + (bucket, key) = split_s3_path(s3_path) + try: + resp = s3.head_object(Bucket=bucket, Key=key, ChecksumMode="ENABLED") + except botocore.exceptions.ClientError as e: + if e.response["Error"]["Code"] == "404": + return None + raise + return { + "size": resp["ContentLength"], + "metadata": resp.get("Metadata", {}), + "checksum_crc64nvme": resp.get("ChecksumCRC64NVME"), + } + + +def copy_object_with_metadata( + current_s3_path: str, + new_s3_path: str, + metadata: dict[str, str], +) -> dict[str, Any]: + """Copy an S3 object to a new location, replacing its user metadata. + + Uses ``MetadataDirective='REPLACE'`` so the destination object carries + exactly the supplied *metadata* rather than inheriting the source's metadata. + + A successful copy returns a response where + ``resp["ResponseMetadata"]["HTTPStatusCode"] == 200``. + + :param current_s3_path: source path on s3, INCLUDING the bucket name + :type current_s3_path: str + :param new_s3_path: destination path on s3, INCLUDING the bucket name + :type new_s3_path: str + :param metadata: user metadata to set on the destination object + :type metadata: dict[str, str] + :return: dictionary containing response + :rtype: dict[str, Any] + """ + s3 = get_s3_client() + (current_bucket, current_key) = split_s3_path(current_s3_path) + (new_bucket, new_key) = split_s3_path(new_s3_path) + + return s3.copy_object( + CopySource={"Bucket": current_bucket, "Key": current_key}, + Bucket=new_bucket, + Key=new_key, + Metadata=metadata, + MetadataDirective="REPLACE", + **DEFAULT_EXTRA_ARGS, + ) diff --git a/tests/ncbi_ftp/__init__.py b/tests/ncbi_ftp/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/ncbi_ftp/conftest.py b/tests/ncbi_ftp/conftest.py new file mode 100644 index 00000000..2c0923dd --- /dev/null +++ b/tests/ncbi_ftp/conftest.py @@ -0,0 +1,80 @@ +"""Shared fixtures for ncbi_ftp tests.""" + +import functools +from collections.abc import Callable, Generator +from unittest.mock import patch + +import boto3 +import botocore.client +import pytest +from moto import mock_aws + +import cdm_data_loaders.ncbi_ftp.promote as promote_mod +import cdm_data_loaders.utils.s3 as s3_utils +from cdm_data_loaders.utils.s3 import CDM_LAKE_BUCKET, reset_s3_client + +AWS_REGION = "us-east-1" +TEST_BUCKET = CDM_LAKE_BUCKET + + +# Minimal assembly_summary_refseq.txt content (tab-separated, 20+ columns) +SAMPLE_SUMMARY = ( + "# assembly_accession\tbioproject\tbiosample\twgs_master\trefseq_category\t" + "taxid\tspecies_taxid\torganism_name\tinfraspecific_name\tisolate\t" + "version_status\tassembly_level\trelease_type\tgenome_rep\tseq_rel_date\t" + "asm_name\t16\t17\t18\tftp_path\n" + "GCF_000001215.4\tPRJNA13812\tSAMN02803731\t\treference genome\t7227\t7227\t" + "Drosophila melanogaster\t\t\tlatest\tChromosome\tMajor\tFull\t2014/10/21\t" + "Release_6_plus_ISO1_MT\t\t\t\t" + "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT\n" + "GCF_000001405.40\tPRJNA168\tna\t\treference genome\t9606\t9606\t" + "Homo sapiens\t\t\tlatest\tChromosome\tPatch\tFull\t2022/02/03\t" + "GRCh38.p14\t\t\t\t" + "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/405/GCF_000001405.40_GRCh38.p14\n" + "GCF_000005845.2\tPRJNA57779\tSAMN02604091\t\trepresentative genome\t511145\t562\t" + "Escherichia coli\t\t\treplaced\tComplete Genome\tMajor\tFull\t2013/09/26\t" + "ASM584v2\t\t\t\t" + "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/005/845/GCF_000005845.2_ASM584v2\n" + "GCF_000009999.1\tPRJNA999\tSAMN999\t\tna\t0\t0\t" + "Test organism\t\t\tsuppressed\tScaffold\tMajor\tFull\t2010/01/01\t" + "ASM999v1\t\t\t\t" + "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/009/999/GCF_000009999.1_ASM999v1\n" + "GCF_000099999.1\tPRJNA888\tSAMN888\t\tna\t0\t0\t" + "Test organism 2\t\t\tlatest\tContig\tMajor\tFull\t2023/06/15\t" + "ASM9999v1\t\t\t\tna\n" +) + + +def strip_checksum_algorithm(method: Callable) -> Callable: + """Wrap a boto3 S3 method to remove ChecksumAlgorithm (moto CRC64NVME workaround).""" + + @functools.wraps(method) + def wrapper(*args: object, **kwargs: object) -> object: + kwargs.pop("ChecksumAlgorithm", None) # type: ignore[arg-type] + return method(*args, **kwargs) + + return wrapper + + +@pytest.fixture +def mock_s3_client() -> Generator[botocore.client.BaseClient]: + """Yield a mocked S3 client with the CDM Lake bucket created.""" + with mock_aws(): + client = boto3.client("s3", region_name=AWS_REGION) + client.create_bucket(Bucket=TEST_BUCKET) + + reset_s3_client() + with ( + patch.object(s3_utils, "get_s3_client", return_value=client), + patch.object(promote_mod, "get_s3_client", return_value=client), + ): + yield client + reset_s3_client() + + +@pytest.fixture +def mock_s3_client_no_checksum(mock_s3_client: botocore.client.BaseClient) -> botocore.client.BaseClient: + """Mocked S3 client with copy_object and upload_file patched to strip ChecksumAlgorithm.""" + mock_s3_client.copy_object = strip_checksum_algorithm(mock_s3_client.copy_object) # type: ignore[method-assign] + mock_s3_client.upload_file = strip_checksum_algorithm(mock_s3_client.upload_file) # type: ignore[method-assign] + return mock_s3_client diff --git a/tests/ncbi_ftp/test_assembly.py b/tests/ncbi_ftp/test_assembly.py new file mode 100644 index 00000000..8aa24b1c --- /dev/null +++ b/tests/ncbi_ftp/test_assembly.py @@ -0,0 +1,114 @@ +"""Tests for ncbi_ftp.assembly module — path helpers, file filtering, checksum parsing.""" + +import pytest + +from cdm_data_loaders.ncbi_ftp.assembly import ( + FILE_FILTERS, + build_accession_path, + parse_assembly_path, + parse_md5_checksums_file, +) + +_EXPECTED_TWO_ENTRIES = 2 + + +# ── Path helpers ───────────────────────────────────────────────────────── + + +class TestBuildAccessionPath: + """Test output directory path construction from assembly names.""" + + def test_basic(self) -> None: + """Verify standard GCF accession path construction.""" + result = build_accession_path("GCF_000001215.4_Release_6_plus_ISO1_MT") + assert result == "raw_data/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/" + + def test_gca_prefix(self) -> None: + """Verify GCA prefix path construction.""" + result = build_accession_path("GCA_012345678.1_ASM1234v1") + assert result == "raw_data/GCA/012/345/678/GCA_012345678.1_ASM1234v1/" + + def test_invalid_raises(self) -> None: + """Verify ValueError on invalid assembly name.""" + with pytest.raises(ValueError, match="Cannot parse"): + build_accession_path("invalid_name") + + +class TestParseAssemblyPath: + """Test FTP path parsing.""" + + def test_basic(self) -> None: + """Verify db, assembly_dir, and accession are parsed correctly.""" + path = "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/" + _db, _assembly_dir, accession = parse_assembly_path(path) + assert _db == "GCF" + assert _assembly_dir == "GCF_000001215.4_Release_6_plus_ISO1_MT" + assert accession == "GCF_000001215.4" + + def test_without_trailing_slash(self) -> None: + """Verify parsing works without trailing slash.""" + path = "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT" + _db, _assembly_dir, accession = parse_assembly_path(path) + assert accession == "GCF_000001215.4" + + def test_invalid_raises(self) -> None: + """Verify ValueError on invalid path.""" + with pytest.raises(ValueError, match="Cannot parse"): + parse_assembly_path("/random/path/") + + +# ── FILE_FILTERS sanity ───────────────────────────────────────────────── + + +class TestFileFilters: + """Sanity checks for the file suffix filter list.""" + + def test_not_empty(self) -> None: + """Verify FILE_FILTERS is not empty.""" + assert len(FILE_FILTERS) > 0 + + def test_all_start_with_underscore(self) -> None: + """Verify all filter patterns start with an underscore.""" + for f in FILE_FILTERS: + assert f.startswith("_"), f"Filter should start with underscore: {f}" + + def test_genomic_fna_included(self) -> None: + """Verify _genomic.fna.gz is in the filter list.""" + assert "_genomic.fna.gz" in FILE_FILTERS + + def test_assembly_report_included(self) -> None: + """Verify _assembly_report.txt is in the filter list.""" + assert "_assembly_report.txt" in FILE_FILTERS + + +# ── parse_md5_checksums_file ───────────────────────────────────────────── + + +class TestParseMd5ChecksumsFile: + """Test NCBI md5checksums.txt parsing.""" + + def test_basic(self) -> None: + """Verify parsing of standard md5checksums.txt format.""" + text = "abc123 ./GCF_000001215.4_genomic.fna.gz\ndef456 ./GCF_000001215.4_genomic.gff.gz\n" + result = parse_md5_checksums_file(text) + assert result == { + "GCF_000001215.4_genomic.fna.gz": "abc123", + "GCF_000001215.4_genomic.gff.gz": "def456", + } + + def test_no_leading_dot_slash(self) -> None: + """Verify parsing works without leading ./ prefix.""" + text = "abc123 GCF_000001215.4_genomic.fna.gz\n" + result = parse_md5_checksums_file(text) + assert result == {"GCF_000001215.4_genomic.fna.gz": "abc123"} + + def test_empty(self) -> None: + """Verify empty or whitespace-only input returns empty dict.""" + assert parse_md5_checksums_file("") == {} + assert parse_md5_checksums_file(" \n \n") == {} + + def test_blank_lines_ignored(self) -> None: + """Verify blank lines between entries are skipped.""" + text = "abc123 file1.txt\n\n\ndef456 file2.txt\n" + result = parse_md5_checksums_file(text) + assert len(result) == _EXPECTED_TWO_ENTRIES diff --git a/tests/ncbi_ftp/test_manifest.py b/tests/ncbi_ftp/test_manifest.py new file mode 100644 index 00000000..61db7e5a --- /dev/null +++ b/tests/ncbi_ftp/test_manifest.py @@ -0,0 +1,516 @@ +"""Tests for ncbi_ftp.manifest module — assembly summary parsing, diff, filtering, writing.""" + +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +from cdm_data_loaders.ncbi_ftp.manifest import ( + DiffResult, + _ftp_dir_from_url, + accession_prefix, + compute_diff, + filter_by_prefix_range, + get_latest_assembly_paths, + parse_assembly_summary, + verify_transfer_candidates, + write_diff_summary, + write_removed_manifest, + write_transfer_manifest, + write_updated_manifest, +) + +from .conftest import SAMPLE_SUMMARY + +_EXPECTED_ENTRIES = 4 +_EXPECTED_TWO = 2 +_EXPECTED_TOTAL_TRANSFER = 2 + + +# ── parse_assembly_summary ─────────────────────────────────────────────── + + +class TestParseAssemblySummary: + """Test assembly summary parsing.""" + + def test_parse_basic(self) -> None: + """Verify basic parsing returns expected number of assemblies.""" + assemblies = parse_assembly_summary(SAMPLE_SUMMARY) + assert len(assemblies) == _EXPECTED_ENTRIES + assert "GCF_000001215.4" in assemblies + assert "GCF_000005845.2" in assemblies + assert "GCF_000099999.1" not in assemblies # ftp_path == "na" + + def test_parse_status(self) -> None: + """Verify status field is parsed correctly.""" + assemblies = parse_assembly_summary(SAMPLE_SUMMARY) + assert assemblies["GCF_000001215.4"].status == "latest" + assert assemblies["GCF_000005845.2"].status == "replaced" + assert assemblies["GCF_000009999.1"].status == "suppressed" + + def test_parse_seq_rel_date(self) -> None: + """Verify seq_rel_date field is parsed correctly.""" + assemblies = parse_assembly_summary(SAMPLE_SUMMARY) + assert assemblies["GCF_000001215.4"].seq_rel_date == "2014/10/21" + + def test_parse_assembly_dir(self) -> None: + """Verify assembly_dir is extracted from the FTP path.""" + assemblies = parse_assembly_summary(SAMPLE_SUMMARY) + assert assemblies["GCF_000001215.4"].assembly_dir == "GCF_000001215.4_Release_6_plus_ISO1_MT" + + def test_parse_ftp_path(self) -> None: + """Verify full FTP path is stored.""" + assemblies = parse_assembly_summary(SAMPLE_SUMMARY) + assert assemblies["GCF_000001215.4"].ftp_path == ( + "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT" + ) + + def test_parse_empty(self) -> None: + """Verify empty or comment-only input returns empty dict.""" + assemblies = parse_assembly_summary("# comment only\n") + assert len(assemblies) == 0 + + def test_parse_skips_comments(self) -> None: + """Verify comment lines are not included in results.""" + assemblies = parse_assembly_summary(SAMPLE_SUMMARY) + for acc in assemblies: + assert acc.startswith("GCF_") + + def test_parse_from_file(self, tmp_path: Path) -> None: + """Verify parsing from a file path object.""" + f = tmp_path / "summary.tsv" + f.write_text(SAMPLE_SUMMARY) + assemblies = parse_assembly_summary(f) + assert len(assemblies) == _EXPECTED_ENTRIES + + def test_parse_from_file_str(self, tmp_path: Path) -> None: + """Verify parsing from a string file path.""" + f = tmp_path / "summary.tsv" + f.write_text(SAMPLE_SUMMARY) + assemblies = parse_assembly_summary(str(f)) + assert len(assemblies) == _EXPECTED_ENTRIES + + def test_parse_from_list_of_lines(self) -> None: + """Verify parsing from a list of lines.""" + lines = SAMPLE_SUMMARY.splitlines(keepends=True) + assemblies = parse_assembly_summary(lines) + assert len(assemblies) == _EXPECTED_ENTRIES + + +# ── get_latest_assembly_paths ──────────────────────────────────────────── + + +class TestGetLatestAssemblyPaths: + """Test extraction of FTP paths for latest assemblies.""" + + def test_only_latest(self) -> None: + """Verify only assemblies with status 'latest' are returned.""" + assemblies = parse_assembly_summary(SAMPLE_SUMMARY) + paths = get_latest_assembly_paths(assemblies) + accessions = [acc for acc, _ in paths] + assert "GCF_000001215.4" in accessions + assert "GCF_000001405.40" in accessions + assert "GCF_000005845.2" not in accessions # replaced + assert "GCF_000009999.1" not in accessions # suppressed + + def test_path_conversion(self) -> None: + """Verify HTTPS paths are converted to FTP-relative paths.""" + assemblies = parse_assembly_summary(SAMPLE_SUMMARY) + paths = dict(get_latest_assembly_paths(assemblies)) + assert paths["GCF_000001215.4"] == "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/" + + def test_paths_end_with_slash(self) -> None: + """Verify all returned paths end with a trailing slash.""" + assemblies = parse_assembly_summary(SAMPLE_SUMMARY) + for _, path in get_latest_assembly_paths(assemblies): + assert path.endswith("/") + + def test_empty(self) -> None: + """Verify empty input returns empty list.""" + assemblies = parse_assembly_summary("# empty\n") + assert get_latest_assembly_paths(assemblies) == [] + + +# ── compute_diff ───────────────────────────────────────────────────────── + + +class TestComputeDiff: + """Test diff computation between current and previous assembly state.""" + + def test_all_new_no_previous(self) -> None: + """Verify all latest assemblies are marked new when no previous state.""" + current = parse_assembly_summary(SAMPLE_SUMMARY) + diff = compute_diff(current, previous_accessions=set()) + assert "GCF_000001215.4" in diff.new + assert "GCF_000001405.40" in diff.new + assert "GCF_000005845.2" not in diff.new # replaced + + def test_nothing_new_when_all_known(self) -> None: + """Verify no new assemblies when all are already known.""" + current = parse_assembly_summary(SAMPLE_SUMMARY) + known = {"GCF_000001215.4", "GCF_000001405.40"} + diff = compute_diff(current, previous_accessions=known) + assert len(diff.new) == 0 + + def test_detects_updated_seq_rel_date(self) -> None: + """Verify assemblies with changed seq_rel_date are marked updated.""" + current = parse_assembly_summary(SAMPLE_SUMMARY) + previous = parse_assembly_summary(SAMPLE_SUMMARY) + previous["GCF_000001215.4"].seq_rel_date = "2010/01/01" + diff = compute_diff(current, previous_assemblies=previous) + assert "GCF_000001215.4" in diff.updated + + def test_detects_replaced(self) -> None: + """Verify assemblies with status 'replaced' are detected.""" + current = parse_assembly_summary(SAMPLE_SUMMARY) + diff = compute_diff(current, previous_accessions={"GCF_000005845.2"}) + assert "GCF_000005845.2" in diff.replaced + + def test_detects_suppressed(self) -> None: + """Verify assemblies with status 'suppressed' are detected.""" + current = parse_assembly_summary(SAMPLE_SUMMARY) + diff = compute_diff(current, previous_accessions={"GCF_000009999.1"}) + assert "GCF_000009999.1" in diff.suppressed + + def test_detects_withdrawn(self) -> None: + """Accessions in previous but entirely absent from current.""" + current = parse_assembly_summary("# empty\n") + diff = compute_diff(current, previous_accessions={"GCF_000001215.4"}) + assert "GCF_000001215.4" in diff.suppressed + + def test_scan_store_fallback(self) -> None: + """Verify known accessions are not marked as new.""" + current = parse_assembly_summary(SAMPLE_SUMMARY) + diff = compute_diff(current, previous_accessions={"GCF_000001215.4"}) + assert "GCF_000001215.4" not in diff.new + assert "GCF_000001405.40" in diff.new + + def test_results_are_sorted(self) -> None: + """Verify diff results are sorted alphabetically.""" + current = parse_assembly_summary(SAMPLE_SUMMARY) + diff = compute_diff(current, previous_accessions=set()) + assert diff.new == sorted(diff.new) + + +# ── accession_prefix & filter_by_prefix_range ──────────────────────────── + + +class TestPrefixFiltering: + """Test prefix extraction and range filtering.""" + + def test_accession_prefix(self) -> None: + """Verify 3-digit prefix extraction from accessions.""" + assert accession_prefix("GCF_000001215.4") == "000" + assert accession_prefix("GCF_123456789.1") == "123" + assert accession_prefix("invalid") is None + + def test_filter_range_inclusive(self) -> None: + """Verify prefix range filter is inclusive.""" + assemblies = parse_assembly_summary(SAMPLE_SUMMARY) + filtered = filter_by_prefix_range(assemblies, "000", "000") + assert len(filtered) == len(assemblies) + + def test_filter_excludes_out_of_range(self) -> None: + """Verify assemblies outside the prefix range are excluded.""" + assemblies = parse_assembly_summary(SAMPLE_SUMMARY) + filtered = filter_by_prefix_range(assemblies, "001", "999") + assert len(filtered) == 0 + + def test_no_filter_returns_all(self) -> None: + """Verify no prefix range returns all assemblies.""" + assemblies = parse_assembly_summary(SAMPLE_SUMMARY) + filtered = filter_by_prefix_range(assemblies) + assert len(filtered) == len(assemblies) + + +# ── Manifest writing ──────────────────────────────────────────────────── + + +class TestManifestWriting: + """Test manifest file writing.""" + + def test_write_transfer_manifest(self, tmp_path: Path) -> None: + """Verify transfer manifest file is written correctly.""" + current = parse_assembly_summary(SAMPLE_SUMMARY) + diff = compute_diff(current, previous_accessions=set()) + manifest_file = tmp_path / "transfer.txt" + paths = write_transfer_manifest(diff, current, manifest_file) + assert len(paths) > 0 + lines = [line.strip() for line in manifest_file.read_text().splitlines() if line.strip()] + assert len(lines) == len(paths) + for line in lines: + assert line.startswith("/genomes/") + assert line.endswith("/") + + def test_write_removed_manifest(self, tmp_path: Path) -> None: + """Verify removed manifest lists replaced and suppressed accessions.""" + current = parse_assembly_summary(SAMPLE_SUMMARY) + diff = compute_diff(current, previous_accessions={"GCF_000005845.2", "GCF_000009999.1"}) + removed_file = tmp_path / "removed.txt" + removed = write_removed_manifest(diff, removed_file) + assert len(removed) == _EXPECTED_TWO + lines = [line.strip() for line in removed_file.read_text().splitlines() if line.strip()] + assert len(lines) == _EXPECTED_TWO + + def test_write_updated_manifest(self, tmp_path: Path) -> None: + """Verify updated manifest lists only updated accessions.""" + diff = DiffResult(new=["GCF_000001215.4"], updated=["GCF_000005845.2", "GCF_000001405.40"]) + updated_file = tmp_path / "updated.txt" + updated = write_updated_manifest(diff, updated_file) + assert len(updated) == _EXPECTED_TWO + lines = [line.strip() for line in updated_file.read_text().splitlines() if line.strip()] + assert len(lines) == _EXPECTED_TWO + # Should be sorted + assert lines[0] == "GCF_000001405.40" + assert lines[1] == "GCF_000005845.2" + + def test_write_diff_summary(self, tmp_path: Path) -> None: + """Verify diff summary JSON is written with correct counts.""" + diff = DiffResult(new=["a"], updated=["b"], replaced=["c"], suppressed=[]) + summary_file = tmp_path / "summary.json" + summary = write_diff_summary(diff, summary_file, "refseq", "000", "003") + assert summary["counts"]["new"] == 1 + assert summary["counts"]["total_to_transfer"] == _EXPECTED_TOTAL_TRANSFER + assert summary["prefix_range"]["from"] == "000" + + loaded = json.loads(summary_file.read_text()) + assert loaded["database"] == "refseq" + + +# ── _ftp_dir_from_url ─────────────────────────────────────────────────── + + +class TestFtpDirFromUrl: + """Test FTP URL to directory path conversion.""" + + def test_https_url(self) -> None: + """Verify https:// URLs are converted to FTP paths.""" + url = "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6" + assert _ftp_dir_from_url(url) == "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6" + + def test_ftp_url(self) -> None: + """Verify ftp:// URLs are converted to FTP paths.""" + url = "ftp://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6" + assert _ftp_dir_from_url(url) == "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6" + + def test_bare_path(self) -> None: + """Verify bare paths are returned unchanged.""" + path = "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6" + assert _ftp_dir_from_url(path) == path + + def test_custom_ftp_host(self) -> None: + """Verify custom FTP host is stripped from ftp:// URLs.""" + url = "ftp://custom.host.example.com/genomes/all/GCF/000/001/215" + assert _ftp_dir_from_url(url, ftp_host="custom.host.example.com") == "/genomes/all/GCF/000/001/215" + + +# ── verify_transfer_candidates ─────────────────────────────────────────── + + +_MD5_CHECKSUMS_TXT = ( + "d41d8cd98f00b204e9800998ecf8427e ./GCF_000001215.4_Release_6_plus_ISO1_MT_genomic.fna.gz\n" + "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4 ./GCF_000001215.4_Release_6_plus_ISO1_MT_protein.faa.gz\n" + "ffffffffffffffffffffffffffffffff ./GCF_000001215.4_Release_6_plus_ISO1_MT_assembly_report.txt\n" + "0000000000000000000000000000dead ./GCF_000001215.4_Release_6_plus_ISO1_MT_README.txt\n" +) + + +class TestVerifyTransferCandidates: + """Test S3 checksum verification to prune transfer candidates.""" + + def _assemblies(self) -> dict: + return parse_assembly_summary(SAMPLE_SUMMARY) + + @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") + @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) + @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") + def test_prunes_when_all_match( + self, + mock_connect: MagicMock, + mock_retrieve: MagicMock, + mock_head: MagicMock, + ) -> None: + """Assemblies where every file matches S3 are pruned from the list.""" + mock_connect.return_value = MagicMock() + + def head_side_effect(s3_path: str) -> dict | None: + if "_genomic.fna.gz" in s3_path: + return { + "size": 100, + "metadata": {"md5": "d41d8cd98f00b204e9800998ecf8427e"}, + "checksum_crc64nvme": None, + } + if "_protein.faa.gz" in s3_path: + return { + "size": 100, + "metadata": {"md5": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4"}, + "checksum_crc64nvme": None, + } + if "_assembly_report.txt" in s3_path: + return { + "size": 100, + "metadata": {"md5": "ffffffffffffffffffffffffffffffff"}, + "checksum_crc64nvme": None, + } + return None + + mock_head.side_effect = head_side_effect + result = verify_transfer_candidates( + ["GCF_000001215.4"], + self._assemblies(), + "cdm-lake", + "tenant-general-warehouse/kbase/datasets/ncbi/", + ) + assert result == [] + + @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") + @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) + @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") + def test_keeps_when_md5_differs( + self, + mock_connect: MagicMock, + mock_retrieve: MagicMock, + mock_head: MagicMock, + ) -> None: + """Assembly is kept when at least one file has a different MD5.""" + mock_connect.return_value = MagicMock() + mock_head.return_value = {"size": 100, "metadata": {"md5": "WRONG"}, "checksum_crc64nvme": None} + + result = verify_transfer_candidates( + ["GCF_000001215.4"], + self._assemblies(), + "cdm-lake", + "tenant-general-warehouse/kbase/datasets/ncbi/", + ) + assert result == ["GCF_000001215.4"] + + @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object", return_value=None) + @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) + @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") + def test_keeps_when_s3_object_missing( + self, + mock_connect: MagicMock, + mock_retrieve: MagicMock, + mock_head: MagicMock, + ) -> None: + """Assembly is kept when at least one file doesn't exist in S3.""" + mock_connect.return_value = MagicMock() + + result = verify_transfer_candidates( + ["GCF_000001215.4"], + self._assemblies(), + "cdm-lake", + "tenant-general-warehouse/kbase/datasets/ncbi/", + ) + assert result == ["GCF_000001215.4"] + + @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") + @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) + @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") + def test_keeps_when_s3_has_no_md5_metadata( + self, + mock_connect: MagicMock, + mock_retrieve: MagicMock, + mock_head: MagicMock, + ) -> None: + """Assembly is kept when S3 object exists but has no md5 metadata.""" + mock_connect.return_value = MagicMock() + mock_head.return_value = {"size": 100, "metadata": {}, "checksum_crc64nvme": None} + + result = verify_transfer_candidates( + ["GCF_000001215.4"], + self._assemblies(), + "cdm-lake", + "tenant-general-warehouse/kbase/datasets/ncbi/", + ) + assert result == ["GCF_000001215.4"] + + @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", side_effect=Exception("FTP error")) + @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") + def test_keeps_when_ftp_fails(self, mock_connect: MagicMock, mock_retrieve: MagicMock) -> None: + """Assembly is kept (conservative) when md5checksums.txt cannot be fetched.""" + mock_connect.return_value = MagicMock() + + result = verify_transfer_candidates( + ["GCF_000001215.4"], + self._assemblies(), + "cdm-lake", + "tenant-general-warehouse/kbase/datasets/ncbi/", + ) + assert result == ["GCF_000001215.4"] + + @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") + def test_empty_input(self, mock_connect: MagicMock) -> None: + """Empty accession list returns empty result without connecting.""" + result = verify_transfer_candidates([], {}, "cdm-lake", "prefix/") + assert result == [] + + @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") + def test_unknown_accession_kept(self, mock_connect: MagicMock) -> None: + """Accessions not in assemblies dict are kept (conservative).""" + mock_connect.return_value = MagicMock() + result = verify_transfer_candidates(["GCF_999999999.1"], {}, "cdm-lake", "prefix/") + assert result == ["GCF_999999999.1"] + + @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object", return_value=None) + @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) + @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") + def test_short_circuits_on_first_mismatch( + self, + mock_connect: MagicMock, + mock_retrieve: MagicMock, + mock_head: MagicMock, + ) -> None: + """Verification stops checking after the first missing/mismatched file.""" + mock_connect.return_value = MagicMock() + + verify_transfer_candidates( + ["GCF_000001215.4"], + self._assemblies(), + "cdm-lake", + "tenant-general-warehouse/kbase/datasets/ncbi/", + ) + assert mock_head.call_count == 1 + + @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") + @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) + @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") + def test_mixed_candidates( + self, + mock_connect: MagicMock, + mock_retrieve: MagicMock, + mock_head: MagicMock, + ) -> None: + """Verify a mix of matching and non-matching assemblies.""" + mock_connect.return_value = MagicMock() + + def head_side_effect(s3_path: str) -> dict | None: + # GCF_000001215.4 assembly dir → all match; GCF_000001405.40 → missing + if "GCF_000001215.4_Release_6_plus_ISO1_MT/" in s3_path: + if "_genomic.fna.gz" in s3_path: + return { + "size": 1, + "metadata": {"md5": "d41d8cd98f00b204e9800998ecf8427e"}, + "checksum_crc64nvme": None, + } + if "_protein.faa.gz" in s3_path: + return { + "size": 1, + "metadata": {"md5": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4"}, + "checksum_crc64nvme": None, + } + if "_assembly_report.txt" in s3_path: + return { + "size": 1, + "metadata": {"md5": "ffffffffffffffffffffffffffffffff"}, + "checksum_crc64nvme": None, + } + return None + + mock_head.side_effect = head_side_effect + result = verify_transfer_candidates( + ["GCF_000001215.4", "GCF_000001405.40"], + self._assemblies(), + "cdm-lake", + "tenant-general-warehouse/kbase/datasets/ncbi/", + ) + assert result == ["GCF_000001405.40"] diff --git a/tests/ncbi_ftp/test_notebooks.py b/tests/ncbi_ftp/test_notebooks.py new file mode 100644 index 00000000..6935c46d --- /dev/null +++ b/tests/ncbi_ftp/test_notebooks.py @@ -0,0 +1,89 @@ +"""Smoke tests for NCBI FTP notebooks — syntax and import validation.""" + +import ast +import json +from pathlib import Path + +import pytest + +from cdm_data_loaders.ncbi_ftp.assembly import FTP_HOST # noqa: F401 +from cdm_data_loaders.ncbi_ftp.manifest import ( # noqa: F401 + AssemblyRecord, + compute_diff, + download_assembly_summary, + filter_by_prefix_range, + parse_assembly_summary, + write_diff_summary, + write_removed_manifest, + write_transfer_manifest, + write_updated_manifest, +) +from cdm_data_loaders.ncbi_ftp.promote import ( + DEFAULT_PATH_PREFIX, + promote_from_s3, +) +from cdm_data_loaders.utils.s3 import get_s3_client, split_s3_path # noqa: F401 + +NOTEBOOKS_DIR = Path(__file__).resolve().parents[2] / "notebooks" + +NCBI_NOTEBOOKS = [ + "ncbi_ftp_manifest.ipynb", + "ncbi_ftp_promote.ipynb", +] + + +def _extract_code_cells(notebook_path: Path) -> list[str]: + """Extract source code from all code cells in a notebook. + + :param notebook_path: path to the .ipynb file + :return: list of source code strings, one per code cell + """ + with notebook_path.open() as f: + nb = json.load(f) + return [ + "".join(cell.get("source", [])) + for cell in nb.get("cells", []) + if cell.get("cell_type") == "code" + ] + + +@pytest.mark.parametrize("notebook", NCBI_NOTEBOOKS) +class TestNotebookSyntax: + """Validate that every code cell in each notebook is syntactically valid Python.""" + + def test_all_cells_parse(self, notebook: str) -> None: + """Verify every code cell compiles without SyntaxError.""" + path = NOTEBOOKS_DIR / notebook + assert path.exists(), f"Notebook not found: {path}" + cells = _extract_code_cells(path) + assert len(cells) > 0, f"No code cells found in {notebook}" + for i, source in enumerate(cells, 1): + try: + ast.parse(source, filename=f"{notebook}:cell{i}") + except SyntaxError as exc: + pytest.fail(f"{notebook} cell {i} has a syntax error: {exc}") + + def test_no_empty_code_cells(self, notebook: str) -> None: + """Verify no code cell is completely empty.""" + path = NOTEBOOKS_DIR / notebook + cells = _extract_code_cells(path) + for i, source in enumerate(cells, 1): + assert source.strip(), f"{notebook} cell {i} is empty" + + +class TestManifestNotebookImports: + """Verify that all imports in the manifest notebook resolve.""" + + def test_imports_resolve(self) -> None: + """All manifest notebook imports are verified at module load time above.""" + assert callable(download_assembly_summary) + assert callable(write_updated_manifest) + + +class TestPromoteNotebookImports: + """Verify that all imports in the promote notebook resolve.""" + + def test_imports_resolve(self) -> None: + """All promote notebook imports are verified at module load time above.""" + assert callable(promote_from_s3) + assert isinstance(DEFAULT_PATH_PREFIX, str) diff --git a/tests/ncbi_ftp/test_promote.py b/tests/ncbi_ftp/test_promote.py new file mode 100644 index 00000000..bd1d7f0f --- /dev/null +++ b/tests/ncbi_ftp/test_promote.py @@ -0,0 +1,278 @@ +"""Tests for ncbi_ftp.promote module — S3 promote, archive, manifest trimming.""" + +from pathlib import Path + +import botocore.client +import pytest + +from cdm_data_loaders.ncbi_ftp.promote import ( + DEFAULT_PATH_PREFIX, + _archive_assemblies, + _trim_manifest, + promote_from_s3, +) +from tests.ncbi_ftp.conftest import TEST_BUCKET + + +@pytest.mark.s3 +class TestPromoteFromS3: + """Test promote_from_s3 with moto-mocked S3.""" + + def _stage_files(self, s3_client: botocore.client.BaseClient, prefix: str) -> None: + """Upload sample staged files to mock S3.""" + for key in [ + f"{prefix}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz", + f"{prefix}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz.md5", + f"{prefix}download_report.json", + ]: + body = b"md5hash123" if key.endswith(".md5") else b"data" + s3_client.put_object(Bucket=TEST_BUCKET, Key=key, Body=body) + + def test_dry_run_no_writes(self, mock_s3_client_no_checksum: botocore.client.BaseClient) -> None: + """Verify dry_run does not write any objects.""" + prefix = "staging/run1/" + self._stage_files(mock_s3_client_no_checksum, prefix) + + report = promote_from_s3( + staging_prefix=prefix, + bucket=TEST_BUCKET, + dry_run=True, + ) + assert report["promoted"] == 1 + assert report["dry_run"] is True + + # Final path should NOT exist + final_key = ( + f"{DEFAULT_PATH_PREFIX}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz" + ) + resp = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=final_key) + assert resp.get("KeyCount", 0) == 0 + + def test_promotes_with_metadata(self, mock_s3_client_no_checksum: botocore.client.BaseClient) -> None: + """Verify objects are promoted with MD5 metadata attached.""" + prefix = "staging/run1/" + self._stage_files(mock_s3_client_no_checksum, prefix) + + report = promote_from_s3( + staging_prefix=prefix, + bucket=TEST_BUCKET, + ) + assert report["promoted"] == 1 + assert report["failed"] == 0 + + # Check final object exists with metadata + final_key = ( + f"{DEFAULT_PATH_PREFIX}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz" + ) + resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=final_key) + assert resp["Metadata"].get("md5") == "md5hash123" + + def test_skips_download_report(self, mock_s3_client_no_checksum: botocore.client.BaseClient) -> None: + """Verify download_report.json is not promoted.""" + prefix = "staging/run1/" + self._stage_files(mock_s3_client_no_checksum, prefix) + + report = promote_from_s3(staging_prefix=prefix, bucket=TEST_BUCKET) + # Only the .fna.gz data file, not download_report.json + assert report["promoted"] == 1 + + +@pytest.mark.s3 +class TestTrimManifest: + """Test _trim_manifest removes promoted accessions from S3 manifest.""" + + def test_trims_promoted(self, mock_s3_client_no_checksum: botocore.client.BaseClient) -> None: + """Verify promoted accessions are removed from manifest.""" + manifest_key = "manifests/transfer_manifest.txt" + manifest_body = ( + "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6/\n" + "/genomes/all/GCF/000/001/405/GCF_000001405.40_GRCh38.p14/\n" + ) + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=manifest_key, Body=manifest_body.encode()) + + _trim_manifest(manifest_key, TEST_BUCKET, {"GCF_000001215.4"}) + + resp = mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=manifest_key) + remaining = resp["Body"].read().decode() + assert "GCF_000001215.4" not in remaining + assert "GCF_000001405.40" in remaining + + def test_trims_all(self, mock_s3_client_no_checksum: botocore.client.BaseClient) -> None: + """Verify all entries can be trimmed leaving an empty manifest.""" + manifest_key = "manifests/transfer_manifest.txt" + manifest_body = "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6/\n" + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=manifest_key, Body=manifest_body.encode()) + + _trim_manifest(manifest_key, TEST_BUCKET, {"GCF_000001215.4"}) + + resp = mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=manifest_key) + remaining = resp["Body"].read().decode().strip() + assert remaining == "" + + +@pytest.mark.s3 +class TestArchiveAssemblies: + """Test _archive_assemblies with moto-mocked S3.""" + + def test_archives_and_deletes_removed( + self, mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path + ) -> None: + """Verify removed accessions are archived and originals deleted.""" + accession = "GCF_000005845.2" + key = f"{DEFAULT_PATH_PREFIX}raw_data/GCF/000/005/845/{accession}_ASM584v2/{accession}_genomic.fna.gz" + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"data") + + manifest = tmp_path / "removed.txt" + manifest.write_text(f"{accession}\n") + + count = _archive_assemblies( + str(manifest), + bucket=TEST_BUCKET, + ncbi_release="2024-01", + archive_reason="replaced_or_suppressed", + delete_source=True, + ) + assert count == 1 + + # Original should be deleted + resp = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=key) + assert resp.get("KeyCount", 0) == 0 + + # Archived copy should exist + archive_key = ( + f"{DEFAULT_PATH_PREFIX}archive/2024-01/" + f"raw_data/GCF/000/005/845/{accession}_ASM584v2/{accession}_genomic.fna.gz" + ) + resp = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_key) + assert resp.get("KeyCount", 0) == 1 + + def test_archives_updated_without_deleting( + self, mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path + ) -> None: + """Verify updated accessions are archived but originals remain.""" + accession = "GCF_000001215.4" + asm_dir = f"{accession}_Release_6" + key = f"{DEFAULT_PATH_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"original-data") + + manifest = tmp_path / "updated.txt" + manifest.write_text(f"{accession}\n") + + count = _archive_assemblies( + str(manifest), + bucket=TEST_BUCKET, + ncbi_release="2024-06", + archive_reason="updated", + delete_source=False, + ) + assert count == 1 + + # Original still exists (promote will overwrite it) + resp = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=key) + assert resp.get("KeyCount", 0) == 1 + + # Archived copy exists with correct metadata + archive_key = ( + f"{DEFAULT_PATH_PREFIX}archive/2024-06/" + f"raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" + ) + resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=archive_key) + assert resp["Metadata"]["archive_reason"] == "updated" + assert resp["Metadata"]["ncbi_last_release"] == "2024-06" + + def test_multiple_releases_no_collision( + self, mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path + ) -> None: + """Verify archiving the same accession in different releases creates distinct folders.""" + accession = "GCF_000001215.4" + asm_dir = f"{accession}_Release_6" + key = f"{DEFAULT_PATH_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"v1-data") + + manifest = tmp_path / "updated.txt" + manifest.write_text(f"{accession}\n") + + # First archive: release 2024-01 + _archive_assemblies(str(manifest), bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated") + + # Simulate promote overwriting source + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"v2-data") + + # Second archive: release 2024-06 + _archive_assemblies(str(manifest), bucket=TEST_BUCKET, ncbi_release="2024-06", archive_reason="updated") + + archive_key_1 = ( + f"{DEFAULT_PATH_PREFIX}archive/2024-01/" + f"raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" + ) + archive_key_2 = ( + f"{DEFAULT_PATH_PREFIX}archive/2024-06/" + f"raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" + ) + resp1 = mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=archive_key_1) + assert resp1["Body"].read() == b"v1-data" + + resp2 = mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=archive_key_2) + assert resp2["Body"].read() == b"v2-data" + + def test_dry_run_no_side_effects( + self, mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path + ) -> None: + """Verify dry_run does not copy or delete anything.""" + accession = "GCF_000005845.2" + key = f"{DEFAULT_PATH_PREFIX}raw_data/GCF/000/005/845/{accession}_ASM584v2/{accession}_genomic.fna.gz" + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"data") + + manifest = tmp_path / "removed.txt" + manifest.write_text(f"{accession}\n") + + count = _archive_assemblies( + str(manifest), + bucket=TEST_BUCKET, + ncbi_release="2024-01", + archive_reason="replaced_or_suppressed", + delete_source=True, + dry_run=True, + ) + assert count == 1 + + # Original still exists + resp = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=key) + assert resp.get("KeyCount", 0) == 1 + + # No archive created + archive_prefix = f"{DEFAULT_PATH_PREFIX}archive/2024-01/" + resp = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_prefix) + assert resp.get("KeyCount", 0) == 0 + + def test_no_existing_objects_skips( + self, mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path + ) -> None: + """Verify accessions with no existing S3 objects are silently skipped.""" + manifest = tmp_path / "updated.txt" + manifest.write_text("GCF_000001215.4\n") + + count = _archive_assemblies(str(manifest), bucket=TEST_BUCKET, ncbi_release="2024-01") + assert count == 0 + + def test_unknown_release_fallback( + self, mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path + ) -> None: + """Verify ncbi_release=None falls back to 'unknown'.""" + accession = "GCF_000001215.4" + asm_dir = f"{accession}_Release_6" + key = f"{DEFAULT_PATH_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"data") + + manifest = tmp_path / "updated.txt" + manifest.write_text(f"{accession}\n") + + count = _archive_assemblies(str(manifest), bucket=TEST_BUCKET, ncbi_release=None) + assert count == 1 + + archive_key = ( + f"{DEFAULT_PATH_PREFIX}archive/unknown/" + f"raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" + ) + resp = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_key) + assert resp.get("KeyCount", 0) == 1 diff --git a/tests/pipelines/test_ncbi_ftp_download.py b/tests/pipelines/test_ncbi_ftp_download.py new file mode 100644 index 00000000..1dfc0997 --- /dev/null +++ b/tests/pipelines/test_ncbi_ftp_download.py @@ -0,0 +1,230 @@ +"""Tests for pipelines.ncbi_ftp_download — settings, batch orchestration, CLI.""" + +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from pydantic import ValidationError + +from cdm_data_loaders.ncbi_ftp.assembly import FTP_HOST +from cdm_data_loaders.pipelines.cts_defaults import INPUT_MOUNT, OUTPUT_MOUNT +from cdm_data_loaders.pipelines.ncbi_ftp_download import DownloadSettings, download_batch + +_DEFAULT_THREADS = 4 +_CUSTOM_THREADS = 8 +_ALIAS_THREADS = 16 +_BOUNDARY_MIN = 1 +_BOUNDARY_MAX = 32 +_OVER_MAX = 64 +_CUSTOM_LIMIT = 100 +_ALIAS_LIMIT = 50 +_EXPECTED_ATTEMPTED = 2 + + +def make_settings(**kwargs: str | int) -> DownloadSettings: + """Generate a validated DownloadSettings object.""" + return DownloadSettings(_cli_parse_args=[], **kwargs) + + +# ── Settings defaults ──────────────────────────────────────────────────── + + +class TestDownloadSettingsDefaults: + """Test default settings.""" + + def test_manifest_default(self) -> None: + """Verify default manifest path uses INPUT_MOUNT.""" + s = make_settings() + assert s.manifest == f"{INPUT_MOUNT}/transfer_manifest.txt" + + def test_output_dir_default(self) -> None: + """Verify default output_dir uses OUTPUT_MOUNT.""" + s = make_settings() + assert s.output_dir == OUTPUT_MOUNT + + def test_threads_default(self) -> None: + """Verify default threads is 4.""" + s = make_settings() + assert s.threads == _DEFAULT_THREADS + + def test_ftp_host_default(self) -> None: + """Verify default ftp_host matches FTP_HOST constant.""" + s = make_settings() + assert s.ftp_host == FTP_HOST + + def test_limit_default_none(self) -> None: + """Verify default limit is None.""" + s = make_settings() + assert s.limit is None + + +# ── Settings all params ────────────────────────────────────────────────── + + +class TestDownloadSettingsAllParams: + """Test with all params set.""" + + def test_all_params(self) -> None: + """Verify all parameters are correctly set when provided.""" + s = make_settings( + manifest="/data/my_manifest.txt", + output_dir="/data/output", + threads=_CUSTOM_THREADS, + ftp_host="ftp.example.com", + limit=_CUSTOM_LIMIT, + ) + assert s.manifest == "/data/my_manifest.txt" + assert s.output_dir == "/data/output" + assert s.threads == _CUSTOM_THREADS + assert s.ftp_host == "ftp.example.com" + assert s.limit == _CUSTOM_LIMIT + + +# ── Settings aliases ───────────────────────────────────────────────────── + + +class TestDownloadSettingsAliases: + """Test CLI alias resolution.""" + + def test_manifest_alias_m(self) -> None: + """Verify 'm' alias resolves to manifest.""" + s = make_settings(m="/data/m.txt") + assert s.manifest == "/data/m.txt" + + def test_output_dir_alias_o(self) -> None: + """Verify 'o' alias resolves to output_dir.""" + s = make_settings(o="/data/o") + assert s.output_dir == "/data/o" + + def test_threads_alias_t(self) -> None: + """Verify 't' alias resolves to threads.""" + s = make_settings(t=_ALIAS_THREADS) + assert s.threads == _ALIAS_THREADS + + def test_limit_alias_l(self) -> None: + """Verify 'l' alias resolves to limit.""" + s = make_settings(l=_ALIAS_LIMIT) + assert s.limit == _ALIAS_LIMIT + + +# ── Settings validation ────────────────────────────────────────────────── + + +class TestDownloadSettingsValidation: + """Test validation constraints.""" + + def test_threads_too_low(self) -> None: + """Verify threads=0 raises ValidationError.""" + with pytest.raises(ValidationError): + make_settings(threads=0) + + def test_threads_too_high(self) -> None: + """Verify threads above 32 raises ValidationError.""" + with pytest.raises(ValidationError): + make_settings(threads=_OVER_MAX) + + def test_threads_boundary_1(self) -> None: + """Verify threads=1 is accepted.""" + s = make_settings(threads=_BOUNDARY_MIN) + assert s.threads == _BOUNDARY_MIN + + def test_threads_boundary_32(self) -> None: + """Verify threads=32 is accepted.""" + s = make_settings(threads=_BOUNDARY_MAX) + assert s.threads == _BOUNDARY_MAX + + def test_limit_must_be_positive(self) -> None: + """Verify limit=0 raises ValidationError.""" + with pytest.raises(ValidationError): + make_settings(limit=0) + + +# ── download_batch ─────────────────────────────────────────────────────── + + +class TestDownloadBatch: + """Test download_batch with mocked internals.""" + + @pytest.fixture(autouse=True) + def _mock_ftp_pool(self) -> None: + """Prevent real FTP connections from the ThreadLocalFTP pool.""" + mock_pool = MagicMock() + with patch("cdm_data_loaders.pipelines.ncbi_ftp_download.ThreadLocalFTP", return_value=mock_pool): + yield + + def test_reads_manifest_and_calls_download(self, tmp_path: Path) -> None: + """Verify manifest is read and download is called for each entry.""" + manifest = tmp_path / "manifest.txt" + manifest.write_text( + "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/\n" + "/genomes/all/GCF/000/001/405/GCF_000001405.40_GRCh38.p14/\n" + ) + output = tmp_path / "output" + output.mkdir() + + mock_stats = {"accession": "test", "files_downloaded": 3} + with patch("cdm_data_loaders.pipelines.ncbi_ftp_download.download_assembly_to_local", return_value=mock_stats): + report = download_batch( + manifest_path=str(manifest), + output_dir=str(output), + threads=1, + ftp_host="ftp.example.com", + ) + + assert report["total_attempted"] == _EXPECTED_ATTEMPTED + assert report["succeeded"] == _EXPECTED_ATTEMPTED + assert report["failed"] == 0 + + def test_limit_truncates(self, tmp_path: Path) -> None: + """Verify limit parameter truncates the number of assemblies processed.""" + manifest = tmp_path / "manifest.txt" + manifest.write_text( + "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/\n" + "/genomes/all/GCF/000/001/405/GCF_000001405.40_GRCh38.p14/\n" + ) + output = tmp_path / "output" + output.mkdir() + + mock_stats = {"accession": "test", "files_downloaded": 1} + with patch("cdm_data_loaders.pipelines.ncbi_ftp_download.download_assembly_to_local", return_value=mock_stats): + report = download_batch( + manifest_path=str(manifest), + output_dir=str(output), + threads=1, + limit=1, + ) + assert report["total_attempted"] == 1 + + def test_writes_report_json(self, tmp_path: Path) -> None: + """Verify download_report.json is written to the output directory.""" + manifest = tmp_path / "manifest.txt" + manifest.write_text("/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/\n") + output = tmp_path / "output" + output.mkdir() + + mock_stats = {"accession": "GCF_000001215.4", "files_downloaded": 5} + with patch("cdm_data_loaders.pipelines.ncbi_ftp_download.download_assembly_to_local", return_value=mock_stats): + download_batch(manifest_path=str(manifest), output_dir=str(output), threads=1) + + report_file = output / "download_report.json" + assert report_file.exists() + report = json.loads(report_file.read_text()) + assert "timestamp" in report + assert report["succeeded"] == 1 + + def test_handles_download_failure(self, tmp_path: Path) -> None: + """Verify failed downloads are counted and do not crash the batch.""" + manifest = tmp_path / "manifest.txt" + manifest.write_text("/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/\n") + output = tmp_path / "output" + output.mkdir() + + with patch( + "cdm_data_loaders.pipelines.ncbi_ftp_download.download_assembly_to_local", + side_effect=RuntimeError("connection lost"), + ): + report = download_batch(manifest_path=str(manifest), output_dir=str(output), threads=1) + + assert report["failed"] == 1 + assert report["succeeded"] == 0 diff --git a/tests/utils/test_checksums.py b/tests/utils/test_checksums.py new file mode 100644 index 00000000..6c7bdbc6 --- /dev/null +++ b/tests/utils/test_checksums.py @@ -0,0 +1,76 @@ +"""Tests for utils.checksums module — MD5 and CRC64/NVME checksum utilities.""" + +import base64 +import hashlib +from pathlib import Path + +import pytest + +from cdm_data_loaders.utils.checksums import compute_md5, verify_md5 + +_EXPECTED_CRC64_BYTE_LEN = 8 + + +class TestComputeMd5: + """Test MD5 computation.""" + + def test_correct_hash(self, tmp_path: Path) -> None: + """Verify MD5 matches hashlib reference.""" + f = tmp_path / "test.bin" + f.write_bytes(b"Hello, World!") + assert compute_md5(f) == hashlib.md5(b"Hello, World!").hexdigest() # noqa: S324 + + def test_empty_file(self, tmp_path: Path) -> None: + """Verify MD5 of an empty file.""" + f = tmp_path / "empty" + f.write_bytes(b"") + assert compute_md5(f) == hashlib.md5(b"").hexdigest() # noqa: S324 + + def test_accepts_str_path(self, tmp_path: Path) -> None: + """Verify compute_md5 accepts a string path.""" + f = tmp_path / "test.txt" + f.write_bytes(b"data") + assert compute_md5(str(f)) == hashlib.md5(b"data").hexdigest() # noqa: S324 + + +class TestVerifyMd5: + """Test MD5 verification.""" + + def test_correct(self, tmp_path: Path) -> None: + """Verify True when MD5 matches.""" + f = tmp_path / "test.bin" + f.write_bytes(b"Hello, World!") + expected = hashlib.md5(b"Hello, World!").hexdigest() # noqa: S324 + assert verify_md5(f, expected) is True + + def test_incorrect(self, tmp_path: Path) -> None: + """Verify False when MD5 does not match.""" + f = tmp_path / "test.bin" + f.write_bytes(b"Hello, World!") + assert verify_md5(f, "0000000000000000") is False + + +class TestComputeCrc64nvme: + """Test CRC64/NVME computation (skipped if awscrt unavailable).""" + + @pytest.fixture(autouse=True) + def _skip_if_no_awscrt(self) -> None: + pytest.importorskip("awscrt") + + def test_returns_base64(self, tmp_path: Path) -> None: + """Verify CRC64/NVME returns an 8-byte base64 string.""" + from cdm_data_loaders.utils.checksums import compute_crc64nvme # noqa: PLC0415 + + f = tmp_path / "test.bin" + f.write_bytes(b"Hello, World!") + crc = compute_crc64nvme(f) + decoded = base64.b64decode(crc) + assert len(decoded) == _EXPECTED_CRC64_BYTE_LEN + + def test_deterministic(self, tmp_path: Path) -> None: + """Verify repeated calls return the same checksum.""" + from cdm_data_loaders.utils.checksums import compute_crc64nvme # noqa: PLC0415 + + f = tmp_path / "test.bin" + f.write_bytes(b"test data for checksum") + assert compute_crc64nvme(f) == compute_crc64nvme(f) diff --git a/tests/utils/test_ftp_client.py b/tests/utils/test_ftp_client.py new file mode 100644 index 00000000..385fd329 --- /dev/null +++ b/tests/utils/test_ftp_client.py @@ -0,0 +1,199 @@ +"""Tests for utils.ftp_client module — mock ftplib for keepalive, retry, thread-local.""" + +import socket +import time +from collections.abc import Callable +from ftplib import FTP, error_temp +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from cdm_data_loaders.utils.ftp_client import ( + ThreadLocalFTP, + _set_keepalive, + connect_ftp, + ftp_download_file, + ftp_list_dir, + ftp_noop_keepalive, + ftp_retrieve_text, +) + +_IDLE_SECONDS = 30 +_KEEPIDLE_VALUE = 60 +_KEEPALIVE_INTERVAL = 25 +_EXPECTED_RETRY_COUNT = 2 +_FTP_TIMEOUT = 30 +_ERR_421 = "421 timeout" + + +class TestSetKeepalive: + """Test TCP keepalive socket options.""" + + def test_sets_so_keepalive(self) -> None: + """Verify SO_KEEPALIVE is set on the socket.""" + mock_ftp = MagicMock(spec=FTP) + mock_sock = MagicMock() + mock_ftp.sock = mock_sock + _set_keepalive(mock_ftp) + mock_sock.setsockopt.assert_any_call(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + + def test_sets_tcp_keepidle(self) -> None: + """Verify TCP_KEEPIDLE is set when available.""" + mock_ftp = MagicMock(spec=FTP) + mock_sock = MagicMock() + mock_ftp.sock = mock_sock + _set_keepalive(mock_ftp, idle=_KEEPIDLE_VALUE) + if hasattr(socket, "TCP_KEEPIDLE"): + mock_sock.setsockopt.assert_any_call(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, _KEEPIDLE_VALUE) + + +class TestConnectFtp: + """Test connect_ftp creates and configures an FTP connection.""" + + @patch("cdm_data_loaders.utils.ftp_client.FTP") + def test_connect_and_login(self, mock_ftp_cls: MagicMock) -> None: + """Verify FTP object is created, login called, and returned.""" + mock_ftp = MagicMock() + mock_ftp.sock = MagicMock() + mock_ftp_cls.return_value = mock_ftp + result = connect_ftp("ftp.example.com", timeout=_FTP_TIMEOUT) + mock_ftp_cls.assert_called_once_with("ftp.example.com", timeout=_FTP_TIMEOUT) + mock_ftp.login.assert_called_once() + assert result is mock_ftp + + +class TestFtpNoopKeepalive: + """Test NOOP keepalive logic.""" + + def test_sends_noop_when_idle(self) -> None: + """Verify NOOP is sent when idle exceeds interval.""" + mock_ftp = MagicMock(spec=FTP) + old_time = time.monotonic() - _IDLE_SECONDS + new_time = ftp_noop_keepalive(mock_ftp, old_time, interval=_KEEPALIVE_INTERVAL) + mock_ftp.sendcmd.assert_called_once_with("NOOP") + assert new_time > old_time + + def test_no_noop_when_recent(self) -> None: + """Verify no NOOP is sent when activity is recent.""" + mock_ftp = MagicMock(spec=FTP) + recent = time.monotonic() + result = ftp_noop_keepalive(mock_ftp, recent, interval=_KEEPALIVE_INTERVAL) + mock_ftp.sendcmd.assert_not_called() + assert result == recent + + +class TestFtpListDir: + """Test ftp_list_dir with retry.""" + + def test_returns_file_list(self) -> None: + """Verify file listing is returned correctly.""" + mock_ftp = MagicMock(spec=FTP) + + def fake_retrlines(_cmd: str, callback: Callable[[str], None]) -> None: + for name in ["file1.txt", "file2.gz"]: + callback(name) + + mock_ftp.retrlines.side_effect = fake_retrlines + result = ftp_list_dir(mock_ftp, "/some/path") + assert result == ["file1.txt", "file2.gz"] + mock_ftp.cwd.assert_called_once_with("/some/path") + + def test_retries_on_error_temp(self) -> None: + """Verify retry logic on FTP temporary errors.""" + mock_ftp = MagicMock(spec=FTP) + call_count = 0 + + def fake_retrlines(_cmd: str, callback: Callable[[str], None]) -> None: + nonlocal call_count + call_count += 1 + if call_count < _EXPECTED_RETRY_COUNT: + raise error_temp(_ERR_421) # noqa: S321 + callback("file.txt") + + mock_ftp.retrlines.side_effect = fake_retrlines + result = ftp_list_dir(mock_ftp, "/path", retries=3) + assert result == ["file.txt"] + assert call_count == _EXPECTED_RETRY_COUNT + + def test_raises_after_exhausted_retries(self) -> None: + """Verify error is raised after all retries are exhausted.""" + mock_ftp = MagicMock(spec=FTP) + mock_ftp.retrlines.side_effect = error_temp(_ERR_421) # noqa: S321 + with pytest.raises(error_temp): + ftp_list_dir(mock_ftp, "/path", retries=_EXPECTED_RETRY_COUNT) + + +class TestFtpDownloadFile: + """Test ftp_download_file with retry.""" + + def test_downloads_file(self, tmp_path: Path) -> None: + """Verify file is downloaded and written to disk.""" + mock_ftp = MagicMock(spec=FTP) + + def fake_retrbinary(_cmd: str, callback: Callable[[bytes], None]) -> None: + callback(b"file data") + + mock_ftp.retrbinary.side_effect = fake_retrbinary + local = tmp_path / "out.bin" + ftp_download_file(mock_ftp, "remote.bin", str(local)) + assert local.read_bytes() == b"file data" + + def test_retries_on_error_temp(self, tmp_path: Path) -> None: + """Verify download retries on FTP temporary errors.""" + mock_ftp = MagicMock(spec=FTP) + call_count = 0 + + def fake_retrbinary(_cmd: str, callback: Callable[[bytes], None]) -> None: + nonlocal call_count + call_count += 1 + if call_count < _EXPECTED_RETRY_COUNT: + msg = "421" + raise error_temp(msg) # noqa: S321 + callback(b"ok") + + mock_ftp.retrbinary.side_effect = fake_retrbinary + local = str(tmp_path / "out.bin") + ftp_download_file(mock_ftp, "remote.bin", local, retries=3) + assert call_count == _EXPECTED_RETRY_COUNT + + +class TestFtpRetrieveText: + """Test ftp_retrieve_text.""" + + def test_returns_content(self) -> None: + """Verify text content is retrieved and joined with newlines.""" + mock_ftp = MagicMock(spec=FTP) + + def fake_retrlines(_cmd: str, callback: Callable[[str], None]) -> None: + for line in ["line1", "line2"]: + callback(line) + + mock_ftp.retrlines.side_effect = fake_retrlines + result = ftp_retrieve_text(mock_ftp, "remote.txt") + assert result == "line1\nline2" + + +class TestThreadLocalFTP: + """Test thread-local FTP connection management.""" + + @patch("cdm_data_loaders.utils.ftp_client.connect_ftp") + def test_get_returns_same_connection(self, mock_connect: MagicMock) -> None: + """Verify get() returns the same FTP connection on repeated calls.""" + mock_ftp = MagicMock() + mock_connect.return_value = mock_ftp + pool = ThreadLocalFTP("ftp.example.com") + ftp1 = pool.get() + ftp2 = pool.get() + assert ftp1 is ftp2 + mock_connect.assert_called_once() + + @patch("cdm_data_loaders.utils.ftp_client.connect_ftp") + def test_close_all(self, mock_connect: MagicMock) -> None: + """Verify close_all() quits the FTP connection.""" + mock_ftp = MagicMock() + mock_connect.return_value = mock_ftp + pool = ThreadLocalFTP("ftp.example.com") + pool.get() + pool.close_all() + mock_ftp.quit.assert_called_once() diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index 7d6398d2..26292a16 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -16,15 +16,18 @@ CDM_LAKE_BUCKET, DEFAULT_EXTRA_ARGS, copy_object, + copy_object_with_metadata, delete_object, download_file, get_s3_client, + head_object, list_matching_objects, object_exists, reset_s3_client, split_s3_path, upload_dir, upload_file, + upload_file_with_metadata, ) AWS_REGION = "us-east-1" @@ -595,3 +598,128 @@ def test_delete_object_removes_object(mock_s3_client: Any, bucket: str, protocol resp = delete_object(s3_path) assert object_exists(s3_path) is False assert resp.get("ResponseMetadata", {}).get("HTTPStatusCode") == 204 + + +# upload_file_with_metadata +@pytest.mark.parametrize("bucket", BUCKETS) +@pytest.mark.s3 +def test_upload_file_with_metadata_attaches_metadata(mock_s3_client: Any, sample_file: Path, bucket: str) -> None: + """Verify that upload_file_with_metadata stores user metadata on the uploaded object.""" + metadata = {"md5": "abc123", "source": "ncbi"} + result = upload_file_with_metadata(sample_file, f"{bucket}/uploads", metadata=metadata) + assert result is True + + resp = mock_s3_client.head_object(Bucket=bucket, Key=f"uploads/{sample_file.name}") + assert resp["Metadata"]["md5"] == "abc123" + assert resp["Metadata"]["source"] == "ncbi" + + +@pytest.mark.s3 +def test_upload_file_with_metadata_custom_object_name(mock_s3_client: Any, sample_file: Path) -> None: + """Verify that the object_name parameter overrides the filename.""" + result = upload_file_with_metadata( + sample_file, f"{CDM_LAKE_BUCKET}/uploads", metadata={"k": "v"}, object_name="renamed.txt" + ) + assert result is True + obj = mock_s3_client.get_object(Bucket=CDM_LAKE_BUCKET, Key="uploads/renamed.txt") + assert obj["Body"].read() == b"hello s3" + + +@pytest.mark.s3 +def test_upload_file_with_metadata_overwrites_existing(mock_s3_client: Any, sample_file: Path) -> None: + """Verify that upload_file_with_metadata uploads even when the object already exists.""" + mock_s3_client.put_object(Bucket=CDM_LAKE_BUCKET, Key=f"uploads/{sample_file.name}", Body=b"old") + result = upload_file_with_metadata(sample_file, f"{CDM_LAKE_BUCKET}/uploads", metadata={"new": "true"}) + assert result is True + obj = mock_s3_client.get_object(Bucket=CDM_LAKE_BUCKET, Key=f"uploads/{sample_file.name}") + assert obj["Body"].read() == b"hello s3" + + +@pytest.mark.usefixtures("mock_s3_client") +@pytest.mark.s3 +def test_upload_file_with_metadata_raises_on_empty_destination(sample_file: Path) -> None: + """Verify ValueError when destination_dir is empty.""" + with pytest.raises(ValueError, match="No destination directory"): + upload_file_with_metadata(sample_file, "", metadata={"k": "v"}) + + +@pytest.mark.usefixtures("mock_s3_client") +@pytest.mark.parametrize("path_type", [str, Path]) +@pytest.mark.s3 +def test_upload_file_with_metadata_accepts_str_and_path(sample_file: Path, path_type: type[str] | type[Path]) -> None: + """Verify that upload_file_with_metadata accepts both str and Path.""" + result = upload_file_with_metadata(path_type(sample_file), f"{CDM_LAKE_BUCKET}/uploads", metadata={}) + assert result is True + + +# head_object +@pytest.mark.s3 +def test_head_object_returns_info(mock_s3_client: Any) -> None: + """Verify that head_object returns size, metadata, and checksum fields.""" + mock_s3_client.put_object( + Bucket=CDM_LAKE_BUCKET, Key="info/file.txt", Body=b"hello", Metadata={"md5": "abc123"} + ) + result = head_object(f"{CDM_LAKE_BUCKET}/info/file.txt") + assert result is not None + assert result["size"] == 5 + assert result["metadata"]["md5"] == "abc123" + # moto may not populate CRC64NVME, but the key should be present + assert "checksum_crc64nvme" in result + + +@pytest.mark.s3 +def test_head_object_returns_none_for_missing(mock_s3_client: Any) -> None: + """Verify that head_object returns None for a non-existent object.""" + result = head_object(f"{CDM_LAKE_BUCKET}/does/not/exist.txt") + assert result is None + + +@pytest.mark.parametrize("protocol", ["", "s3://", "s3a://"]) +@pytest.mark.s3 +def test_head_object_with_protocols(mock_s3_client: Any, protocol: str) -> None: + """Verify that head_object handles all valid protocol prefixes.""" + mock_s3_client.put_object(Bucket=CDM_LAKE_BUCKET, Key="proto/file.txt", Body=b"data") + result = head_object(f"{protocol}{CDM_LAKE_BUCKET}/proto/file.txt") + assert result is not None + assert result["size"] == 4 + + +# copy_object_with_metadata +@pytest.mark.parametrize("destination", BUCKETS) +@pytest.mark.s3 +def test_copy_object_with_metadata_replaces_metadata( + mocked_s3_client_no_checksum: Any, destination: str +) -> None: + """Verify that copy_object_with_metadata copies and replaces metadata.""" + mocked_s3_client_no_checksum.put_object( + Bucket=CDM_LAKE_BUCKET, Key="src/file.txt", Body=b"archive me", Metadata={"old_key": "old_val"} + ) + new_metadata = {"archive_reason": "replaced", "archive_date": "2026-04-16"} + response = copy_object_with_metadata( + f"{CDM_LAKE_BUCKET}/src/file.txt", + f"{destination}/archive/file.txt", + metadata=new_metadata, + ) + assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # verify the destination has the new metadata, not the old + resp = mocked_s3_client_no_checksum.head_object(Bucket=destination, Key="archive/file.txt") + assert resp["Metadata"]["archive_reason"] == "replaced" + assert resp["Metadata"]["archive_date"] == "2026-04-16" + assert "old_key" not in resp["Metadata"] + + # verify source still exists + assert object_exists(f"{CDM_LAKE_BUCKET}/src/file.txt") + + +@pytest.mark.s3 +def test_copy_object_with_metadata_preserves_content(mocked_s3_client_no_checksum: Any) -> None: + """Verify that the content of the copied object matches the original.""" + mocked_s3_client_no_checksum.put_object(Bucket=CDM_LAKE_BUCKET, Key="src/data.bin", Body=b"binary data") + copy_object_with_metadata( + f"{CDM_LAKE_BUCKET}/src/data.bin", + f"{CDM_LAKE_BUCKET}/dst/data.bin", + metadata={"tag": "value"}, + ) + obj = mocked_s3_client_no_checksum.get_object(Bucket=CDM_LAKE_BUCKET, Key="dst/data.bin") + assert obj["Body"].read() == b"binary data" From fa5fa972c10ac64c31fde7169e53e4a069302af3 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 16 Apr 2026 16:02:35 -0700 Subject: [PATCH 002/128] add local minio testing --- README.md | 41 +++ pyproject.toml | 3 +- tests/integration/__init__.py | 0 tests/integration/conftest.py | 251 +++++++++++++++++++ tests/integration/test_download_e2e.py | 129 ++++++++++ tests/integration/test_full_pipeline.py | 216 ++++++++++++++++ tests/integration/test_manifest_e2e.py | 211 ++++++++++++++++ tests/integration/test_promote_e2e.py | 320 ++++++++++++++++++++++++ 8 files changed, 1170 insertions(+), 1 deletion(-) create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/conftest.py create mode 100644 tests/integration/test_download_e2e.py create mode 100644 tests/integration/test_full_pipeline.py create mode 100644 tests/integration/test_manifest_e2e.py create mode 100644 tests/integration/test_promote_e2e.py diff --git a/README.md b/README.md index 98aba015..2273f299 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,47 @@ To generate coverage for the tests, run The standard python `coverage` package is used and coverage can be generated as html or other formats by changing the parameters. +#### Integration tests (MinIO + NCBI FTP) + +End-to-end integration tests for the NCBI assembly pipeline live in `tests/integration/`. They exercise the full flow — manifest diffing, FTP download, S3 promote/archive — against a locally running [MinIO](https://min.io/) container and the real NCBI FTP server. + +**Requirements:** +- Docker (for MinIO) +- Network access to `ftp.ncbi.nlm.nih.gov` + +**1. Start MinIO locally:** + +```sh +docker run -d \ + --name minio \ + -p 9000:9000 \ + -p 9001:9001 \ + -e MINIO_ROOT_USER=minioadmin \ + -e MINIO_ROOT_PASSWORD=minioadmin \ + minio/minio server /data --console-address ":9001" +``` + +**2. Run the integration tests:** + +```sh +> uv run pytest tests/integration/ -m integration -v +``` + +Tests are automatically skipped when MinIO is not reachable, so the default `uv run pytest` will never fail due to a missing MinIO instance. + +**3. Inspect results:** + +Buckets are **not** cleaned up after tests. Browse the MinIO console at [http://localhost:9001](http://localhost:9001) (login: `minioadmin` / `minioadmin`) to inspect the final state of each test bucket. Each test method creates its own bucket (e.g. `integ-test-promote-dry-run`). + +**4. Stop MinIO when done:** + +```sh +docker stop minio && docker rm minio +``` + +> **Note:** These tests download real assemblies from NCBI FTP and are inherently slow (~30–60s per assembly). They are also marked `slow_test` so you can exclude them independently: `uv run pytest -m "not slow_test"`. + + ## Loading genomes, contigs, and features The [genome loader](src/cdm_data_loaders/parsers/genome_loader.py) can be used to load and integrate data from related GFF and FASTA files. Currently, the loader requires a GFF file and two FASTA files (one for amino acid seqs, one for nucleic acid seqs) for each genome. The list of files to be processed should be specified in the genome paths file, which has the following format: diff --git a/pyproject.toml b/pyproject.toml index 5898efde..de5e8154 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -172,6 +172,7 @@ ignore = [ [tool.ruff.lint.per-file-ignores] "*.ipynb" = ["T201"] # ignore printing in notebooks "tests/**/*.py" = ["S101", "T201", "FBT001", "FBT002", "ARG002"] # use of assert, booleans, unused mock args +"tests/integration/**/*.py" = ["S101", "T201", "FBT001", "FBT002", "ARG002", "ANN401"] "tests/utils/test_s3.py" = ["ANN401"] "**/__init__.py" = ["D104"] @@ -192,7 +193,7 @@ log_cli = true log_cli_level = "INFO" log_level = "INFO" addopts = ["-v"] -markers = ["requires_spark: must be run in an environment where spark is available", "s3: tests that mock s3 interactions", "slow_test: does what it says on the tin"] +markers = ["requires_spark: must be run in an environment where spark is available", "s3: tests that mock s3 interactions", "slow_test: does what it says on the tin", "integration: end-to-end tests requiring a running MinIO instance and network access"] # environment settings for running tests [tool.pytest_env] diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 00000000..1d90faa6 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,251 @@ +"""Shared fixtures and helpers for MinIO-backed integration tests. + +Integration tests are auto-skipped when MinIO is not reachable. Each test +method gets its own bucket (derived from the test node name) that is emptied +on re-run but **never deleted** after the test — this lets developers inspect +the final state of the object store via the MinIO console. +""" + +from __future__ import annotations + +import functools +import hashlib +import os +import re +from pathlib import Path +from typing import TYPE_CHECKING, Any +from unittest.mock import patch + +import boto3 +import botocore.client +import pytest + +import cdm_data_loaders.ncbi_ftp.manifest as manifest_mod +import cdm_data_loaders.ncbi_ftp.promote as promote_mod +import cdm_data_loaders.utils.s3 as s3_utils +from cdm_data_loaders.ncbi_ftp.assembly import build_accession_path +from cdm_data_loaders.utils.s3 import reset_s3_client + +if TYPE_CHECKING: + from collections.abc import Callable + +# ── MinIO connection defaults ─────────────────────────────────────────── + +MINIO_ENDPOINT_URL = os.environ.get("MINIO_ENDPOINT_URL", "http://localhost:9000") +MINIO_ACCESS_KEY = os.environ.get("MINIO_ACCESS_KEY", "minioadmin") +MINIO_SECRET_KEY = os.environ.get("MINIO_SECRET_KEY", "minioadmin") + +# Maximum length of a bucket name per S3/DNS spec +_MAX_BUCKET_LEN = 63 + + +# ── MinIO reachability check ──────────────────────────────────────────── + +_minio_available: bool | None = None + + +def _minio_reachable() -> bool: + """Return True if the MinIO endpoint accepts connections.""" + try: + client = boto3.client( + "s3", + endpoint_url=MINIO_ENDPOINT_URL, + aws_access_key_id=MINIO_ACCESS_KEY, + aws_secret_access_key=MINIO_SECRET_KEY, + ) + client.list_buckets() + except Exception: # noqa: BLE001 + return False + return True + + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: # noqa: ARG001 + """Auto-skip ``@pytest.mark.integration`` tests when MinIO is unreachable.""" + global _minio_available # noqa: PLW0603 + if _minio_available is None: + _minio_available = _minio_reachable() + if _minio_available: + return + skip_marker = pytest.mark.skip(reason="MinIO not reachable — skipping integration tests") + for item in items: + if "integration" in item.keywords: + item.add_marker(skip_marker) + + +# ── CRC64NVME workaround ─────────────────────────────────────────────── + + +def strip_checksum_algorithm(method: Callable) -> Callable: + """Wrap a boto3 S3 method to remove ``ChecksumAlgorithm`` if unsupported.""" + + @functools.wraps(method) + def wrapper(*args: object, **kwargs: object) -> object: + kwargs.pop("ChecksumAlgorithm", None) # type: ignore[arg-type] + return method(*args, **kwargs) + + return wrapper + + +# ── Fixtures ──────────────────────────────────────────────────────────── + + +@pytest.fixture(scope="session") +def minio_s3_client() -> botocore.client.BaseClient: + """Session-scoped real boto3 S3 client pointed at the local MinIO instance. + + Patches ``get_s3_client`` on every module that uses it so internal calls + are transparently routed to MinIO. + """ + client = boto3.client( + "s3", + endpoint_url=MINIO_ENDPOINT_URL, + aws_access_key_id=MINIO_ACCESS_KEY, + aws_secret_access_key=MINIO_SECRET_KEY, + ) + + # MinIO may not support CRC64NVME — strip to be safe + client.upload_file = strip_checksum_algorithm(client.upload_file) # type: ignore[method-assign] + client.copy_object = strip_checksum_algorithm(client.copy_object) # type: ignore[method-assign] + + reset_s3_client() + with ( + patch.object(s3_utils, "get_s3_client", return_value=client), + patch.object(promote_mod, "get_s3_client", return_value=client), + patch.object(manifest_mod, "head_object", wraps=s3_utils.head_object), + patch.object(s3_utils, "_s3_client", client), + ): + yield client + reset_s3_client() + + +def _bucket_name_from_node(node_id: str) -> str: + """Derive a DNS-compliant S3 bucket name from a pytest node ID. + + :param node_id: e.g. ``tests/integration/test_promote_e2e.py::test_dry_run`` + :return: e.g. ``integ-test-dry-run`` + """ + # Extract test function name from the node ID + parts = node_id.split("::") + name = parts[-1] if parts else node_id + # Lowercase, replace non-alphanumeric with hyphens, collapse multiples + name = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-") + name = f"integ-{name}" + if len(name) > _MAX_BUCKET_LEN: + # Truncate but keep it unique via a short hash suffix + suffix = hashlib.md5(name.encode()).hexdigest()[:6] # noqa: S324 + name = f"{name[: _MAX_BUCKET_LEN - 7]}-{suffix}" + return name + + +@pytest.fixture +def test_bucket(minio_s3_client: botocore.client.BaseClient, request: pytest.FixtureRequest) -> str: + """Create a per-test-method bucket in MinIO and return its name. + + On re-run, any existing objects are deleted first so the test starts clean. + The bucket is **not** deleted after the test. + """ + bucket = _bucket_name_from_node(request.node.nodeid) + s3 = minio_s3_client + + try: + s3.head_bucket(Bucket=bucket) + # Bucket exists — empty it for a clean run + paginator = s3.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=bucket): + for obj in page.get("Contents", []): + s3.delete_object(Bucket=bucket, Key=obj["Key"]) + except s3.exceptions.NoSuchBucket: + s3.create_bucket(Bucket=bucket) + except botocore.exceptions.ClientError as e: + if e.response["Error"]["Code"] in ("404", "NoSuchBucket"): + s3.create_bucket(Bucket=bucket) + else: + raise + + return bucket + + +# ── Helpers ───────────────────────────────────────────────────────────── + + +def stage_files_to_minio( + s3: botocore.client.BaseClient, + bucket: str, + local_dir: str | Path, + staging_prefix: str, +) -> list[str]: + """Upload a local directory tree to a MinIO staging prefix. + + :param s3: boto3 S3 client + :param bucket: target bucket + :param local_dir: local root directory to upload + :param staging_prefix: S3 key prefix (e.g. ``"staging/run1/"``) + :return: list of S3 keys uploaded + """ + local_dir = Path(local_dir) + keys: list[str] = [] + for path in sorted(local_dir.rglob("*")): + if path.is_dir(): + continue + rel = path.relative_to(local_dir) + key = f"{staging_prefix.rstrip('/')}/{rel}" + s3.upload_file(Filename=str(path), Bucket=bucket, Key=key) + keys.append(key) + return keys + + +def seed_lakehouse( # noqa: PLR0913 + s3: botocore.client.BaseClient, + bucket: str, + accession: str, + files: dict[str, str | bytes], + path_prefix: str, + assembly_dir: str | None = None, +) -> list[str]: + """Seed assembly files at the final Lakehouse path in MinIO. + + :param s3: boto3 S3 client + :param bucket: target bucket + :param accession: assembly accession (e.g. ``"GCF_000001215.4"``) + :param files: mapping of filename → content (str or bytes) + :param path_prefix: Lakehouse prefix (e.g. ``"tenant-general-warehouse/…/ncbi/"``) + :param assembly_dir: full assembly dir name; if None, uses ``accession`` + :return: list of S3 keys created + """ + adir = assembly_dir or accession + rel = build_accession_path(adir) + keys: list[str] = [] + for fname, content in files.items(): + key = f"{path_prefix}{rel}{fname}" + body = content.encode() if isinstance(content, str) else content + md5 = hashlib.md5(body).hexdigest() # noqa: S324 + s3.put_object(Bucket=bucket, Key=key, Body=body, Metadata={"md5": md5}) + keys.append(key) + return keys + + +def list_all_keys(s3: botocore.client.BaseClient, bucket: str, prefix: str = "") -> list[str]: + """List all object keys in a bucket under a prefix. + + :param s3: boto3 S3 client + :param bucket: bucket name + :param prefix: optional key prefix filter + :return: sorted list of keys + """ + keys: list[str] = [] + paginator = s3.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=bucket, Prefix=prefix): + keys.extend(obj["Key"] for obj in page.get("Contents", [])) + return sorted(keys) + + +def get_object_metadata(s3: botocore.client.BaseClient, bucket: str, key: str) -> dict[str, Any]: + """Return the user metadata dict for an S3 object. + + :param s3: boto3 S3 client + :param bucket: bucket name + :param key: object key + :return: metadata dict + """ + resp = s3.head_object(Bucket=bucket, Key=key) + return resp.get("Metadata", {}) diff --git a/tests/integration/test_download_e2e.py b/tests/integration/test_download_e2e.py new file mode 100644 index 00000000..b527de96 --- /dev/null +++ b/tests/integration/test_download_e2e.py @@ -0,0 +1,129 @@ +"""End-to-end tests for Phase 2 — FTP download of assemblies. + +These tests download real (small) assemblies from the NCBI FTP server. +Marked ``integration`` and ``slow_test``; auto-skipped when MinIO is +unreachable. +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + +from cdm_data_loaders.ncbi_ftp.manifest import ( + compute_diff, + download_assembly_summary, + filter_by_prefix_range, + parse_assembly_summary, + write_transfer_manifest, +) +from cdm_data_loaders.pipelines.ncbi_ftp_download import download_batch + +# Use same stable prefix as manifest tests +STABLE_PREFIX = "900" + + +def _manifest_for_one_assembly(tmp_path: Path) -> tuple[Path, str]: + """Create a transfer manifest containing exactly one FTP path. + + Returns ``(manifest_path, accession)`` for the first latest assembly + in the stable prefix range. + """ + raw = download_assembly_summary(database="refseq") + full = parse_assembly_summary(raw) + filtered = filter_by_prefix_range(full, prefix_from=STABLE_PREFIX, prefix_to=STABLE_PREFIX) + diff = compute_diff(filtered, previous_assemblies=None) + + assert len(diff.new) > 0, f"No new assemblies in prefix {STABLE_PREFIX}" + + manifest_path = tmp_path / "transfer_manifest.txt" + write_transfer_manifest(diff, filtered, manifest_path) + + return manifest_path, diff.new[0] + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestDownloadSmallBatch: + """Download a single assembly from NCBI FTP and verify local output.""" + + def test_download_small_batch(self, tmp_path: Path) -> None: + """Download one assembly and verify directory structure and report.""" + manifest_path, _acc = _manifest_for_one_assembly(tmp_path) + + output_dir = tmp_path / "output" + output_dir.mkdir() + + report = download_batch( + manifest_path=str(manifest_path), + output_dir=str(output_dir), + threads=1, + limit=1, + ) + + assert report["succeeded"] >= 1 + assert report["failed"] == 0 + + # Verify directory structure exists + raw_data = output_dir / "raw_data" + assert raw_data.exists(), "Expected raw_data/ directory in output" + + # Should have at least one assembly directory with files + assembly_dirs = list(raw_data.rglob("GCF_*")) + assert len(assembly_dirs) > 0, "Expected at least one assembly directory" + + # Check for .md5 sidecar files + md5_files = list(raw_data.rglob("*.md5")) + assert len(md5_files) > 0, "Expected .md5 sidecar files" + + # Check download report + report_file = output_dir / "download_report.json" + assert report_file.exists() + with report_file.open() as f: + saved_report = json.load(f) + assert saved_report["succeeded"] >= 1 + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestDownloadResumeIncomplete: + """Verify download handles re-runs when some files are already present.""" + + def test_download_resume(self, tmp_path: Path) -> None: + """Re-running download on the same manifest succeeds without errors.""" + manifest_path, _acc = _manifest_for_one_assembly(tmp_path) + + output_dir = tmp_path / "output" + output_dir.mkdir() + + # First download + report1 = download_batch( + manifest_path=str(manifest_path), + output_dir=str(output_dir), + threads=1, + limit=1, + ) + assert report1["succeeded"] >= 1 + + files_after_first = set(output_dir.rglob("*")) + + # Second download — same manifest + report2 = download_batch( + manifest_path=str(manifest_path), + output_dir=str(output_dir), + threads=1, + limit=1, + ) + + # Should succeed without errors (files overwritten or skipped) + assert report2["succeeded"] >= 1 + assert report2["failed"] == 0 + + # All original files should still exist + files_after_second = set(output_dir.rglob("*")) + assert files_after_first.issubset(files_after_second) diff --git a/tests/integration/test_full_pipeline.py b/tests/integration/test_full_pipeline.py new file mode 100644 index 00000000..b98bf342 --- /dev/null +++ b/tests/integration/test_full_pipeline.py @@ -0,0 +1,216 @@ +"""End-to-end tests for the full NCBI assembly pipeline (Phase 1 → 2 → 3). + +Exercises the entire flow: download summary from real NCBI FTP, compute diff, +download a single assembly, stage in MinIO, promote to final Lakehouse path. + +Marked ``integration`` and ``slow_test``; auto-skipped when MinIO is +unreachable. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + +from cdm_data_loaders.ncbi_ftp.manifest import ( + AssemblyRecord, + compute_diff, + download_assembly_summary, + filter_by_prefix_range, + parse_assembly_summary, + write_removed_manifest, + write_transfer_manifest, + write_updated_manifest, +) +from cdm_data_loaders.ncbi_ftp.promote import DEFAULT_PATH_PREFIX, promote_from_s3 +from cdm_data_loaders.pipelines.ncbi_ftp_download import download_batch + +from .conftest import get_object_metadata, list_all_keys, stage_files_to_minio + +STABLE_PREFIX = "900" +STAGING_PREFIX = "staging/run1/" +PATH_PREFIX = DEFAULT_PATH_PREFIX + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestFullPipelineSmallBatch: + """Run the complete pipeline for a single assembly: diff → download → promote.""" + + def test_full_pipeline_small_batch( + self, + minio_s3_client: object, + test_bucket: str, + tmp_path: Path, + ) -> None: + """Single assembly flows through all three phases into MinIO.""" + s3 = minio_s3_client + + # ── Phase 1: Manifest generation ──────────────────────────────── + raw = download_assembly_summary(database="refseq") + full = parse_assembly_summary(raw) + filtered = filter_by_prefix_range(full, prefix_from=STABLE_PREFIX, prefix_to=STABLE_PREFIX) + + diff = compute_diff(filtered, previous_assemblies=None) + assert len(diff.new) > 0, f"No new assemblies in prefix {STABLE_PREFIX}" + + manifest_path = tmp_path / "transfer_manifest.txt" + write_transfer_manifest(diff, filtered, manifest_path) + + # ── Phase 2: Download one assembly from real FTP ──────────────── + output_dir = tmp_path / "output" + output_dir.mkdir() + + report = download_batch( + manifest_path=str(manifest_path), + output_dir=str(output_dir), + threads=1, + limit=1, + ) + assert report["succeeded"] >= 1 + assert report["failed"] == 0 + + # ── Upload local output to MinIO staging ──────────────────────── + keys = stage_files_to_minio(s3, test_bucket, output_dir, STAGING_PREFIX) + assert len(keys) > 0, "Expected files staged to MinIO" + + # ── Phase 3: Promote from staging to final path ───────────────── + promote_report = promote_from_s3( + staging_prefix=STAGING_PREFIX, + bucket=test_bucket, + path_prefix=PATH_PREFIX, + ) + assert promote_report["promoted"] >= 1 + assert promote_report["failed"] == 0 + + # ── Verify final state ────────────────────────────────────────── + final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + assert len(final_keys) >= 1, "Expected files at final Lakehouse path" + + # At least one file should have MD5 metadata + has_md5 = False + for key in final_keys: + meta = get_object_metadata(s3, test_bucket, key) + if meta.get("md5"): + has_md5 = True + break + assert has_md5, "Expected at least one file with MD5 metadata" + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestFullPipelineIncrementalSync: + """Run the pipeline twice to test incremental sync with archival.""" + + def test_full_pipeline_incremental( + self, + minio_s3_client: object, + test_bucket: str, + tmp_path: Path, + ) -> None: + """Second sync archives the old version and promotes the new one.""" + s3 = minio_s3_client + + # ── First sync: Phase 1 → 2 → 3 ──────────────────────────────── + raw = download_assembly_summary(database="refseq") + full = parse_assembly_summary(raw) + filtered = filter_by_prefix_range(full, prefix_from=STABLE_PREFIX, prefix_to=STABLE_PREFIX) + + diff1 = compute_diff(filtered, previous_assemblies=None) + assert len(diff1.new) > 0, f"No new assemblies in prefix {STABLE_PREFIX}" + + manifest1 = tmp_path / "transfer_manifest_1.txt" + write_transfer_manifest(diff1, filtered, manifest1) + + output1 = tmp_path / "output1" + output1.mkdir() + report1 = download_batch(str(manifest1), str(output1), threads=1, limit=1) + assert report1["succeeded"] >= 1 + + stage_files_to_minio(s3, test_bucket, output1, STAGING_PREFIX) + + # Upload manifest to MinIO for trimming + manifest_key = "ncbi/transfer_manifest.txt" + s3.upload_file(Filename=str(manifest1), Bucket=test_bucket, Key=manifest_key) + + promote1 = promote_from_s3( + staging_prefix=STAGING_PREFIX, + bucket=test_bucket, + manifest_path=manifest_key, + path_prefix=PATH_PREFIX, + ) + assert promote1["promoted"] >= 1 + + first_sync_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + assert len(first_sync_keys) >= 1 + + # ── Second sync: Manufacture "previous" with a tweak ──────────── + # Treat first-sync state as "previous", but modify one assembly's + # seq_rel_date so it shows up as "updated". + previous: dict[str, AssemblyRecord] = {} + for acc, rec in filtered.items(): + previous[acc] = AssemblyRecord( + accession=rec.accession, + status=rec.status, + seq_rel_date=rec.seq_rel_date, + ftp_path=rec.ftp_path, + assembly_dir=rec.assembly_dir, + ) + + # Pick the first accession that was actually downloaded + downloaded_acc = diff1.new[0] + if downloaded_acc in previous: + previous[downloaded_acc].seq_rel_date = "1999/01/01" + + diff2 = compute_diff(filtered, previous_assemblies=previous) + + # The modified assembly should appear as "updated" + if downloaded_acc in previous: + assert downloaded_acc in diff2.updated, f"Expected {downloaded_acc} in updated list" + + manifest2 = tmp_path / "transfer_manifest_2.txt" + write_transfer_manifest(diff2, filtered, manifest2) + + updated_manifest = tmp_path / "updated_manifest.txt" + write_updated_manifest(diff2, updated_manifest) + + removed_manifest = tmp_path / "removed_manifest.txt" + write_removed_manifest(diff2, removed_manifest) + + # Phase 2 — re-download the updated assembly + output2 = tmp_path / "output2" + output2.mkdir() + report2 = download_batch(str(manifest2), str(output2), threads=1, limit=1) + assert report2["succeeded"] >= 1 + + # Clean staging and re-stage + staging_keys = list_all_keys(s3, test_bucket, STAGING_PREFIX) + for key in staging_keys: + s3.delete_object(Bucket=test_bucket, Key=key) + stage_files_to_minio(s3, test_bucket, output2, STAGING_PREFIX) + + # Phase 3 — promote with archival + promote2 = promote_from_s3( + staging_prefix=STAGING_PREFIX, + bucket=test_bucket, + updated_manifest=str(updated_manifest), + ncbi_release="test-incremental", + path_prefix=PATH_PREFIX, + ) + assert promote2["failed"] == 0 + + # Verify archive exists + archive_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "archive/test-incremental/") + if promote2["archived"] > 0: + assert len(archive_keys) >= 1 + for key in archive_keys: + meta = get_object_metadata(s3, test_bucket, key) + assert meta.get("archive_reason") == "updated" + + # Final Lakehouse path should still have files + final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + assert len(final_keys) >= 1 diff --git a/tests/integration/test_manifest_e2e.py b/tests/integration/test_manifest_e2e.py new file mode 100644 index 00000000..a8c22c02 --- /dev/null +++ b/tests/integration/test_manifest_e2e.py @@ -0,0 +1,211 @@ +"""End-to-end tests for Phase 1 — manifest generation and diffing. + +These tests hit the real NCBI FTP server (with tight prefix filters) and +optionally use MinIO for checksum verification. Marked ``integration`` +and ``slow_test``; auto-skipped when MinIO is unreachable. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from cdm_data_loaders.ncbi_ftp.assembly import FILE_FILTERS, FTP_HOST, build_accession_path, parse_md5_checksums_file +from cdm_data_loaders.ncbi_ftp.manifest import ( + AssemblyRecord, + compute_diff, + download_assembly_summary, + filter_by_prefix_range, + parse_assembly_summary, + verify_transfer_candidates, + write_diff_summary, + write_removed_manifest, + write_transfer_manifest, + write_updated_manifest, +) +from cdm_data_loaders.ncbi_ftp.promote import DEFAULT_PATH_PREFIX +from cdm_data_loaders.utils.ftp_client import connect_ftp, ftp_retrieve_text + +if TYPE_CHECKING: + from pathlib import Path + +# Use a high-numbered prefix range that typically has only a handful of +# assemblies, keeping FTP traffic minimal. +STABLE_PREFIX = "900" + + +# ── Helpers ───────────────────────────────────────────────────────────── + + +def _download_and_filter() -> tuple[dict[str, AssemblyRecord], dict[str, AssemblyRecord]]: + """Download the current refseq summary and filter to the stable prefix range. + + Returns ``(full_parsed, filtered)``. + """ + raw = download_assembly_summary(database="refseq") + full = parse_assembly_summary(raw) + filtered = filter_by_prefix_range(full, prefix_from=STABLE_PREFIX, prefix_to=STABLE_PREFIX) + return full, filtered + + +# ── Tests ─────────────────────────────────────────────────────────────── + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestFreshSyncNoPrevious: + """Phase 1 with no previous snapshot — everything is 'new'.""" + + def test_fresh_sync_no_previous(self, tmp_path: Path) -> None: + """All assemblies in range appear as new when there is no previous snapshot.""" + _full, filtered = _download_and_filter() + assert len(filtered) > 0, f"Expected assemblies in prefix {STABLE_PREFIX}" + + diff = compute_diff(filtered, previous_assemblies=None) + + # With no previous, every *latest* assembly is new + latest_count = sum(1 for r in filtered.values() if r.status == "latest") + assert len(diff.new) == latest_count + assert len(diff.updated) == 0 + assert len(diff.replaced) == 0 + assert len(diff.suppressed) == 0 + + # Write manifests + transfer_path = tmp_path / "transfer_manifest.txt" + removed_path = tmp_path / "removed_manifest.txt" + updated_path = tmp_path / "updated_manifest.txt" + summary_path = tmp_path / "diff_summary.json" + + paths = write_transfer_manifest(diff, filtered, transfer_path) + removed = write_removed_manifest(diff, removed_path) + updated = write_updated_manifest(diff, updated_path) + write_diff_summary(diff, summary_path, "refseq", STABLE_PREFIX, STABLE_PREFIX) + + assert len(paths) == latest_count + assert len(removed) == 0 + assert len(updated) == 0 + assert transfer_path.exists() + assert summary_path.exists() + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestIncrementalDiffSyntheticPrevious: + """Phase 1 incremental diff with a manufactured 'previous' snapshot.""" + + def test_incremental_diff(self, tmp_path: Path) -> None: + """Detects new, updated, replaced, and suppressed assemblies correctly.""" + _full, filtered = _download_and_filter() + latest = {a: r for a, r in filtered.items() if r.status == "latest"} + assert len(latest) >= 2, f"Need >=2 latest assemblies in prefix {STABLE_PREFIX}" # noqa: PLR2004 + + accs = sorted(latest.keys()) + + # Build synthetic previous: copy current, then mutate + previous: dict[str, AssemblyRecord] = {} + for acc, rec in filtered.items(): + previous[acc] = AssemblyRecord( + accession=rec.accession, + status=rec.status, + seq_rel_date=rec.seq_rel_date, + ftp_path=rec.ftp_path, + assembly_dir=rec.assembly_dir, + ) + + # Remove the first latest → should appear as "new" in diff + new_acc = accs[0] + del previous[new_acc] + + # Modify seq_rel_date of the second latest → should appear as "updated" + updated_acc = accs[1] + previous[updated_acc].seq_rel_date = "1999/01/01" + + # Add a fake accession to previous that is not in current → "suppressed" + fake_suppressed = "GCF_900999999.1" + previous[fake_suppressed] = AssemblyRecord( + accession=fake_suppressed, + status="latest", + seq_rel_date="2020/01/01", + ftp_path="https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/900/999/999/GCF_900999999.1_FakeAsm", + assembly_dir="GCF_900999999.1_FakeAsm", + ) + + diff = compute_diff(filtered, previous_assemblies=previous) + + assert new_acc in diff.new + assert updated_acc in diff.updated + assert fake_suppressed in diff.suppressed + + # Write and verify manifests + transfer_path = tmp_path / "transfer_manifest.txt" + removed_path = tmp_path / "removed_manifest.txt" + updated_path = tmp_path / "updated_manifest.txt" + + paths = write_transfer_manifest(diff, filtered, transfer_path) + removed = write_removed_manifest(diff, removed_path) + updated_list = write_updated_manifest(diff, updated_path) + + assert len(paths) >= 2 # noqa: PLR2004 # at least the new + updated + assert fake_suppressed in removed + assert updated_acc in updated_list + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestVerifyTransferCandidatesPrunes: + """verify_transfer_candidates should prune assemblies already in the store.""" + + def test_prunes_existing_matching_md5( + self, + minio_s3_client: object, + test_bucket: str, + ) -> None: + """Assemblies with matching MD5 metadata in MinIO are pruned from the transfer list.""" + _full, filtered = _download_and_filter() + latest = {a: r for a, r in filtered.items() if r.status == "latest"} + if not latest: + pytest.skip(f"No latest assemblies in prefix {STABLE_PREFIX}") + + # Pick one assembly to pre-seed in MinIO with correct checksums + acc = next(iter(sorted(latest))) + rec = latest[acc] + ftp_dir = rec.ftp_path.replace("https://ftp.ncbi.nlm.nih.gov", "") + + # Fetch the real md5checksums.txt from FTP + ftp = connect_ftp(FTP_HOST) + try: + md5_text = ftp_retrieve_text(ftp, ftp_dir.rstrip("/") + "/md5checksums.txt") + finally: + ftp.quit() + + checksums = parse_md5_checksums_file(md5_text) + + # Seed MinIO with dummy files that have the right MD5 metadata + rel = build_accession_path(rec.assembly_dir) + s3 = minio_s3_client + path_prefix = DEFAULT_PATH_PREFIX + for fname, md5 in checksums.items(): + if any(fname.endswith(suffix) for suffix in FILE_FILTERS): + key = f"{path_prefix}{rel}{fname}" + s3.put_object( + Bucket=test_bucket, + Key=key, + Body=b"placeholder", + Metadata={"md5": md5}, + ) + + # verify_transfer_candidates should prune the seeded assembly + candidates = sorted(latest.keys()) + result = verify_transfer_candidates( + candidates, + filtered, + bucket=test_bucket, + path_prefix=path_prefix, + ) + + assert acc not in result, f"Expected {acc} to be pruned (MD5 matches)" + # Other candidates without seeded data should remain + remaining_candidates = [c for c in candidates if c != acc] + for c in remaining_candidates: + assert c in result, f"Expected {c} to remain (not seeded)" diff --git a/tests/integration/test_promote_e2e.py b/tests/integration/test_promote_e2e.py new file mode 100644 index 00000000..10670598 --- /dev/null +++ b/tests/integration/test_promote_e2e.py @@ -0,0 +1,320 @@ +"""End-to-end tests for Phase 3 — promote and archive in MinIO. + +Pre-stages fake assembly files in MinIO and exercises ``promote_from_s3`` +with various combinations of manifests, archive operations, dry-run mode, +manifest trimming, and incomplete staging. + +Marked ``integration`` and ``slow_test``; auto-skipped when MinIO is +unreachable. Each test method gets its own bucket. +""" + +from __future__ import annotations + +import hashlib +from typing import TYPE_CHECKING + +import pytest + +from cdm_data_loaders.ncbi_ftp.assembly import build_accession_path +from cdm_data_loaders.ncbi_ftp.promote import DEFAULT_PATH_PREFIX, promote_from_s3 + +from .conftest import get_object_metadata, list_all_keys, seed_lakehouse + +if TYPE_CHECKING: + from pathlib import Path + +# Fake assembly details used across tests +ACCESSION_A = "GCF_900000001.1" +ASSEMBLY_DIR_A = "GCF_900000001.1_FakeAssemblyA" +ACCESSION_B = "GCF_900000002.1" +ASSEMBLY_DIR_B = "GCF_900000002.1_FakeAssemblyB" +ACCESSION_C = "GCF_900000003.1" +ASSEMBLY_DIR_C = "GCF_900000003.1_FakeAssemblyC" + +STAGING_PREFIX = "staging/run1/" +PATH_PREFIX = DEFAULT_PATH_PREFIX + +# Fake file contents for staging +FAKE_GENOMIC = b">seq1\nATCGATCG\n" +FAKE_PROTEIN = b">prot1\nMKKL\n" + + +def _md5(data: bytes) -> str: + return hashlib.md5(data).hexdigest() # noqa: S324 + + +def _stage_assembly( + s3: object, + bucket: str, + assembly_dir: str, +) -> None: + """Stage a fake assembly with data files and .md5 sidecars under the staging prefix.""" + rel = build_accession_path(assembly_dir) + base = f"{STAGING_PREFIX}{rel}" + + files = { + f"{assembly_dir}_genomic.fna.gz": FAKE_GENOMIC, + f"{assembly_dir}_protein.faa.gz": FAKE_PROTEIN, + } + + for fname, content in files.items(): + key = f"{base}{fname}" + s3.put_object(Bucket=bucket, Key=key, Body=content) + # Write .md5 sidecar + md5_key = f"{key}.md5" + s3.put_object(Bucket=bucket, Key=md5_key, Body=_md5(content).encode()) + + +def _write_manifest(tmp_path: Path, accessions: list[str], name: str) -> Path: + """Write a manifest file (one accession per line).""" + path = tmp_path / name + path.write_text("\n".join(accessions) + "\n") + return path + + +# ── Tests ─────────────────────────────────────────────────────────────── + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestPromoteFromStaging: + """Promote staged files to final Lakehouse paths.""" + + def test_promote_from_staging(self, minio_s3_client: object, test_bucket: str) -> None: + """Staged files appear at the final Lakehouse path with MD5 metadata.""" + s3 = minio_s3_client + _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) + + report = promote_from_s3( + staging_prefix=STAGING_PREFIX, + bucket=test_bucket, + path_prefix=PATH_PREFIX, + ) + + assert report["promoted"] >= 2 # noqa: PLR2004 # genomic + protein + assert report["failed"] == 0 + assert report["dry_run"] is False + + # Verify files at final path + final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + assert len(final_keys) >= 2 # noqa: PLR2004 + + # Verify MD5 metadata is set + for key in final_keys: + meta = get_object_metadata(s3, test_bucket, key) + assert "md5" in meta, f"Missing md5 metadata on {key}" + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestPromoteIdempotent: + """Promoting the same staging data twice should succeed without errors.""" + + def test_promote_idempotent(self, minio_s3_client: object, test_bucket: str) -> None: + """Second promote succeeds and produces the same final state.""" + s3 = minio_s3_client + _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) + + report1 = promote_from_s3( + staging_prefix=STAGING_PREFIX, + bucket=test_bucket, + path_prefix=PATH_PREFIX, + ) + keys_after_first = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + + report2 = promote_from_s3( + staging_prefix=STAGING_PREFIX, + bucket=test_bucket, + path_prefix=PATH_PREFIX, + ) + keys_after_second = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + + assert report1["failed"] == 0 + assert report2["failed"] == 0 + assert report2["promoted"] >= 1 + assert keys_after_first == keys_after_second + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestPromoteArchiveUpdated: + """Archive existing assemblies before overwriting with updated versions.""" + + def test_archive_updated(self, minio_s3_client: object, test_bucket: str, tmp_path: Path) -> None: + """Updated assemblies are archived before being overwritten.""" + s3 = minio_s3_client + + # Seed "old" version at the final Lakehouse path + old_files = { + f"{ASSEMBLY_DIR_A}_genomic.fna.gz": "old genomic content", + f"{ASSEMBLY_DIR_A}_protein.faa.gz": "old protein content", + } + seed_lakehouse(s3, test_bucket, ACCESSION_A, old_files, PATH_PREFIX, ASSEMBLY_DIR_A) + + # Stage "new" version + _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) + + updated_manifest = _write_manifest(tmp_path, [ACCESSION_A], "updated_manifest.txt") + + report = promote_from_s3( + staging_prefix=STAGING_PREFIX, + bucket=test_bucket, + updated_manifest=str(updated_manifest), + ncbi_release="2024-01", + path_prefix=PATH_PREFIX, + ) + + assert report["archived"] >= 2 # noqa: PLR2004 + assert report["promoted"] >= 2 # noqa: PLR2004 + assert report["failed"] == 0 + + # Verify archive exists + archive_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "archive/2024-01/") + assert len(archive_keys) >= 2 # noqa: PLR2004 + + # Verify archive metadata + for key in archive_keys: + meta = get_object_metadata(s3, test_bucket, key) + assert meta.get("archive_reason") == "updated" + assert meta.get("ncbi_last_release") == "2024-01" + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestPromoteArchiveRemoved: + """Archive and delete replaced/suppressed assemblies.""" + + def test_archive_removed(self, minio_s3_client: object, test_bucket: str, tmp_path: Path) -> None: + """Removed assemblies are archived and source objects are deleted.""" + s3 = minio_s3_client + + # Seed assemblies at final path + files = { + f"{ASSEMBLY_DIR_A}_genomic.fna.gz": "content to archive", + } + seed_lakehouse(s3, test_bucket, ACCESSION_A, files, PATH_PREFIX, ASSEMBLY_DIR_A) + + removed_manifest = _write_manifest(tmp_path, [ACCESSION_A], "removed_manifest.txt") + + # Stage something (even empty staging is fine — promote won't find data files for this accession) + report = promote_from_s3( + staging_prefix=STAGING_PREFIX, + bucket=test_bucket, + removed_manifest=str(removed_manifest), + ncbi_release="2024-01", + path_prefix=PATH_PREFIX, + ) + + assert report["archived"] >= 1 + assert report["failed"] == 0 + + # Verify archive exists + archive_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "archive/2024-01/") + assert len(archive_keys) >= 1 + + # Verify archive metadata + for key in archive_keys: + meta = get_object_metadata(s3, test_bucket, key) + assert meta.get("archive_reason") == "replaced_or_suppressed" + + # Verify source objects are deleted + rel = build_accession_path(ASSEMBLY_DIR_A) + source_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + rel) + assert len(source_keys) == 0, f"Expected source objects deleted, found: {source_keys}" + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestPromoteDryRun: + """Dry-run mode should not create any objects.""" + + def test_promote_dry_run(self, minio_s3_client: object, test_bucket: str) -> None: + """Dry-run logs actions but creates no objects at the final path.""" + s3 = minio_s3_client + _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) + + report = promote_from_s3( + staging_prefix=STAGING_PREFIX, + bucket=test_bucket, + path_prefix=PATH_PREFIX, + dry_run=True, + ) + + assert report["dry_run"] is True + assert report["promoted"] >= 1 + + # No objects should exist at the final path + final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + assert len(final_keys) == 0, f"Dry-run should not create objects, found: {final_keys}" + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestPromoteTrimsManifest: + """Manifest trimming removes promoted accessions.""" + + def test_trims_manifest(self, minio_s3_client: object, test_bucket: str, tmp_path: Path) -> None: + """Transfer manifest in MinIO is trimmed to exclude promoted accessions.""" + s3 = minio_s3_client + + # Upload a transfer manifest with 3 entries to MinIO + manifest_key = "ncbi/transfer_manifest.txt" + manifest_lines = [ + "/genomes/all/GCF/900/000/001/GCF_900000001.1_FakeAssemblyA/\n", + "/genomes/all/GCF/900/000/002/GCF_900000002.1_FakeAssemblyB/\n", + "/genomes/all/GCF/900/000/003/GCF_900000003.1_FakeAssemblyC/\n", + ] + s3.put_object(Bucket=test_bucket, Key=manifest_key, Body="".join(manifest_lines).encode()) + + # Stage only assemblies A and B (not C) + _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) + _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_B) + + report = promote_from_s3( + staging_prefix=STAGING_PREFIX, + bucket=test_bucket, + manifest_path=manifest_key, + path_prefix=PATH_PREFIX, + ) + + assert report["failed"] == 0 + + # Read back the manifest from MinIO + resp = s3.get_object(Bucket=test_bucket, Key=manifest_key) + remaining = resp["Body"].read().decode() + remaining_lines = [line.strip() for line in remaining.strip().splitlines() if line.strip()] + + # Only C should remain (A and B were promoted) + assert len(remaining_lines) == 1, f"Expected 1 remaining entry, got {len(remaining_lines)}: {remaining_lines}" + assert "GCF_900000003" in remaining_lines[0] + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestPromoteIncompleteStaging: + """Incomplete staging (sidecar only, no data) should not promote anything.""" + + def test_incomplete_staging(self, minio_s3_client: object, test_bucket: str) -> None: + """Only .md5 sidecars staged → nothing promoted.""" + s3 = minio_s3_client + + # Stage only .md5 sidecars (no data files) + rel = build_accession_path(ASSEMBLY_DIR_A) + base = f"{STAGING_PREFIX}{rel}" + fname = f"{ASSEMBLY_DIR_A}_genomic.fna.gz" + md5_key = f"{base}{fname}.md5" + s3.put_object(Bucket=test_bucket, Key=md5_key, Body=_md5(FAKE_GENOMIC).encode()) + + report = promote_from_s3( + staging_prefix=STAGING_PREFIX, + bucket=test_bucket, + path_prefix=PATH_PREFIX, + ) + + # .md5 files are sidecars and should not be promoted as data + assert report["promoted"] == 0 + assert report["failed"] == 0 + + # No objects at final path + final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + assert len(final_keys) == 0 From f4d838ad067fa22ae89dd3382dfdc13138f3679d Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Fri, 17 Apr 2026 09:20:49 -0700 Subject: [PATCH 003/128] add NCBI end-to-end testing instructions --- README.md | 2 +- docs/ncbi_ftp_e2e_walkthrough.md | 355 ++++++++++++++++++++++++++++++ notebooks/ncbi_ftp_manifest.ipynb | 33 +++ tests/integration/conftest.py | 24 +- 4 files changed, 390 insertions(+), 24 deletions(-) create mode 100644 docs/ncbi_ftp_e2e_walkthrough.md diff --git a/README.md b/README.md index 2273f299..6d6e643e 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ docker run -d \ -p 9001:9001 \ -e MINIO_ROOT_USER=minioadmin \ -e MINIO_ROOT_PASSWORD=minioadmin \ - minio/minio server /data --console-address ":9001" + minio/minio:RELEASE.2025-02-28T09-55-16Z server /data --console-address ":9001" ``` **2. Run the integration tests:** diff --git a/docs/ncbi_ftp_e2e_walkthrough.md b/docs/ncbi_ftp_e2e_walkthrough.md new file mode 100644 index 00000000..733345be --- /dev/null +++ b/docs/ncbi_ftp_e2e_walkthrough.md @@ -0,0 +1,355 @@ +# NCBI FTP Pipeline — Local End-to-End Walkthrough + +Step-by-step instructions for running a small (≤ 10 assembly) end-to-end sync +of NCBI RefSeq records against a local MinIO container. The walkthrough uses +the two existing Jupyter notebooks for Phases 1 and 3, and the project's Docker +image for the Phase 2 download step. + +> **Prerequisites:** +> - Docker or Podman +> - [uv](https://docs.astral.sh/uv/) (for running notebooks locally) +> - Network access to `ftp.ncbi.nlm.nih.gov` + +--- + +## Architecture overview + +``` + Phase 1 (notebook) Phase 2 (container) Phase 3 (notebook) +┌────────────────────┐ ┌───────────────────────┐ ┌──────────────────────┐ +│ Manifest notebook │ │ ncbi_ftp_sync CLI │ │ Promote notebook │ +│ ─ download FTP │────▶│ ─ read manifest │────▶│ ─ promote staged │ +│ assembly summary │ │ ─ parallel FTP DL │ │ files to Lakehouse │ +│ ─ diff against │ │ ─ MD5 verify │ │ ─ archive old ver. │ +│ previous │ │ ─ write .md5 sidecars │ │ ─ trim manifest │ +│ ─ write manifests │ └──────────┬────────────┘ └──────────────────────┘ +└────────────────────┘ │ + local volume + mounted into + the container + │ + ▼ + ┌────────────────────┐ + │ MinIO (S3-compat) │ + │ localhost:9000 │ + └────────────────────┘ +``` + +--- + +## 1. Start MinIO + +```sh +docker run -d \ + --name minio \ + -p 9000:9000 \ + -p 9001:9001 \ + -e MINIO_ROOT_USER=minioadmin \ + -e MINIO_ROOT_PASSWORD=minioadmin \ + minio/minio:RELEASE.2025-02-28T09-55-16Z server /data --console-address ":9001" +``` + +Create a test bucket via the [MinIO console](http://localhost:9001) +(login: `minioadmin` / `minioadmin`), or from the command line: + +```sh +# Using the AWS CLI (or `mc` if installed) +aws --endpoint-url http://localhost:9000 \ + s3 mb s3://cdm-lake \ + --no-sign-request 2>/dev/null || true + +# Alternatively, using the MinIO client: +# mc alias set local http://localhost:9000 minioadmin minioadmin +# mc mb local/cdm-lake +``` + +--- + +## 2. Phase 1 — Generate manifests (notebook) + +Open `notebooks/ncbi_ftp_manifest.ipynb` in JupyterLab or VS Code. + +### Constants to change (Cell 3) + +| Constant | Walkthrough value | Why | +|-----------------------|----------------------------------|---------------------------------------------------------| +| `DATABASE` | `"refseq"` | keep as-is | +| `PREFIX_FROM` | `"900"` | high-numbered prefix → few assemblies, fast diffing | +| `PREFIX_TO` | `"900"` | single prefix bucket | +| `LIMIT` | `10` | cap to 10 assemblies | +| `PREVIOUS_SUMMARY_S3` | `None` | first run — everything is "new" | +| `SNAPSHOT_UPLOAD_S3` | `None` | skip S3 upload for local testing | +| `OUTPUT_DIR` | `Path("output")` | keep as-is (local directory) | + +### Run the notebook + +Execute all cells in order. After Cell 7 finishes you should see files in +`output/`: + +``` +output/ + transfer_manifest.txt # ≤ 10 FTP directory paths + removed_manifest.txt # empty on first run + updated_manifest.txt # empty on first run + diff_summary.json # counts of new/updated/replaced/suppressed +``` + +Inspect `transfer_manifest.txt` — each line is an FTP directory path like: + +``` +/genomes/all/GCF/900/000/615/GCF_900000615.1_PRJEB7657_assembly +``` + +### Optional: upload manifests to S3 for CTS + +Cell 7 optionally uploads the manifests to an S3 staging prefix so that CTS +can stage them into the container. For local testing, set +`STAGING_S3_PREFIX = None` (the default) and copy the manifest manually in +Step 3b below. + +If you are testing against MinIO and want to exercise the S3 upload path: + +```python +STAGING_S3_PREFIX = "s3://cdm-lake/staging/run1/" +``` + +> **Tip:** If you re-run later with `PREVIOUS_SUMMARY_S3` pointing at a +> snapshot from a prior run you will see `updated`, `replaced`, and +> `suppressed` entries in the diff. + +--- + +## 3. Phase 2 — Download assemblies (container) + +Phase 2 uses the `ncbi_ftp_sync` CLI, which is the container's built-in entry +point for parallel FTP downloads. + +> **CTS (CDM Task Service):** In production, Phase 2 runs as a CTS job. +> CTS stages input files from S3 into the container's filesystem mount +> (`/input_dir`) and copies container output back to S3 (`/output_dir`). +> The container itself never receives S3 credentials. +> See [cdm-task-service](https://github.com/kbase/cdm-task-service) for details. + +For local testing without a CTS instance we run the container directly with +Docker (or Podman), mounting the manifest produced in Phase 1 as input and a +local staging directory as output. + +### 3a. Build the container image + +```sh +# From the repository root +docker build -t cdm-data-loaders . +``` + +### 3b. Prepare local directories + +```sh +mkdir -p staging +cp output/transfer_manifest.txt staging/ +``` + +### 3c. Run the download + +```sh +docker run --rm \ + -v "$(pwd)/staging:/input:ro" \ + -v "$(pwd)/staging:/output" \ + cdm-data-loaders ncbi_ftp_sync \ + --manifest /input/transfer_manifest.txt \ + --output-dir /output \ + --threads 2 \ + --limit 10 +``` + +| Flag | Purpose | +|-----------------|-----------------------------------------------------------| +| `--manifest` | Path to the transfer manifest inside the container | +| `--output-dir` | Where downloads land (mounted from host `staging/`) | +| `--threads` | Parallel FTP connections (2 is polite for testing) | +| `--limit` | Redundant safety cap (already limited in Phase 1) | + +After the container exits, `staging/` will contain: + +``` +staging/ + raw_data/GCF/900/000/615/GCF_900000615.1_PRJEB7657_assembly/ + GCF_900000615.1_PRJEB7657_assembly_genomic.fna.gz + GCF_900000615.1_PRJEB7657_assembly_genomic.fna.gz.md5 + GCF_900000615.1_PRJEB7657_assembly_protein.faa.gz + GCF_900000615.1_PRJEB7657_assembly_protein.faa.gz.md5 + ... + download_report.json +``` + +Each data file has a `.md5` sidecar containing the hex digest verified against +the FTP server's `md5checksums.txt`. + +> **Without Docker:** You can also run the CLI directly if you have the project +> installed locally: +> +> ```sh +> uv run ncbi_ftp_sync \ +> --manifest output/transfer_manifest.txt \ +> --output-dir staging \ +> --threads 2 --limit 10 +> ``` + +### 3d. Upload staged files to MinIO + +The download step writes to the local filesystem. To feed Phase 3 we need +to upload the staged files into MinIO under a staging prefix: + +```sh +aws --endpoint-url http://localhost:9000 \ + s3 cp staging/raw_data/ s3://cdm-lake/staging/run1/raw_data/ \ + --recursive \ + --no-sign-request +``` + +Verify the upload: + +```sh +aws --endpoint-url http://localhost:9000 \ + s3 ls s3://cdm-lake/staging/run1/ \ + --recursive --no-sign-request | head -20 +``` + +--- + +## 4. Phase 3 — Promote & archive (notebook) + +Open `notebooks/ncbi_ftp_promote.ipynb`. + +### Constants to change (Cell 3) + +| Constant | Walkthrough value | Why | +|---------------------|------------------------------------------------------|---------------------------------------------| +| `BUCKET` | `"cdm-lake"` | matches the bucket created in Step 1 | +| `STAGING_PREFIX` | `"staging/run1/"` | matches the upload prefix from Step 3d | +| `REMOVED_MANIFEST` | `None` | nothing to remove on first run | +| `UPDATED_MANIFEST` | `None` | nothing to archive on first run | +| `NCBI_RELEASE` | `None` | no release tag needed for local testing | +| `MANIFEST_S3_KEY` | `None` | skip manifest trimming | +| `PATH_PREFIX` | `"tenant-general-warehouse/kbase/datasets/ncbi/"` | keep default | +| `DRY_RUN` | `True` | **start with dry-run!** | + +### Initialise the S3 client for MinIO + +The notebook calls `get_s3_client()` which, by default, tries to import +credentials from `berdl_notebook_utils`. For local MinIO you need to +initialise the client manually **before** running Cell 4. Insert a new cell +after Cell 2 (Imports) with: + +```python +from cdm_data_loaders.utils.s3 import get_s3_client, reset_s3_client + +reset_s3_client() # clear any cached client +get_s3_client({ + "endpoint_url": "http://localhost:9000", + "aws_access_key_id": "minioadmin", + "aws_secret_access_key": "minioadmin", +}) +``` + +### Run the notebook + +1. Execute all cells. With `DRY_RUN = True` the promote step will log what it + *would* do without moving any objects. +2. Review the report in Cell 6. +3. If the dry-run looks correct, set `DRY_RUN = False` in Cell 3 and re-run + from Cell 5. + +After promotion the final Lakehouse layout in MinIO will look like: + +``` +cdm-lake/ + tenant-general-warehouse/kbase/datasets/ncbi/ + raw_data/GCF/900/000/615/GCF_900000615.1_.../ + GCF_900000615.1_..._genomic.fna.gz (with md5 in user metadata) + GCF_900000615.1_..._protein.faa.gz + ... +``` + +--- + +## 5. Inspect results in MinIO + +Browse the [MinIO console](http://localhost:9001) or use the CLI: + +```sh +# List final Lakehouse objects +aws --endpoint-url http://localhost:9000 \ + s3 ls s3://cdm-lake/tenant-general-warehouse/kbase/datasets/ncbi/raw_data/ \ + --recursive --no-sign-request | head -20 + +# Check user metadata (md5) on a specific object +aws --endpoint-url http://localhost:9000 \ + s3api head-object \ + --bucket cdm-lake \ + --key "tenant-general-warehouse/kbase/datasets/ncbi/raw_data/GCF/900/000/615/GCF_900000615.1_PRJEB7657_assembly/GCF_900000615.1_PRJEB7657_assembly_genomic.fna.gz" \ + --no-sign-request | jq .Metadata +``` + +--- + +## 6. Incremental run (second sync) + +To exercise the diff/update/archive logic, repeat the pipeline with a +previous snapshot: + +1. **Phase 1:** Set `PREVIOUS_SUMMARY_S3` to an S3 path where you upload the + raw summary from the first run, or save the `raw_summary` string from Cell 4 + to a local file and pass it via `parse_assembly_summary(Path("prev.txt"))`. +2. **Phase 1:** The diff will now show `updated`, `replaced`, and + `suppressed` entries (if any changed between runs). +3. **Phase 2:** Download the new manifest. +4. **Phase 3:** Set `REMOVED_MANIFEST` and `UPDATED_MANIFEST` to the paths + from Phase 1. Updated assemblies will be archived before overwrite; + removed assemblies will be archived and deleted. + +--- + +## 7. Cleanup + +```sh +# Stop and remove MinIO +docker stop minio && docker rm minio + +# Remove local staging data +rm -rf staging/ output/ +``` + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---------|-------|-----| +| `berdl_notebook_utils` import error in notebook | Missing local MinIO client init | Add the `get_s3_client({...})` cell described in Step 4 | +| `connect_ftp() timeout` | NCBI FTP may be slow or rate-limited | Retry; reduce `--threads` to 1 | +| `CRC64NVME` errors uploading to MinIO | MinIO version too old (needs ≥ `2025-02-07`) | Pin to `minio/minio:RELEASE.2025-02-28T09-55-16Z` or newer | +| Phase 3 shows 0 promoted | Staging prefix doesn't match or bucket is wrong | Verify `STAGING_PREFIX` matches the S3 upload path from Step 3d | +| Container can't reach FTP | Docker network isolation | Use `--network host` or ensure DNS resolution works inside the container | + +--- + +## Reference: file filters + +Phase 2 downloads only files matching these suffixes (defined in +`cdm_data_loaders.ncbi_ftp.assembly.FILE_FILTERS`): + +| Suffix | Content | +|--------|---------| +| `_genomic.fna.gz` | Genome nucleotide sequences | +| `_genomic.gff.gz` | Genome annotations (GFF3) | +| `_protein.faa.gz` | Protein sequences | +| `_gene_ontology.gaf.gz` | GO annotations | +| `_assembly_report.txt` | Assembly metadata | +| `_assembly_stats.txt` | Assembly statistics | +| `_assembly_regions.txt` | Assembly regions | +| `_ani_contam_ranges.tsv` | ANI contamination ranges | +| `_gene_expression_counts.txt.gz` | Gene expression counts | +| `_normalized_gene_expression_counts.txt.gz` | Normalised expression counts | + +Plus the per-assembly `md5checksums.txt` which is always downloaded for +integrity verification. diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb index 0cbb5295..3c22c072 100644 --- a/notebooks/ncbi_ftp_manifest.ipynb +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -198,6 +198,39 @@ "else:\n", " print(\"Skipping S3 snapshot upload (SNAPSHOT_UPLOAD_S3 not set)\")" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "112c497a", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Upload manifests to S3 for CTS input staging (optional).\"\"\"\n", + "\n", + "# S3 prefix where CTS will read input files from.\n", + "# Set to None to skip upload (local-only testing).\n", + "STAGING_S3_PREFIX: str | None = None # e.g. \"s3://cdm-lake/staging/run1/\"\n", + "\n", + "if STAGING_S3_PREFIX:\n", + " from cdm_data_loaders.utils.s3 import upload_file_with_metadata\n", + "\n", + " s3 = get_s3_client()\n", + " bucket, prefix = split_s3_path(STAGING_S3_PREFIX)\n", + " prefix = prefix.rstrip(\"/\") + \"/\"\n", + "\n", + " for manifest in [\"transfer_manifest.txt\", \"removed_manifest.txt\",\n", + " \"updated_manifest.txt\", \"diff_summary.json\"]:\n", + " local_path = OUTPUT_DIR / manifest\n", + " if local_path.exists():\n", + " key = f\"{prefix}{manifest}\"\n", + " upload_file_with_metadata(s3, str(local_path), bucket, key)\n", + " print(f\"Uploaded {manifest} -> s3://{bucket}/{key}\")\n", + "\n", + " print(f\"\\nManifests staged for CTS at s3://{bucket}/{prefix}\")\n", + "else:\n", + " print(\"Skipping S3 manifest upload (STAGING_S3_PREFIX not set)\")\n" + ] } ], "metadata": { diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 1d90faa6..33e53519 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -8,12 +8,11 @@ from __future__ import annotations -import functools import hashlib import os import re from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import Any from unittest.mock import patch import boto3 @@ -26,9 +25,6 @@ from cdm_data_loaders.ncbi_ftp.assembly import build_accession_path from cdm_data_loaders.utils.s3 import reset_s3_client -if TYPE_CHECKING: - from collections.abc import Callable - # ── MinIO connection defaults ─────────────────────────────────────────── MINIO_ENDPOINT_URL = os.environ.get("MINIO_ENDPOINT_URL", "http://localhost:9000") @@ -72,20 +68,6 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item item.add_marker(skip_marker) -# ── CRC64NVME workaround ─────────────────────────────────────────────── - - -def strip_checksum_algorithm(method: Callable) -> Callable: - """Wrap a boto3 S3 method to remove ``ChecksumAlgorithm`` if unsupported.""" - - @functools.wraps(method) - def wrapper(*args: object, **kwargs: object) -> object: - kwargs.pop("ChecksumAlgorithm", None) # type: ignore[arg-type] - return method(*args, **kwargs) - - return wrapper - - # ── Fixtures ──────────────────────────────────────────────────────────── @@ -103,10 +85,6 @@ def minio_s3_client() -> botocore.client.BaseClient: aws_secret_access_key=MINIO_SECRET_KEY, ) - # MinIO may not support CRC64NVME — strip to be safe - client.upload_file = strip_checksum_algorithm(client.upload_file) # type: ignore[method-assign] - client.copy_object = strip_checksum_algorithm(client.copy_object) # type: ignore[method-assign] - reset_s3_client() with ( patch.object(s3_utils, "get_s3_client", return_value=client), From 5c6c547e1c58251a78c4d0c518038147286bc068 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Fri, 17 Apr 2026 14:21:03 -0700 Subject: [PATCH 004/128] debug and cleanup --- Dockerfile | 4 +- docs/ncbi_ftp_e2e_walkthrough.md | 185 ++++++++++++------ notebooks/ncbi_ftp_manifest.ipynb | 153 +++++++++++---- notebooks/ncbi_ftp_promote.ipynb | 98 +++++++--- scripts/s3_local.py | 127 ++++++++++++ src/cdm_data_loaders/ncbi_ftp/manifest.py | 41 ++-- src/cdm_data_loaders/ncbi_ftp/promote.py | 73 ++++--- .../pipelines/ncbi_ftp_download.py | 2 +- tests/integration/test_full_pipeline.py | 20 +- tests/integration/test_manifest_e2e.py | 6 +- tests/integration/test_promote_e2e.py | 42 ++-- tests/ncbi_ftp/test_manifest.py | 47 ++++- tests/ncbi_ftp/test_notebooks.py | 4 +- tests/ncbi_ftp/test_promote.py | 34 ++-- tests/pipelines/test_ncbi_ftp_download.py | 6 +- 15 files changed, 612 insertions(+), 230 deletions(-) create mode 100755 scripts/s3_local.py diff --git a/Dockerfile b/Dockerfile index 35a2085c..a218a8bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,7 +54,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ # Place executables in the environment at the front of the path ENV PATH="/app/.venv/bin:$PATH" -COPY --chmod=+x ./scripts/entrypoint.sh /app/ +RUN chmod +x ./scripts/entrypoint.sh # Use the non-root user to run our application USER nonroot -ENTRYPOINT ["./entrypoint.sh"] +ENTRYPOINT ["./scripts/entrypoint.sh"] diff --git a/docs/ncbi_ftp_e2e_walkthrough.md b/docs/ncbi_ftp_e2e_walkthrough.md index 733345be..40b467a7 100644 --- a/docs/ncbi_ftp_e2e_walkthrough.md +++ b/docs/ncbi_ftp_e2e_walkthrough.md @@ -24,15 +24,54 @@ image for the Phase 2 download step. │ previous │ │ ─ write .md5 sidecars │ │ ─ trim manifest │ │ ─ write manifests │ └──────────┬────────────┘ └──────────────────────┘ └────────────────────┘ │ - local volume - mounted into - the container - │ - ▼ - ┌────────────────────┐ - │ MinIO (S3-compat) │ - │ localhost:9000 │ - └────────────────────┘ + local volume + mounted into + the container +``` + +--- + +## Path anatomy + +All S3 paths in this pipeline compose from a small set of variables. +Understanding this decomposition is the key to configuring the notebooks. + +### Path formats used + +| Format | Example | Description | +|--------|---------|-------------| +| **s3:// URI** | `s3://cdm-lake/staging/run1/` | Full URI with scheme + bucket + key | +| **bucket name** | `cdm-lake` | Just the bucket, no scheme | +| **S3 key prefix** | `tenant-general-warehouse/kbase/datasets/ncbi/` | Path within a bucket (no scheme, no bucket) | +| **S3 object key** | `staging/transfer_manifest.txt` | Single object key within a bucket | +| **local path** | `output/removed_manifest.txt` | Filesystem path on the host | + +### Lakehouse object (final location) + +``` +s3://{STORE_BUCKET}/{LAKEHOUSE_KEY_PREFIX}raw_data/{GCF|GCA}/{nnn}/{nnn}/{nnn}/{assembly_dir}/{filename} + └── bucket ──┘ └── key prefix ──────┘└── build_accession_path() ────────────────────────┘ +``` + +Example: +``` +s3://cdm-lake/tenant-general-warehouse/kbase/datasets/ncbi/raw_data/GCF/900/000/615/GCF_900000615.1_PRJEB7657_assembly/GCF_900000615.1_PRJEB7657_assembly_genomic.fna.gz +``` + +### Staging object (Phase 2 output) + +``` +s3://{STORE_BUCKET}/{STAGING_KEY_PREFIX}raw_data/{GCF|GCA}/{nnn}/{nnn}/{nnn}/{assembly_dir}/{filename} + └── bucket ──┘ └── key prefix ────┘└── build_accession_path() ────────────────────────┘ +``` + +### Local output (Phase 1) + +``` +{OUTPUT_DIR}/transfer_manifest.txt +{OUTPUT_DIR}/removed_manifest.txt +{OUTPUT_DIR}/updated_manifest.txt +{OUTPUT_DIR}/diff_summary.json ``` --- @@ -50,17 +89,12 @@ docker run -d \ ``` Create a test bucket via the [MinIO console](http://localhost:9001) -(login: `minioadmin` / `minioadmin`), or from the command line: +(login: `minioadmin` / `minioadmin`), or from the command line using the +included `scripts/s3_local.py` helper (requires no extra installs — only +`boto3` which is already a project dependency): ```sh -# Using the AWS CLI (or `mc` if installed) -aws --endpoint-url http://localhost:9000 \ - s3 mb s3://cdm-lake \ - --no-sign-request 2>/dev/null || true - -# Alternatively, using the MinIO client: -# mc alias set local http://localhost:9000 minioadmin minioadmin -# mc mb local/cdm-lake +uv run python scripts/s3_local.py mb s3://cdm-lake ``` --- @@ -71,15 +105,40 @@ Open `notebooks/ncbi_ftp_manifest.ipynb` in JupyterLab or VS Code. ### Constants to change (Cell 3) -| Constant | Walkthrough value | Why | -|-----------------------|----------------------------------|---------------------------------------------------------| -| `DATABASE` | `"refseq"` | keep as-is | -| `PREFIX_FROM` | `"900"` | high-numbered prefix → few assemblies, fast diffing | -| `PREFIX_TO` | `"900"` | single prefix bucket | -| `LIMIT` | `10` | cap to 10 assemblies | -| `PREVIOUS_SUMMARY_S3` | `None` | first run — everything is "new" | -| `SNAPSHOT_UPLOAD_S3` | `None` | skip S3 upload for local testing | -| `OUTPUT_DIR` | `Path("output")` | keep as-is (local directory) | +| Constant | Walkthrough value | Format | Why | +|-----------------------|----------------------------------|--------|---------------------------------------------------------| +| `DATABASE` | `"refseq"` | string | keep as-is | +| `PREFIX_FROM` | `"900"` | string | high-numbered prefix → few assemblies, fast diffing | +| `PREFIX_TO` | `"900"` | string | single prefix bucket | +| `LIMIT` | `10` | int | cap to 10 assemblies | +| `PREVIOUS_SUMMARY_URI` | `None` | s3:// URI | first run — everything is "new" | +| `SNAPSHOT_UPLOAD_URI` | `None` | s3:// URI | skip S3 upload for local testing | +| `STORE_BUCKET` | `"cdm-lake"` (or `None`) | bucket name | set to prune assemblies already in the Lakehouse | +| `STORE_KEY_PREFIX` | `"tenant-general-warehouse/kbase/datasets/ncbi/"` | S3 key prefix | default Lakehouse path prefix | +| `OUTPUT_DIR` | `Path("output")` | local path | keep as-is (local directory) | + +### Initialise the S3 client for MinIO + +If you set `PREVIOUS_SUMMARY_URI`, `SNAPSHOT_UPLOAD_URI`, `STORE_BUCKET`, +or `STAGING_URI` to point at your local MinIO, you must initialise +the S3 client **before** running the cells that use them. Insert a new cell +after Cell 1 (Imports) with: + +```python +from cdm_data_loaders.utils.s3 import get_s3_client, reset_s3_client + +reset_s3_client() +get_s3_client({ + "endpoint_url": "http://localhost:9000", + "aws_access_key_id": "minioadmin", + "aws_secret_access_key": "minioadmin", +}) +``` + +If all three S3 variables are `None` (purely local testing), this cell can +be skipped — though on repeat runs you should set `STORE_BUCKET` so +assemblies already promoted to the Lakehouse are pruned from the transfer +manifest. ### Run the notebook @@ -104,16 +163,16 @@ Inspect `transfer_manifest.txt` — each line is an FTP directory path like: Cell 7 optionally uploads the manifests to an S3 staging prefix so that CTS can stage them into the container. For local testing, set -`STAGING_S3_PREFIX = None` (the default) and copy the manifest manually in +`STAGING_URI = None` (the default) and copy the manifest manually in Step 3b below. If you are testing against MinIO and want to exercise the S3 upload path: ```python -STAGING_S3_PREFIX = "s3://cdm-lake/staging/run1/" +STAGING_URI = "s3://cdm-lake/staging/run1/" ``` -> **Tip:** If you re-run later with `PREVIOUS_SUMMARY_S3` pointing at a +> **Tip:** If you re-run later with `PREVIOUS_SUMMARY_URI` pointing at a > snapshot from a prior run you will see `updated`, `replaced`, and > `suppressed` entries in the diff. @@ -144,16 +203,17 @@ docker build -t cdm-data-loaders . ### 3b. Prepare local directories ```sh -mkdir -p staging -cp output/transfer_manifest.txt staging/ +mkdir -p notebooks/staging +cp notebooks/output/transfer_manifest.txt notebooks/staging/ ``` ### 3c. Run the download ```sh docker run --rm \ - -v "$(pwd)/staging:/input:ro" \ - -v "$(pwd)/staging:/output" \ + --userns=keep-id \ + -v "$(pwd)/notebooks/staging:/input:ro" \ + -v "$(pwd)/notebooks/staging:/output" \ cdm-data-loaders ncbi_ftp_sync \ --manifest /input/transfer_manifest.txt \ --output-dir /output \ @@ -161,6 +221,10 @@ docker run --rm \ --limit 10 ``` +> **Note:** `--userns=keep-id` maps your host UID into the container so +> bind-mount writes work with Podman's rootless mode. If you use Docker +> instead, replace it with `--user "$(id -u):$(id -g)"`. + | Flag | Purpose | |-----------------|-----------------------------------------------------------| | `--manifest` | Path to the transfer manifest inside the container | @@ -168,7 +232,7 @@ docker run --rm \ | `--threads` | Parallel FTP connections (2 is polite for testing) | | `--limit` | Redundant safety cap (already limited in Phase 1) | -After the container exits, `staging/` will contain: +After the container exits, `notebooks/staging/` will contain: ``` staging/ @@ -189,7 +253,7 @@ the FTP server's `md5checksums.txt`. > > ```sh > uv run ncbi_ftp_sync \ -> --manifest output/transfer_manifest.txt \ +> --manifest notebooks/output/transfer_manifest.txt \ > --output-dir staging \ > --threads 2 --limit 10 > ``` @@ -200,18 +264,13 @@ The download step writes to the local filesystem. To feed Phase 3 we need to upload the staged files into MinIO under a staging prefix: ```sh -aws --endpoint-url http://localhost:9000 \ - s3 cp staging/raw_data/ s3://cdm-lake/staging/run1/raw_data/ \ - --recursive \ - --no-sign-request +uv run python scripts/s3_local.py cp notebooks/staging/raw_data/ s3://cdm-lake/staging/run1/raw_data/ ``` Verify the upload: ```sh -aws --endpoint-url http://localhost:9000 \ - s3 ls s3://cdm-lake/staging/run1/ \ - --recursive --no-sign-request | head -20 +uv run python scripts/s3_local.py ls s3://cdm-lake/staging/run1/ ``` --- @@ -222,16 +281,16 @@ Open `notebooks/ncbi_ftp_promote.ipynb`. ### Constants to change (Cell 3) -| Constant | Walkthrough value | Why | -|---------------------|------------------------------------------------------|---------------------------------------------| -| `BUCKET` | `"cdm-lake"` | matches the bucket created in Step 1 | -| `STAGING_PREFIX` | `"staging/run1/"` | matches the upload prefix from Step 3d | -| `REMOVED_MANIFEST` | `None` | nothing to remove on first run | -| `UPDATED_MANIFEST` | `None` | nothing to archive on first run | -| `NCBI_RELEASE` | `None` | no release tag needed for local testing | -| `MANIFEST_S3_KEY` | `None` | skip manifest trimming | -| `PATH_PREFIX` | `"tenant-general-warehouse/kbase/datasets/ncbi/"` | keep default | -| `DRY_RUN` | `True` | **start with dry-run!** | +| Constant | Walkthrough value | Format | Why | +|-------------------------|------------------------------------------------------|--------|---------------------------------------------| +| `STORE_BUCKET` | `"cdm-lake"` | bucket name | matches the bucket created in Step 1 | +| `STAGING_KEY_PREFIX` | `"staging/run1/"` | S3 key prefix | matches the upload prefix from Step 3d | +| `REMOVED_MANIFEST_PATH` | `None` | local path | nothing to remove on first run | +| `UPDATED_MANIFEST_PATH` | `None` | local path | nothing to archive on first run | +| `NCBI_RELEASE` | `None` | string | no release tag needed for local testing | +| `MANIFEST_S3_KEY` | `None` | S3 object key | skip manifest trimming | +| `LAKEHOUSE_KEY_PREFIX` | `"tenant-general-warehouse/kbase/datasets/ncbi/"` | S3 key prefix | keep default | +| `DRY_RUN` | `True` | bool | **start with dry-run!** | ### Initialise the S3 client for MinIO @@ -257,7 +316,7 @@ get_s3_client({ *would* do without moving any objects. 2. Review the report in Cell 6. 3. If the dry-run looks correct, set `DRY_RUN = False` in Cell 3 and re-run - from Cell 5. + from Cell 3. After promotion the final Lakehouse layout in MinIO will look like: @@ -278,16 +337,12 @@ Browse the [MinIO console](http://localhost:9001) or use the CLI: ```sh # List final Lakehouse objects -aws --endpoint-url http://localhost:9000 \ - s3 ls s3://cdm-lake/tenant-general-warehouse/kbase/datasets/ncbi/raw_data/ \ - --recursive --no-sign-request | head -20 +uv run python scripts/s3_local.py ls \ + s3://cdm-lake/tenant-general-warehouse/kbase/datasets/ncbi/raw_data/ # Check user metadata (md5) on a specific object -aws --endpoint-url http://localhost:9000 \ - s3api head-object \ - --bucket cdm-lake \ - --key "tenant-general-warehouse/kbase/datasets/ncbi/raw_data/GCF/900/000/615/GCF_900000615.1_PRJEB7657_assembly/GCF_900000615.1_PRJEB7657_assembly_genomic.fna.gz" \ - --no-sign-request | jq .Metadata +uv run python scripts/s3_local.py head \ + s3://cdm-lake/tenant-general-warehouse/kbase/datasets/ncbi/raw_data/GCF/900/000/615/GCF_900000615.1_PRJEB7657_assembly/GCF_900000615.1_PRJEB7657_assembly_genomic.fna.gz ``` --- @@ -297,13 +352,13 @@ aws --endpoint-url http://localhost:9000 \ To exercise the diff/update/archive logic, repeat the pipeline with a previous snapshot: -1. **Phase 1:** Set `PREVIOUS_SUMMARY_S3` to an S3 path where you upload the +2. **Phase 1:** Set `PREVIOUS_SUMMARY_URI` to an S3 path where you upload the raw summary from the first run, or save the `raw_summary` string from Cell 4 to a local file and pass it via `parse_assembly_summary(Path("prev.txt"))`. 2. **Phase 1:** The diff will now show `updated`, `replaced`, and `suppressed` entries (if any changed between runs). 3. **Phase 2:** Download the new manifest. -4. **Phase 3:** Set `REMOVED_MANIFEST` and `UPDATED_MANIFEST` to the paths +4. **Phase 3:** Set `REMOVED_MANIFEST_PATH` and `UPDATED_MANIFEST_PATH` to the paths from Phase 1. Updated assemblies will be archived before overwrite; removed assemblies will be archived and deleted. @@ -328,7 +383,7 @@ rm -rf staging/ output/ | `berdl_notebook_utils` import error in notebook | Missing local MinIO client init | Add the `get_s3_client({...})` cell described in Step 4 | | `connect_ftp() timeout` | NCBI FTP may be slow or rate-limited | Retry; reduce `--threads` to 1 | | `CRC64NVME` errors uploading to MinIO | MinIO version too old (needs ≥ `2025-02-07`) | Pin to `minio/minio:RELEASE.2025-02-28T09-55-16Z` or newer | -| Phase 3 shows 0 promoted | Staging prefix doesn't match or bucket is wrong | Verify `STAGING_PREFIX` matches the S3 upload path from Step 3d | +| Phase 3 shows 0 promoted | Staging prefix doesn't match or bucket is wrong | Verify `STAGING_KEY_PREFIX` matches the S3 upload path from Step 3d | | Container can't reach FTP | Docker network isolation | Use `--network host` or ensure DNS resolution works inside the container | --- diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb index 3c22c072..20ee5016 100644 --- a/notebooks/ncbi_ftp_manifest.ipynb +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -15,7 +15,28 @@ "- `diff_summary.json` — human-readable summary of changes\n", "\n", "All filtering (prefix range, limit) is applied here so downstream phases\n", - "receive a final, pre-filtered manifest." + "receive a final, pre-filtered manifest.\n", + "\n", + "Optionally verifies candidates against the S3 Lakehouse (`STORE_BUCKET`) so\n", + "assemblies that were already downloaded and promoted are pruned from the\n", + "transfer manifest." + ] + }, + { + "cell_type": "markdown", + "id": "d0d3063c", + "metadata": {}, + "source": [ + "## Path formats quick reference\n", + "\n", + "| Suffix in variable name | Format | Example |\n", + "|-------------------------|--------|---------|\n", + "| `_URI` | `s3://bucket/key/…` | `s3://cdm-lake/staging/run1/` |\n", + "| `_BUCKET` | bucket name only | `cdm-lake` |\n", + "| `_KEY_PREFIX` | S3 key prefix (no scheme/bucket) | `tenant-general-warehouse/kbase/datasets/ncbi/` |\n", + "| `_DIR` / `_PATH` | local filesystem path | `output/removed_manifest.txt` |\n", + "\n", + "Lakehouse object: `s3://{STORE_BUCKET}/{STORE_KEY_PREFIX}raw_data/…/{filename}`" ] }, { @@ -60,28 +81,37 @@ "DATABASE = \"refseq\"\n", "\n", "# Accession prefix filtering (3-digit, inclusive). Set to None to skip.\n", - "PREFIX_FROM: str | None = None # e.g. \"000\"\n", - "PREFIX_TO: str | None = None # e.g. \"003\"\n", + "PREFIX_FROM: str | None = \"900\" # e.g. \"000\"\n", + "PREFIX_TO: str | None = \"900\" # e.g. \"003\"\n", "\n", "# Maximum number of new/updated assemblies to include (None = unlimited)\n", - "LIMIT: int | None = None\n", + "LIMIT: int | None = 10\n", "\n", - "# S3 path to the previous assembly summary snapshot (set to None on first run)\n", - "PREVIOUS_SUMMARY_S3: str | None = None # e.g. \"s3://cdm-lake/tenant-general-warehouse/kbase/datasets/ncbi/metadata/assembly_summary_refseq_prev.txt\"\n", + "# Previous assembly summary snapshot\n", + "# format: s3:// URI (e.g. \"s3://cdm-lake/.../assembly_summary_refseq_prev.txt\")\n", + "PREVIOUS_SUMMARY_URI: str | None = None\n", "\n", - "# S3 path where the new snapshot will be uploaded after diffing\n", - "SNAPSHOT_UPLOAD_S3: str | None = None # e.g. \"s3://cdm-lake/tenant-general-warehouse/kbase/datasets/ncbi/metadata/assembly_summary_refseq_curr.txt\"\n", + "# S3 location where the new snapshot will be uploaded after diffing\n", + "# format: s3:// URI\n", + "SNAPSHOT_UPLOAD_URI: str | None = None\n", + "\n", + "# Verify candidates against the S3 Lakehouse — prune assemblies already present.\n", + "# Set STORE_BUCKET to your bucket name to enable, or None to skip.\n", + "# STORE_KEY_PREFIX should point to the directory containing `raw_data/`.\n", + "# format: bucket name (no s3:// scheme)\n", + "STORE_BUCKET: str | None = \"cdm-lake\"\n", + "# format: S3 key prefix within STORE_BUCKET (no scheme, no bucket)\n", + "STORE_KEY_PREFIX = \"tenant-general-warehouse/kbase/datasets/ncbi/\"\n", "\n", "# Local output directory for manifest files\n", + "# format: local directory path\n", "OUTPUT_DIR = Path(\"output\")\n", "OUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n", "\n", - "# FTP hostname (default is the standard NCBI FTP server)\n", - "FTP_HOSTNAME = FTP_HOST\n", - "\n", "print(f\"Database: {DATABASE}\")\n", "print(f\"Prefix range: {PREFIX_FROM} -> {PREFIX_TO}\")\n", "print(f\"Limit: {LIMIT}\")\n", + "print(f\"Verify against S3: {STORE_BUCKET or 'disabled'}\")\n", "print(f\"Output dir: {OUTPUT_DIR}\")" ] }, @@ -94,7 +124,7 @@ "source": [ "\"\"\"Download current assembly summary from NCBI FTP.\"\"\"\n", "\n", - "raw_summary = download_assembly_summary(database=DATABASE, ftp_host=FTP_HOSTNAME)\n", + "raw_summary = download_assembly_summary(database=DATABASE, ftp_host=FTP_HOST)\n", "current = parse_assembly_summary(raw_summary)\n", "print(f\"Parsed {len(current)} assemblies from current {DATABASE} summary\")" ] @@ -110,9 +140,9 @@ "\n", "previous: dict[str, AssemblyRecord] | None = None\n", "\n", - "if PREVIOUS_SUMMARY_S3:\n", + "if PREVIOUS_SUMMARY_URI:\n", " s3 = get_s3_client()\n", - " bucket, key = split_s3_path(PREVIOUS_SUMMARY_S3)\n", + " bucket, key = split_s3_path(PREVIOUS_SUMMARY_URI)\n", " resp = s3.get_object(Bucket=bucket, Key=key)\n", " prev_text = resp[\"Body\"].read().decode(\"utf-8\")\n", " previous = parse_assembly_summary(prev_text)\n", @@ -128,7 +158,7 @@ "metadata": {}, "outputs": [], "source": [ - "\"\"\"Compute diff and apply filters.\"\"\"\n", + "\"\"\"Compute diff and apply prefix filter.\"\"\"\n", "\n", "# Filter current assemblies by prefix range\n", "filtered = filter_by_prefix_range(current, prefix_from=PREFIX_FROM, prefix_to=PREFIX_TO)\n", @@ -145,16 +175,52 @@ "print(f\"Replaced: {len(diff.replaced)}\")\n", "print(f\"Suppressed: {len(diff.suppressed)}\")\n", "print(f\"Total to transfer: {len(diff.new) + len(diff.updated)}\")\n", - "print(f\"Total to remove: {len(diff.replaced) + len(diff.suppressed)}\")\n", + "print(f\"Total to remove: {len(diff.replaced) + len(diff.suppressed)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "91ad314a", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Verify candidates against S3 Lakehouse, then apply LIMIT.\n", + "\n", + "Verification (optional): for each candidate, fetch md5checksums.txt from\n", + "NCBI FTP and compare against md5 metadata on existing S3 objects.\n", + "Assemblies already present with matching checksums are pruned.\n", + "\n", + "LIMIT is applied *after* verification so the cap counts only assemblies\n", + "that genuinely need downloading.\n", + "\"\"\"\n", + "\n", + "# ── Verify against Lakehouse ──\n", + "if STORE_BUCKET:\n", + " from cdm_data_loaders.ncbi_ftp.manifest import verify_transfer_candidates\n", + "\n", + " candidates = diff.new + diff.updated\n", + " print(f\"Verifying {len(candidates)} candidates against s3://{STORE_BUCKET}/{STORE_KEY_PREFIX} ...\")\n", + " confirmed = set(verify_transfer_candidates(\n", + " candidates, filtered, STORE_BUCKET, STORE_KEY_PREFIX, ftp_host=FTP_HOST,\n", + " ))\n", + " before = len(diff.new) + len(diff.updated)\n", + " diff.new = [a for a in diff.new if a in confirmed]\n", + " diff.updated = [a for a in diff.updated if a in confirmed]\n", + " after = len(diff.new) + len(diff.updated)\n", + " print(f\"Verified: {after} need downloading, {before - after} pruned (already in store)\")\n", + "else:\n", + " print(\"Skipping S3 verification (STORE_BUCKET not set)\")\n", "\n", - "# Apply limit if set\n", + "# ── Apply LIMIT ──\n", "if LIMIT is not None:\n", " original_new = len(diff.new)\n", " original_updated = len(diff.updated)\n", " combined = diff.new + diff.updated\n", " limited = combined[:LIMIT]\n", - " diff.new = [a for a in diff.new if a in set(limited)]\n", - " diff.updated = [a for a in diff.updated if a in set(limited)]\n", + " limited_set = set(limited)\n", + " diff.new = [a for a in diff.new if a in limited_set]\n", + " diff.updated = [a for a in diff.updated if a in limited_set]\n", " print(f\"After limit ({LIMIT}): {len(diff.new)} new, {len(diff.updated)} updated\")\n", " print(f\" (was {original_new} new, {original_updated} updated)\")" ] @@ -170,7 +236,7 @@ "\n", "# Write transfer manifest\n", "transfer_path = OUTPUT_DIR / \"transfer_manifest.txt\"\n", - "paths = write_transfer_manifest(diff, filtered, transfer_path, ftp_host=FTP_HOSTNAME)\n", + "paths = write_transfer_manifest(diff, filtered, transfer_path, ftp_host=FTP_HOST)\n", "print(f\"Transfer manifest: {len(paths)} entries -> {transfer_path}\")\n", "\n", "# Write removed manifest\n", @@ -190,13 +256,13 @@ "print(json.dumps(summary[\"counts\"], indent=2))\n", "\n", "# Upload new snapshot to S3 for future diffing\n", - "if SNAPSHOT_UPLOAD_S3:\n", + "if SNAPSHOT_UPLOAD_URI:\n", " s3 = get_s3_client()\n", - " bucket, key = split_s3_path(SNAPSHOT_UPLOAD_S3)\n", + " bucket, key = split_s3_path(SNAPSHOT_UPLOAD_URI)\n", " s3.put_object(Bucket=bucket, Key=key, Body=raw_summary.encode(\"utf-8\"))\n", - " print(f\"Uploaded new snapshot to {SNAPSHOT_UPLOAD_S3}\")\n", + " print(f\"Uploaded new snapshot to {SNAPSHOT_UPLOAD_URI}\")\n", "else:\n", - " print(\"Skipping S3 snapshot upload (SNAPSHOT_UPLOAD_S3 not set)\")" + " print(\"Skipping S3 snapshot upload (SNAPSHOT_UPLOAD_URI not set)\")" ] }, { @@ -206,17 +272,22 @@ "metadata": {}, "outputs": [], "source": [ - "\"\"\"Upload manifests to S3 for CTS input staging (optional).\"\"\"\n", + "\"\"\"Upload manifests to S3 for CTS input staging (optional).\n", "\n", - "# S3 prefix where CTS will read input files from.\n", - "# Set to None to skip upload (local-only testing).\n", - "STAGING_S3_PREFIX: str | None = None # e.g. \"s3://cdm-lake/staging/run1/\"\n", + "Note: STAGING_URI is a full s3:// URI. The promote notebook splits this into\n", + "STORE_BUCKET + STAGING_KEY_PREFIX (separate bucket and key prefix parameters).\n", "\n", - "if STAGING_S3_PREFIX:\n", - " from cdm_data_loaders.utils.s3 import upload_file_with_metadata\n", + "This is for local testing. The CTS will stage the container's input folder in production.\n", + "\"\"\"\n", "\n", + "# S3 location where CTS will read input files from.\n", + "# Set to None to skip upload (local-only testing).\n", + "# format: s3:// URI (e.g. \"s3://cdm-lake/staging/run1/\")\n", + "STAGING_URI: str | None = \"s3://cdm-lake/staging/run1/input/\"\n", + "\n", + "if STAGING_URI:\n", " s3 = get_s3_client()\n", - " bucket, prefix = split_s3_path(STAGING_S3_PREFIX)\n", + " bucket, prefix = split_s3_path(STAGING_URI)\n", " prefix = prefix.rstrip(\"/\") + \"/\"\n", "\n", " for manifest in [\"transfer_manifest.txt\", \"removed_manifest.txt\",\n", @@ -224,18 +295,32 @@ " local_path = OUTPUT_DIR / manifest\n", " if local_path.exists():\n", " key = f\"{prefix}{manifest}\"\n", - " upload_file_with_metadata(s3, str(local_path), bucket, key)\n", + " s3.upload_file(Filename=str(local_path), Bucket=bucket, Key=key)\n", " print(f\"Uploaded {manifest} -> s3://{bucket}/{key}\")\n", "\n", " print(f\"\\nManifests staged for CTS at s3://{bucket}/{prefix}\")\n", "else:\n", - " print(\"Skipping S3 manifest upload (STAGING_S3_PREFIX not set)\")\n" + " print(\"Skipping S3 manifest upload (STAGING_URI not set)\")" ] } ], "metadata": { + "kernelspec": { + "display_name": "cdm-data-loaders (3.13.11)", + "language": "python", + "name": "python3" + }, "language_info": { - "name": "python" + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.11" } }, "nbformat": 4, diff --git a/notebooks/ncbi_ftp_promote.ipynb b/notebooks/ncbi_ftp_promote.ipynb index de19c0e6..5dd46312 100644 --- a/notebooks/ncbi_ftp_promote.ipynb +++ b/notebooks/ncbi_ftp_promote.ipynb @@ -20,6 +20,24 @@ "6. Trim manifest (remove promoted entries)" ] }, + { + "cell_type": "markdown", + "id": "2f98c43e", + "metadata": {}, + "source": [ + "## Path formats quick reference\n", + "\n", + "| Suffix in variable name | Format | Example |\n", + "|-------------------------|--------|---------|\n", + "| `_BUCKET` | bucket name only | `cdm-lake` |\n", + "| `_KEY_PREFIX` | S3 key prefix (no scheme/bucket) | `staging/run1/` |\n", + "| `_S3_KEY` | S3 object key (no scheme/bucket) | `staging/transfer_manifest.txt` |\n", + "| `_PATH` | local filesystem path | `output/removed_manifest.txt` |\n", + "\n", + "Lakehouse object: `s3://{STORE_BUCKET}/{LAKEHOUSE_KEY_PREFIX}raw_data/…/{filename}`\n", + "Staging object: `s3://{STORE_BUCKET}/{STAGING_KEY_PREFIX}raw_data/…/{filename}`" + ] + }, { "cell_type": "code", "execution_count": null, @@ -34,7 +52,7 @@ "import json\n", "\n", "from cdm_data_loaders.ncbi_ftp.promote import (\n", - " DEFAULT_PATH_PREFIX,\n", + " DEFAULT_LAKEHOUSE_KEY_PREFIX,\n", " promote_from_s3,\n", ")\n", "from cdm_data_loaders.utils.s3 import get_s3_client" @@ -47,39 +65,51 @@ "metadata": {}, "outputs": [], "source": [ - "\"\"\"Configure parameters.\"\"\"\n", + "\"\"\"Configure parameters.\n", + "\n", + "Path layout (how variables compose into a full S3 object path):\n", + " s3://{STORE_BUCKET}/{LAKEHOUSE_KEY_PREFIX}raw_data/{GCF|GCA}/{nnn}/{nnn}/{nnn}/{assembly_dir}/{file}\n", + " s3://{STORE_BUCKET}/{STAGING_KEY_PREFIX}raw_data/{assembly_dir}/{file}\n", + "\"\"\"\n", "\n", "# S3 bucket where staged files and final Lakehouse data live\n", - "BUCKET = \"cdm-lake\" # e.g. \"cdm-lake\"\n", + "# format: bucket name (no s3:// scheme)\n", + "STORE_BUCKET = \"cdm-lake\"\n", "\n", - "# Staging prefix written by CTS Phase 2 (from the CTS output mount)\n", - "STAGING_PREFIX = \"staging/run1/\" # e.g. \"staging/run1/\"\n", + "# Staging prefix written by CTS Phase 2\n", + "# format: S3 key prefix within STORE_BUCKET (no scheme, no bucket)\n", + "STAGING_KEY_PREFIX = \"staging/run1/\"\n", "\n", "# Local path to removed_manifest.txt from Phase 1 (or None to skip archiving)\n", - "REMOVED_MANIFEST: str | None = None # e.g. \"output/removed_manifest.txt\"\n", + "# format: local file path\n", + "REMOVED_MANIFEST_PATH: str | None = None # e.g. \"output/removed_manifest.txt\"\n", "\n", "# Local path to updated_manifest.txt from Phase 1 (or None to skip pre-overwrite archiving)\n", - "UPDATED_MANIFEST: str | None = None # e.g. \"output/updated_manifest.txt\"\n", + "# format: local file path\n", + "UPDATED_MANIFEST_PATH: str | None = None # e.g. \"output/updated_manifest.txt\"\n", "\n", "# NCBI release tag for archive metadata (e.g. \"2024-01\")\n", "NCBI_RELEASE: str | None = None\n", "\n", - "# S3 key of transfer_manifest.txt for trimming (or None to skip)\n", - "MANIFEST_S3_KEY: str | None = None # e.g. \"ncbi/transfer_manifest.txt\"\n", + "# S3 key of transfer_manifest.txt for trimming after promotion (or None to skip).\n", + "# Only needed if the manifest was uploaded to S3 (e.g. via the staging cell in Phase 1).\n", + "# format: S3 object key within STORE_BUCKET (no scheme, no bucket)\n", + "MANIFEST_S3_KEY: str | None = \"staging/run1/input/transfer_manifest.txt\" # e.g. \"staging/transfer_manifest.txt\"\n", "\n", "# Final Lakehouse path prefix\n", - "PATH_PREFIX = DEFAULT_PATH_PREFIX\n", + "# format: S3 key prefix within STORE_BUCKET (no scheme, no bucket)\n", + "LAKEHOUSE_KEY_PREFIX = DEFAULT_LAKEHOUSE_KEY_PREFIX\n", "\n", "# Dry-run mode — log actions without making changes\n", - "DRY_RUN = True\n", - "\n", - "print(f\"Updated manifest: {UPDATED_MANIFEST}\")\n", - "print(f\"NCBI release: {NCBI_RELEASE}\")\n", - "print(f\"Manifest S3 key: {MANIFEST_S3_KEY}\")\n", - "print(f\"Path prefix: {PATH_PREFIX}\")\n", - "\n", - "print(f\"Dry-run: {DRY_RUN}\")\n", - "print(f\"Path prefix: {PATH_PREFIX}\")" + "DRY_RUN = False\n", + "\n", + "print(f\"Bucket: {STORE_BUCKET}\")\n", + "print(f\"Staging key prefix: {STAGING_KEY_PREFIX}\")\n", + "print(f\"Updated manifest: {UPDATED_MANIFEST_PATH}\")\n", + "print(f\"NCBI release: {NCBI_RELEASE}\")\n", + "print(f\"Manifest S3 key: {MANIFEST_S3_KEY}\")\n", + "print(f\"Lakehouse prefix: {LAKEHOUSE_KEY_PREFIX}\")\n", + "print(f\"Dry-run: {DRY_RUN}\")" ] }, { @@ -95,7 +125,7 @@ "paginator = s3.get_paginator(\"list_objects_v2\")\n", "\n", "staged: list[str] = []\n", - "for page in paginator.paginate(Bucket=BUCKET, Prefix=STAGING_PREFIX):\n", + "for page in paginator.paginate(Bucket=STORE_BUCKET, Prefix=STAGING_KEY_PREFIX):\n", " staged.extend(obj[\"Key\"] for obj in page.get(\"Contents\", []))\n", "\n", "sidecars = [k for k in staged if k.endswith((\".md5\", \".crc64nvme\"))]\n", @@ -123,13 +153,13 @@ "\"\"\"Promote staged files to final Lakehouse paths.\"\"\"\n", "\n", "report = promote_from_s3(\n", - " staging_prefix=STAGING_PREFIX,\n", - " bucket=BUCKET,\n", - " removed_manifest=REMOVED_MANIFEST,\n", - " updated_manifest=UPDATED_MANIFEST,\n", + " staging_key_prefix=STAGING_KEY_PREFIX,\n", + " bucket=STORE_BUCKET,\n", + " removed_manifest_path=REMOVED_MANIFEST_PATH,\n", + " updated_manifest_path=UPDATED_MANIFEST_PATH,\n", " ncbi_release=NCBI_RELEASE,\n", - " manifest_path=MANIFEST_S3_KEY,\n", - " path_prefix=PATH_PREFIX,\n", + " manifest_s3_key=MANIFEST_S3_KEY,\n", + " lakehouse_key_prefix=LAKEHOUSE_KEY_PREFIX,\n", " dry_run=DRY_RUN,\n", ")\n", "\n", @@ -163,8 +193,22 @@ } ], "metadata": { + "kernelspec": { + "display_name": "cdm-data-loaders (3.13.11)", + "language": "python", + "name": "python3" + }, "language_info": { - "name": "python" + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.11" } }, "nbformat": 4, diff --git a/scripts/s3_local.py b/scripts/s3_local.py new file mode 100755 index 00000000..65f80396 --- /dev/null +++ b/scripts/s3_local.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +# ruff: noqa: T201, EM101, EM102, TRY003, D103 +"""Thin S3 CLI for local MinIO testing (no aws-cli install required). + +Usage (all commands assume ``uv run`` from the repo root): + + uv run python scripts/s3_local.py mb s3://cdm-lake + uv run python scripts/s3_local.py cp staging/raw_data/ s3://cdm-lake/staging/run1/raw_data/ + uv run python scripts/s3_local.py ls s3://cdm-lake/staging/run1/ + uv run python scripts/s3_local.py head s3://cdm-lake/some/key.gz + +Environment variables (with defaults for the walkthrough): + + MINIO_ENDPOINT_URL http://localhost:9000 + MINIO_ACCESS_KEY minioadmin + MINIO_SECRET_KEY minioadmin +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +import boto3 + + +def _client() -> boto3.client: + return boto3.client( + "s3", + endpoint_url=os.environ.get("MINIO_ENDPOINT_URL", "http://localhost:9000"), + aws_access_key_id=os.environ.get("MINIO_ACCESS_KEY", "minioadmin"), + aws_secret_access_key=os.environ.get("MINIO_SECRET_KEY", "minioadmin"), + ) + + +def _split(uri: str) -> tuple[str, str]: + """Split ``s3://bucket/key`` into ``(bucket, key)``.""" + if not uri.startswith("s3://"): + raise SystemExit(f"Expected s3:// URI, got: {uri}") + parts = uri[5:].split("/", 1) + return parts[0], parts[1] if len(parts) > 1 else "" + + +# ── subcommands ───────────────────────────────────────────────────────── + + +def cmd_mb(args: list[str]) -> None: + """Create a bucket: ``mb s3://bucket``.""" + if not args: + raise SystemExit("Usage: s3_local.py mb s3://BUCKET") + bucket, _ = _split(args[0]) + s3 = _client() + try: + s3.head_bucket(Bucket=bucket) + print(f"Bucket already exists: {bucket}") + except Exception: # noqa: BLE001 + s3.create_bucket(Bucket=bucket) + print(f"Created bucket: {bucket}") + + +def cmd_cp(args: list[str]) -> None: + """Recursive upload: ``cp LOCAL_DIR s3://bucket/prefix/``.""" + if len(args) < 2: # noqa: PLR2004 + raise SystemExit("Usage: s3_local.py cp LOCAL_DIR s3://BUCKET/PREFIX/") + local_dir = Path(args[0]) + bucket, prefix = _split(args[1]) + prefix = prefix.rstrip("/") + "/" if prefix else "" + s3 = _client() + count = 0 + for path in sorted(local_dir.rglob("*")): + if path.is_dir(): + continue + rel = path.relative_to(local_dir) + key = f"{prefix}{rel}" + s3.upload_file(Filename=str(path), Bucket=bucket, Key=key) + count += 1 + print(f" {key}") + print(f"Uploaded {count} files to s3://{bucket}/{prefix}") + + +def cmd_ls(args: list[str]) -> None: + """List objects: ``ls s3://bucket/prefix/ [--limit N]``.""" + if not args: + raise SystemExit("Usage: s3_local.py ls s3://BUCKET/PREFIX/ [--limit N]") + bucket, prefix = _split(args[0]) + limit = 20 + if "--limit" in args: + idx = args.index("--limit") + limit = int(args[idx + 1]) + s3 = _client() + paginator = s3.get_paginator("list_objects_v2") + shown = 0 + for page in paginator.paginate(Bucket=bucket, Prefix=prefix): + for obj in page.get("Contents", []): + print(f" {obj['Size']:>10} {obj['Key']}") + shown += 1 + if shown >= limit: + return + + +def cmd_head(args: list[str]) -> None: + """Show metadata: ``head s3://bucket/key``.""" + if not args: + raise SystemExit("Usage: s3_local.py head s3://BUCKET/KEY") + bucket, key = _split(args[0]) + s3 = _client() + resp = s3.head_object(Bucket=bucket, Key=key) + meta = resp.get("Metadata", {}) + print(json.dumps(meta, indent=2)) + + +# ── dispatch ──────────────────────────────────────────────────────────── + +COMMANDS = {"mb": cmd_mb, "cp": cmd_cp, "ls": cmd_ls, "head": cmd_head} + + +def main() -> None: + if len(sys.argv) < 2 or sys.argv[1] not in COMMANDS: # noqa: PLR2004 + cmds = ", ".join(COMMANDS) + raise SystemExit(f"Usage: s3_local.py <{cmds}> [args ...]\n\n{__doc__}") + COMMANDS[sys.argv[1]](sys.argv[2:]) + + +if __name__ == "__main__": + main() diff --git a/src/cdm_data_loaders/ncbi_ftp/manifest.py b/src/cdm_data_loaders/ncbi_ftp/manifest.py index 21aa084d..e67c2185 100644 --- a/src/cdm_data_loaders/ncbi_ftp/manifest.py +++ b/src/cdm_data_loaders/ncbi_ftp/manifest.py @@ -26,7 +26,7 @@ ) from cdm_data_loaders.utils.cdm_logger import get_cdm_logger from cdm_data_loaders.utils.ftp_client import connect_ftp, ftp_noop_keepalive, ftp_retrieve_text -from cdm_data_loaders.utils.s3 import head_object +from cdm_data_loaders.utils.s3 import get_s3_client, head_object logger = get_cdm_logger() @@ -264,7 +264,7 @@ def verify_transfer_candidates( accessions: list[str], current_assemblies: dict[str, AssemblyRecord], bucket: str, - path_prefix: str, + key_prefix: str, ftp_host: str = FTP_HOST, ) -> list[str]: """Verify which transfer candidates actually need downloading. @@ -280,16 +280,18 @@ def verify_transfer_candidates( :param accessions: list of candidate accessions (new + updated from diff) :param current_assemblies: parsed current assembly summary :param bucket: S3 bucket name - :param path_prefix: Lakehouse path prefix (e.g. ``"tenant-general-warehouse/kbase/datasets/ncbi/"``) + :param key_prefix: S3 key prefix for the Lakehouse dataset root :param ftp_host: NCBI FTP hostname :return: filtered list of accessions that actually need downloading """ if not accessions: return [] - ftp = connect_ftp(ftp_host) + s3 = get_s3_client() + ftp: Any = None # lazily connected only when needed confirmed: list[str] = [] pruned = 0 + skipped_missing = 0 last_activity = time.monotonic() try: @@ -299,10 +301,24 @@ def verify_transfer_candidates( confirmed.append(acc) continue - # Keep FTP alive between assemblies + # Build S3 prefix for this assembly + s3_rel = build_accession_path(rec.assembly_dir) + s3_prefix = f"{key_prefix}{s3_rel}" + + # Quick check: does *anything* exist under this prefix? + resp = s3.list_objects_v2(Bucket=bucket, Prefix=s3_prefix, MaxKeys=1) + if resp.get("KeyCount", 0) == 0: + # Nothing in the store — definitely needs downloading + confirmed.append(acc) + skipped_missing += 1 + continue + + # Objects exist — need FTP md5 checksums to decide + if ftp is None: + ftp = connect_ftp(ftp_host) + last_activity = ftp_noop_keepalive(ftp, last_activity) - # Download md5checksums.txt from FTP ftp_dir = _ftp_dir_from_url(rec.ftp_path, ftp_host) try: md5_text = ftp_retrieve_text(ftp, ftp_dir.rstrip("/") + "/md5checksums.txt") @@ -324,13 +340,10 @@ def verify_transfer_candidates( confirmed.append(acc) continue - # Build S3 prefix for this assembly - s3_rel = build_accession_path(rec.assembly_dir) - # Short-circuit: if any file differs or is missing, keep the assembly needs_update = False for fname, expected_md5 in target_checksums.items(): - s3_path = f"{bucket}/{path_prefix}{s3_rel}{fname}" + s3_path = f"{bucket}/{s3_prefix}{fname}" obj_info = head_object(s3_path) if obj_info is None: @@ -349,12 +362,14 @@ def verify_transfer_candidates( pruned += 1 logger.debug("Pruned %s — all files match S3 checksums", acc) finally: - with contextlib.suppress(Exception): - ftp.quit() + if ftp is not None: + with contextlib.suppress(Exception): + ftp.quit() logger.info( - "Checksum verification: %d confirmed, %d pruned (of %d candidates)", + "Checksum verification: %d confirmed (%d missing from store), %d pruned (of %d candidates)", len(confirmed), + skipped_missing, pruned, len(accessions), ) diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index 37f1fa82..52d9e57f 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -12,6 +12,8 @@ from pathlib import Path from typing import Any +import botocore.exceptions + from cdm_data_loaders.utils.cdm_logger import get_cdm_logger from cdm_data_loaders.utils.s3 import ( copy_object_with_metadata, @@ -22,20 +24,20 @@ logger = get_cdm_logger() -DEFAULT_PATH_PREFIX = "tenant-general-warehouse/kbase/datasets/ncbi/" +DEFAULT_LAKEHOUSE_KEY_PREFIX = "tenant-general-warehouse/kbase/datasets/ncbi/" # ── Promote from S3 staging prefix ────────────────────────────────────── def promote_from_s3( # noqa: PLR0913 - staging_prefix: str, + staging_key_prefix: str, bucket: str, - removed_manifest: str | Path | None = None, - updated_manifest: str | Path | None = None, + removed_manifest_path: str | Path | None = None, + updated_manifest_path: str | Path | None = None, ncbi_release: str | None = None, - manifest_path: str | None = None, - path_prefix: str = DEFAULT_PATH_PREFIX, + manifest_s3_key: str | None = None, + lakehouse_key_prefix: str = DEFAULT_LAKEHOUSE_KEY_PREFIX, *, dry_run: bool = False, ) -> dict[str, Any]: @@ -44,13 +46,13 @@ def promote_from_s3( # noqa: PLR0913 Downloads each file to a temp location and re-uploads to the final path with MD5 metadata from ``.md5`` sidecar files. - :param staging_prefix: S3 key prefix where CTS output was written + :param staging_key_prefix: S3 key prefix where CTS output was written :param bucket: S3 bucket name - :param removed_manifest: local path to the removed_manifest file - :param updated_manifest: local path to the updated_manifest file + :param removed_manifest_path: local path to the removed_manifest file + :param updated_manifest_path: local path to the updated_manifest file :param ncbi_release: NCBI release version tag for archiving - :param manifest_path: S3 path to transfer_manifest.txt for trimming - :param path_prefix: Lakehouse path prefix for final locations + :param manifest_s3_key: S3 object key for transfer_manifest.txt (for trimming) + :param lakehouse_key_prefix: S3 key prefix for final Lakehouse locations :param dry_run: if True, log actions without side effects :return: report dict with counts """ @@ -62,7 +64,7 @@ def promote_from_s3( # noqa: PLR0913 # Collect all objects under the staging prefix staged_objects: list[str] = [] - for page in paginator.paginate(Bucket=bucket, Prefix=staging_prefix): + for page in paginator.paginate(Bucket=bucket, Prefix=staging_key_prefix): staged_objects.extend(obj["Key"] for obj in page.get("Contents", [])) # Separate data files from sidecars @@ -74,15 +76,15 @@ def promote_from_s3( # noqa: PLR0913 # Archive all affected assemblies BEFORE promoting or deleting archived = 0 for manifest_file, reason, delete in [ - (updated_manifest, "updated", False), - (removed_manifest, "replaced_or_suppressed", True), + (updated_manifest_path, "updated", False), + (removed_manifest_path, "replaced_or_suppressed", True), ]: if manifest_file and Path(str(manifest_file)).is_file(): archived += _archive_assemblies( str(manifest_file), bucket=bucket, ncbi_release=ncbi_release, - path_prefix=path_prefix, + lakehouse_key_prefix=lakehouse_key_prefix, archive_reason=reason, delete_source=delete, dry_run=dry_run, @@ -94,10 +96,10 @@ def promote_from_s3( # noqa: PLR0913 if staged_key.endswith("download_report.json"): continue - rel_path = staged_key[len(staging_prefix) :] + rel_path = staged_key[len(staging_key_prefix) :] if not rel_path.startswith("raw_data/"): continue - final_key = path_prefix + rel_path + final_key = lakehouse_key_prefix + rel_path if dry_run: logger.info("[dry-run] would promote: %s -> %s", staged_key, final_key) @@ -137,8 +139,8 @@ def promote_from_s3( # noqa: PLR0913 failed += 1 # Trim manifest for resumability - if manifest_path and promoted_accessions and not dry_run: - _trim_manifest(manifest_path, bucket, promoted_accessions) + if manifest_s3_key and promoted_accessions and not dry_run: + _trim_manifest(manifest_s3_key, bucket, promoted_accessions) report: dict[str, Any] = { "timestamp": datetime.now(UTC).isoformat(), @@ -162,10 +164,10 @@ def promote_from_s3( # noqa: PLR0913 def _archive_assemblies( # noqa: PLR0913 - manifest_path: str, + manifest_local_path: str, bucket: str, ncbi_release: str | None = None, - path_prefix: str = DEFAULT_PATH_PREFIX, + lakehouse_key_prefix: str = DEFAULT_LAKEHOUSE_KEY_PREFIX, archive_reason: str = "unknown", *, delete_source: bool = False, @@ -178,10 +180,10 @@ def _archive_assemblies( # noqa: PLR0913 objects are deleted after copying. When False (updated), the originals remain in place to be overwritten by the promote step. - :param manifest_path: local path to a manifest file (one accession per line) + :param manifest_local_path: local path to a manifest file (one accession per line) :param bucket: S3 bucket name :param ncbi_release: release tag used in the archive path - :param path_prefix: Lakehouse path prefix + :param lakehouse_key_prefix: S3 key prefix for the Lakehouse dataset root :param archive_reason: metadata value describing why the object was archived :param delete_source: if True, delete the source object after copying :param dry_run: if True, log without making changes @@ -192,7 +194,7 @@ def _archive_assemblies( # noqa: PLR0913 datestamp = datetime.now(UTC).strftime("%Y-%m-%d") archived = 0 - with Path(manifest_path).open() as f: + with Path(manifest_local_path).open() as f: accessions = [line.strip() for line in f if line.strip()] for accession in accessions: @@ -203,7 +205,7 @@ def _archive_assemblies( # noqa: PLR0913 db = m.group(1) p1, p2, p3 = m.group(2), m.group(3), m.group(4) - source_prefix = f"{path_prefix}raw_data/{db}/{p1}/{p2}/{p3}/" + source_prefix = f"{lakehouse_key_prefix}raw_data/{db}/{p1}/{p2}/{p3}/" paginator = s3.get_paginator("list_objects_v2") matching_keys: list[str] = [] @@ -215,8 +217,8 @@ def _archive_assemblies( # noqa: PLR0913 continue for source_key in matching_keys: - rel = source_key[len(path_prefix) :] - archive_key = f"{path_prefix}archive/{release_tag}/{rel}" + rel = source_key[len(lakehouse_key_prefix) :] + archive_key = f"{lakehouse_key_prefix}archive/{release_tag}/{rel}" if dry_run: logger.info("[dry-run] would archive: %s -> %s", source_key, archive_key) @@ -247,10 +249,10 @@ def _archive_assemblies( # noqa: PLR0913 # ── Manifest trimming ─────────────────────────────────────────────────── -def _trim_manifest(manifest_s3_path: str, bucket: str, promoted_accessions: set[str]) -> None: +def _trim_manifest(manifest_s3_key: str, bucket: str, promoted_accessions: set[str]) -> None: """Remove promoted accessions from the transfer manifest in S3. - :param manifest_s3_path: S3 key of the transfer_manifest.txt + :param manifest_s3_key: S3 object key of the transfer_manifest.txt :param bucket: S3 bucket name :param promoted_accessions: set of accessions that were successfully promoted """ @@ -260,7 +262,16 @@ def _trim_manifest(manifest_s3_path: str, bucket: str, promoted_accessions: set[ tmp_path = tmp.name try: - s3.download_file(Bucket=bucket, Key=manifest_s3_path, Filename=tmp_path) + try: + s3.download_file(Bucket=bucket, Key=manifest_s3_key, Filename=tmp_path) + except s3.exceptions.NoSuchKey: + logger.warning("Manifest not found in S3 (s3://%s/%s) — skipping trim", bucket, manifest_s3_key) + return + except botocore.exceptions.ClientError as e: + if e.response["Error"]["Code"] == "404": + logger.warning("Manifest not found in S3 (s3://%s/%s) — skipping trim", bucket, manifest_s3_key) + return + raise with Path(tmp_path).open() as f: lines = f.readlines() @@ -270,7 +281,7 @@ def _trim_manifest(manifest_s3_path: str, bucket: str, promoted_accessions: set[ with Path(tmp_path).open("w") as f: f.writelines(remaining) - s3.upload_file(Filename=tmp_path, Bucket=bucket, Key=manifest_s3_path) + s3.upload_file(Filename=tmp_path, Bucket=bucket, Key=manifest_s3_key) logger.info( "Trimmed manifest: %d -> %d entries (%d promoted)", len(lines), diff --git a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py index 6508fc1f..616afee4 100644 --- a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py +++ b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py @@ -40,7 +40,7 @@ class DownloadSettings(CtsDefaultSettings): output_dir: str = Field( default=OUTPUT_MOUNT, description="Output directory for downloaded assembly files", - validation_alias=AliasChoices("o", "output-dir", "output_dir"), + validation_alias=AliasChoices("output-dir", "output_dir"), ) threads: int = Field( default=4, diff --git a/tests/integration/test_full_pipeline.py b/tests/integration/test_full_pipeline.py index b98bf342..2d1faab7 100644 --- a/tests/integration/test_full_pipeline.py +++ b/tests/integration/test_full_pipeline.py @@ -26,14 +26,14 @@ write_transfer_manifest, write_updated_manifest, ) -from cdm_data_loaders.ncbi_ftp.promote import DEFAULT_PATH_PREFIX, promote_from_s3 +from cdm_data_loaders.ncbi_ftp.promote import DEFAULT_LAKEHOUSE_KEY_PREFIX, promote_from_s3 from cdm_data_loaders.pipelines.ncbi_ftp_download import download_batch from .conftest import get_object_metadata, list_all_keys, stage_files_to_minio STABLE_PREFIX = "900" STAGING_PREFIX = "staging/run1/" -PATH_PREFIX = DEFAULT_PATH_PREFIX +PATH_PREFIX = DEFAULT_LAKEHOUSE_KEY_PREFIX @pytest.mark.integration @@ -80,9 +80,9 @@ def test_full_pipeline_small_batch( # ── Phase 3: Promote from staging to final path ───────────────── promote_report = promote_from_s3( - staging_prefix=STAGING_PREFIX, + staging_key_prefix=STAGING_PREFIX, bucket=test_bucket, - path_prefix=PATH_PREFIX, + lakehouse_key_prefix=PATH_PREFIX, ) assert promote_report["promoted"] >= 1 assert promote_report["failed"] == 0 @@ -138,10 +138,10 @@ def test_full_pipeline_incremental( s3.upload_file(Filename=str(manifest1), Bucket=test_bucket, Key=manifest_key) promote1 = promote_from_s3( - staging_prefix=STAGING_PREFIX, + staging_key_prefix=STAGING_PREFIX, bucket=test_bucket, - manifest_path=manifest_key, - path_prefix=PATH_PREFIX, + manifest_s3_key=manifest_key, + lakehouse_key_prefix=PATH_PREFIX, ) assert promote1["promoted"] >= 1 @@ -195,11 +195,11 @@ def test_full_pipeline_incremental( # Phase 3 — promote with archival promote2 = promote_from_s3( - staging_prefix=STAGING_PREFIX, + staging_key_prefix=STAGING_PREFIX, bucket=test_bucket, - updated_manifest=str(updated_manifest), + updated_manifest_path=str(updated_manifest), ncbi_release="test-incremental", - path_prefix=PATH_PREFIX, + lakehouse_key_prefix=PATH_PREFIX, ) assert promote2["failed"] == 0 diff --git a/tests/integration/test_manifest_e2e.py b/tests/integration/test_manifest_e2e.py index a8c22c02..df25d410 100644 --- a/tests/integration/test_manifest_e2e.py +++ b/tests/integration/test_manifest_e2e.py @@ -24,7 +24,7 @@ write_transfer_manifest, write_updated_manifest, ) -from cdm_data_loaders.ncbi_ftp.promote import DEFAULT_PATH_PREFIX +from cdm_data_loaders.ncbi_ftp.promote import DEFAULT_LAKEHOUSE_KEY_PREFIX from cdm_data_loaders.utils.ftp_client import connect_ftp, ftp_retrieve_text if TYPE_CHECKING: @@ -184,7 +184,7 @@ def test_prunes_existing_matching_md5( # Seed MinIO with dummy files that have the right MD5 metadata rel = build_accession_path(rec.assembly_dir) s3 = minio_s3_client - path_prefix = DEFAULT_PATH_PREFIX + path_prefix = DEFAULT_LAKEHOUSE_KEY_PREFIX for fname, md5 in checksums.items(): if any(fname.endswith(suffix) for suffix in FILE_FILTERS): key = f"{path_prefix}{rel}{fname}" @@ -201,7 +201,7 @@ def test_prunes_existing_matching_md5( candidates, filtered, bucket=test_bucket, - path_prefix=path_prefix, + key_prefix=path_prefix, ) assert acc not in result, f"Expected {acc} to be pruned (MD5 matches)" diff --git a/tests/integration/test_promote_e2e.py b/tests/integration/test_promote_e2e.py index 10670598..69fa3999 100644 --- a/tests/integration/test_promote_e2e.py +++ b/tests/integration/test_promote_e2e.py @@ -16,7 +16,7 @@ import pytest from cdm_data_loaders.ncbi_ftp.assembly import build_accession_path -from cdm_data_loaders.ncbi_ftp.promote import DEFAULT_PATH_PREFIX, promote_from_s3 +from cdm_data_loaders.ncbi_ftp.promote import DEFAULT_LAKEHOUSE_KEY_PREFIX, promote_from_s3 from .conftest import get_object_metadata, list_all_keys, seed_lakehouse @@ -32,7 +32,7 @@ ASSEMBLY_DIR_C = "GCF_900000003.1_FakeAssemblyC" STAGING_PREFIX = "staging/run1/" -PATH_PREFIX = DEFAULT_PATH_PREFIX +PATH_PREFIX = DEFAULT_LAKEHOUSE_KEY_PREFIX # Fake file contents for staging FAKE_GENOMIC = b">seq1\nATCGATCG\n" @@ -86,9 +86,9 @@ def test_promote_from_staging(self, minio_s3_client: object, test_bucket: str) - _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) report = promote_from_s3( - staging_prefix=STAGING_PREFIX, + staging_key_prefix=STAGING_PREFIX, bucket=test_bucket, - path_prefix=PATH_PREFIX, + lakehouse_key_prefix=PATH_PREFIX, ) assert report["promoted"] >= 2 # noqa: PLR2004 # genomic + protein @@ -116,16 +116,16 @@ def test_promote_idempotent(self, minio_s3_client: object, test_bucket: str) -> _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) report1 = promote_from_s3( - staging_prefix=STAGING_PREFIX, + staging_key_prefix=STAGING_PREFIX, bucket=test_bucket, - path_prefix=PATH_PREFIX, + lakehouse_key_prefix=PATH_PREFIX, ) keys_after_first = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") report2 = promote_from_s3( - staging_prefix=STAGING_PREFIX, + staging_key_prefix=STAGING_PREFIX, bucket=test_bucket, - path_prefix=PATH_PREFIX, + lakehouse_key_prefix=PATH_PREFIX, ) keys_after_second = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") @@ -157,11 +157,11 @@ def test_archive_updated(self, minio_s3_client: object, test_bucket: str, tmp_pa updated_manifest = _write_manifest(tmp_path, [ACCESSION_A], "updated_manifest.txt") report = promote_from_s3( - staging_prefix=STAGING_PREFIX, + staging_key_prefix=STAGING_PREFIX, bucket=test_bucket, - updated_manifest=str(updated_manifest), + updated_manifest_path=str(updated_manifest), ncbi_release="2024-01", - path_prefix=PATH_PREFIX, + lakehouse_key_prefix=PATH_PREFIX, ) assert report["archived"] >= 2 # noqa: PLR2004 @@ -198,11 +198,11 @@ def test_archive_removed(self, minio_s3_client: object, test_bucket: str, tmp_pa # Stage something (even empty staging is fine — promote won't find data files for this accession) report = promote_from_s3( - staging_prefix=STAGING_PREFIX, + staging_key_prefix=STAGING_PREFIX, bucket=test_bucket, - removed_manifest=str(removed_manifest), + removed_manifest_path=str(removed_manifest), ncbi_release="2024-01", - path_prefix=PATH_PREFIX, + lakehouse_key_prefix=PATH_PREFIX, ) assert report["archived"] >= 1 @@ -234,9 +234,9 @@ def test_promote_dry_run(self, minio_s3_client: object, test_bucket: str) -> Non _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) report = promote_from_s3( - staging_prefix=STAGING_PREFIX, + staging_key_prefix=STAGING_PREFIX, bucket=test_bucket, - path_prefix=PATH_PREFIX, + lakehouse_key_prefix=PATH_PREFIX, dry_run=True, ) @@ -271,10 +271,10 @@ def test_trims_manifest(self, minio_s3_client: object, test_bucket: str, tmp_pat _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_B) report = promote_from_s3( - staging_prefix=STAGING_PREFIX, + staging_key_prefix=STAGING_PREFIX, bucket=test_bucket, - manifest_path=manifest_key, - path_prefix=PATH_PREFIX, + manifest_s3_key=manifest_key, + lakehouse_key_prefix=PATH_PREFIX, ) assert report["failed"] == 0 @@ -306,9 +306,9 @@ def test_incomplete_staging(self, minio_s3_client: object, test_bucket: str) -> s3.put_object(Bucket=test_bucket, Key=md5_key, Body=_md5(FAKE_GENOMIC).encode()) report = promote_from_s3( - staging_prefix=STAGING_PREFIX, + staging_key_prefix=STAGING_PREFIX, bucket=test_bucket, - path_prefix=PATH_PREFIX, + lakehouse_key_prefix=PATH_PREFIX, ) # .md5 files are sidecars and should not be promoted as data diff --git a/tests/ncbi_ftp/test_manifest.py b/tests/ncbi_ftp/test_manifest.py index 61db7e5a..dac0400f 100644 --- a/tests/ncbi_ftp/test_manifest.py +++ b/tests/ncbi_ftp/test_manifest.py @@ -314,12 +314,27 @@ def test_custom_ftp_host(self) -> None: ) +def _mock_s3_with_objects() -> MagicMock: + """Return a mock S3 client whose list_objects_v2 always reports objects exist.""" + client = MagicMock() + client.list_objects_v2.return_value = {"KeyCount": 1} + return client + + +def _mock_s3_empty() -> MagicMock: + """Return a mock S3 client whose list_objects_v2 reports no objects.""" + client = MagicMock() + client.list_objects_v2.return_value = {"KeyCount": 0} + return client + + class TestVerifyTransferCandidates: """Test S3 checksum verification to prune transfer candidates.""" def _assemblies(self) -> dict: return parse_assembly_summary(SAMPLE_SUMMARY) + @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") @@ -328,6 +343,7 @@ def test_prunes_when_all_match( mock_connect: MagicMock, mock_retrieve: MagicMock, mock_head: MagicMock, + mock_s3: MagicMock, ) -> None: """Assemblies where every file matches S3 are pruned from the list.""" mock_connect.return_value = MagicMock() @@ -362,6 +378,7 @@ def head_side_effect(s3_path: str) -> dict | None: ) assert result == [] + @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") @@ -370,6 +387,7 @@ def test_keeps_when_md5_differs( mock_connect: MagicMock, mock_retrieve: MagicMock, mock_head: MagicMock, + mock_s3: MagicMock, ) -> None: """Assembly is kept when at least one file has a different MD5.""" mock_connect.return_value = MagicMock() @@ -383,6 +401,7 @@ def test_keeps_when_md5_differs( ) assert result == ["GCF_000001215.4"] + @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object", return_value=None) @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") @@ -391,6 +410,7 @@ def test_keeps_when_s3_object_missing( mock_connect: MagicMock, mock_retrieve: MagicMock, mock_head: MagicMock, + mock_s3: MagicMock, ) -> None: """Assembly is kept when at least one file doesn't exist in S3.""" mock_connect.return_value = MagicMock() @@ -403,6 +423,7 @@ def test_keeps_when_s3_object_missing( ) assert result == ["GCF_000001215.4"] + @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") @@ -411,6 +432,7 @@ def test_keeps_when_s3_has_no_md5_metadata( mock_connect: MagicMock, mock_retrieve: MagicMock, mock_head: MagicMock, + mock_s3: MagicMock, ) -> None: """Assembly is kept when S3 object exists but has no md5 metadata.""" mock_connect.return_value = MagicMock() @@ -424,9 +446,10 @@ def test_keeps_when_s3_has_no_md5_metadata( ) assert result == ["GCF_000001215.4"] + @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", side_effect=Exception("FTP error")) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") - def test_keeps_when_ftp_fails(self, mock_connect: MagicMock, mock_retrieve: MagicMock) -> None: + def test_keeps_when_ftp_fails(self, mock_connect: MagicMock, mock_retrieve: MagicMock, mock_s3: MagicMock) -> None: """Assembly is kept (conservative) when md5checksums.txt cannot be fetched.""" mock_connect.return_value = MagicMock() @@ -451,6 +474,7 @@ def test_unknown_accession_kept(self, mock_connect: MagicMock) -> None: result = verify_transfer_candidates(["GCF_999999999.1"], {}, "cdm-lake", "prefix/") assert result == ["GCF_999999999.1"] + @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object", return_value=None) @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") @@ -459,6 +483,7 @@ def test_short_circuits_on_first_mismatch( mock_connect: MagicMock, mock_retrieve: MagicMock, mock_head: MagicMock, + mock_s3: MagicMock, ) -> None: """Verification stops checking after the first missing/mismatched file.""" mock_connect.return_value = MagicMock() @@ -471,6 +496,7 @@ def test_short_circuits_on_first_mismatch( ) assert mock_head.call_count == 1 + @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") @@ -479,6 +505,7 @@ def test_mixed_candidates( mock_connect: MagicMock, mock_retrieve: MagicMock, mock_head: MagicMock, + mock_s3: MagicMock, ) -> None: """Verify a mix of matching and non-matching assemblies.""" mock_connect.return_value = MagicMock() @@ -514,3 +541,21 @@ def head_side_effect(s3_path: str) -> dict | None: "tenant-general-warehouse/kbase/datasets/ncbi/", ) assert result == ["GCF_000001405.40"] + + @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_empty()) + @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") + def test_skips_ftp_when_folder_missing_from_store( + self, + mock_connect: MagicMock, + mock_s3: MagicMock, + ) -> None: + """Accessions with no objects in S3 are confirmed without FTP round-trip.""" + result = verify_transfer_candidates( + ["GCF_000001215.4"], + self._assemblies(), + "cdm-lake", + "tenant-general-warehouse/kbase/datasets/ncbi/", + ) + assert result == ["GCF_000001215.4"] + # FTP should never have been connected (lazy init) + mock_connect.assert_not_called() diff --git a/tests/ncbi_ftp/test_notebooks.py b/tests/ncbi_ftp/test_notebooks.py index 6935c46d..fdd4a372 100644 --- a/tests/ncbi_ftp/test_notebooks.py +++ b/tests/ncbi_ftp/test_notebooks.py @@ -19,7 +19,7 @@ write_updated_manifest, ) from cdm_data_loaders.ncbi_ftp.promote import ( - DEFAULT_PATH_PREFIX, + DEFAULT_LAKEHOUSE_KEY_PREFIX, promote_from_s3, ) from cdm_data_loaders.utils.s3 import get_s3_client, split_s3_path # noqa: F401 @@ -86,4 +86,4 @@ class TestPromoteNotebookImports: def test_imports_resolve(self) -> None: """All promote notebook imports are verified at module load time above.""" assert callable(promote_from_s3) - assert isinstance(DEFAULT_PATH_PREFIX, str) + assert isinstance(DEFAULT_LAKEHOUSE_KEY_PREFIX, str) diff --git a/tests/ncbi_ftp/test_promote.py b/tests/ncbi_ftp/test_promote.py index bd1d7f0f..a2a98e2b 100644 --- a/tests/ncbi_ftp/test_promote.py +++ b/tests/ncbi_ftp/test_promote.py @@ -6,7 +6,7 @@ import pytest from cdm_data_loaders.ncbi_ftp.promote import ( - DEFAULT_PATH_PREFIX, + DEFAULT_LAKEHOUSE_KEY_PREFIX, _archive_assemblies, _trim_manifest, promote_from_s3, @@ -34,7 +34,7 @@ def test_dry_run_no_writes(self, mock_s3_client_no_checksum: botocore.client.Bas self._stage_files(mock_s3_client_no_checksum, prefix) report = promote_from_s3( - staging_prefix=prefix, + staging_key_prefix=prefix, bucket=TEST_BUCKET, dry_run=True, ) @@ -43,7 +43,7 @@ def test_dry_run_no_writes(self, mock_s3_client_no_checksum: botocore.client.Bas # Final path should NOT exist final_key = ( - f"{DEFAULT_PATH_PREFIX}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz" + f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz" ) resp = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=final_key) assert resp.get("KeyCount", 0) == 0 @@ -54,7 +54,7 @@ def test_promotes_with_metadata(self, mock_s3_client_no_checksum: botocore.clien self._stage_files(mock_s3_client_no_checksum, prefix) report = promote_from_s3( - staging_prefix=prefix, + staging_key_prefix=prefix, bucket=TEST_BUCKET, ) assert report["promoted"] == 1 @@ -62,7 +62,7 @@ def test_promotes_with_metadata(self, mock_s3_client_no_checksum: botocore.clien # Check final object exists with metadata final_key = ( - f"{DEFAULT_PATH_PREFIX}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz" + f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz" ) resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=final_key) assert resp["Metadata"].get("md5") == "md5hash123" @@ -72,7 +72,7 @@ def test_skips_download_report(self, mock_s3_client_no_checksum: botocore.client prefix = "staging/run1/" self._stage_files(mock_s3_client_no_checksum, prefix) - report = promote_from_s3(staging_prefix=prefix, bucket=TEST_BUCKET) + report = promote_from_s3(staging_key_prefix=prefix, bucket=TEST_BUCKET) # Only the .fna.gz data file, not download_report.json assert report["promoted"] == 1 @@ -119,7 +119,7 @@ def test_archives_and_deletes_removed( ) -> None: """Verify removed accessions are archived and originals deleted.""" accession = "GCF_000005845.2" - key = f"{DEFAULT_PATH_PREFIX}raw_data/GCF/000/005/845/{accession}_ASM584v2/{accession}_genomic.fna.gz" + key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/005/845/{accession}_ASM584v2/{accession}_genomic.fna.gz" mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"data") manifest = tmp_path / "removed.txt" @@ -140,7 +140,7 @@ def test_archives_and_deletes_removed( # Archived copy should exist archive_key = ( - f"{DEFAULT_PATH_PREFIX}archive/2024-01/" + f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/" f"raw_data/GCF/000/005/845/{accession}_ASM584v2/{accession}_genomic.fna.gz" ) resp = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_key) @@ -152,7 +152,7 @@ def test_archives_updated_without_deleting( """Verify updated accessions are archived but originals remain.""" accession = "GCF_000001215.4" asm_dir = f"{accession}_Release_6" - key = f"{DEFAULT_PATH_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" + key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"original-data") manifest = tmp_path / "updated.txt" @@ -173,7 +173,7 @@ def test_archives_updated_without_deleting( # Archived copy exists with correct metadata archive_key = ( - f"{DEFAULT_PATH_PREFIX}archive/2024-06/" + f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-06/" f"raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" ) resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=archive_key) @@ -186,7 +186,7 @@ def test_multiple_releases_no_collision( """Verify archiving the same accession in different releases creates distinct folders.""" accession = "GCF_000001215.4" asm_dir = f"{accession}_Release_6" - key = f"{DEFAULT_PATH_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" + key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"v1-data") manifest = tmp_path / "updated.txt" @@ -202,11 +202,11 @@ def test_multiple_releases_no_collision( _archive_assemblies(str(manifest), bucket=TEST_BUCKET, ncbi_release="2024-06", archive_reason="updated") archive_key_1 = ( - f"{DEFAULT_PATH_PREFIX}archive/2024-01/" + f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/" f"raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" ) archive_key_2 = ( - f"{DEFAULT_PATH_PREFIX}archive/2024-06/" + f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-06/" f"raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" ) resp1 = mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=archive_key_1) @@ -220,7 +220,7 @@ def test_dry_run_no_side_effects( ) -> None: """Verify dry_run does not copy or delete anything.""" accession = "GCF_000005845.2" - key = f"{DEFAULT_PATH_PREFIX}raw_data/GCF/000/005/845/{accession}_ASM584v2/{accession}_genomic.fna.gz" + key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/005/845/{accession}_ASM584v2/{accession}_genomic.fna.gz" mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"data") manifest = tmp_path / "removed.txt" @@ -241,7 +241,7 @@ def test_dry_run_no_side_effects( assert resp.get("KeyCount", 0) == 1 # No archive created - archive_prefix = f"{DEFAULT_PATH_PREFIX}archive/2024-01/" + archive_prefix = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/" resp = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_prefix) assert resp.get("KeyCount", 0) == 0 @@ -261,7 +261,7 @@ def test_unknown_release_fallback( """Verify ncbi_release=None falls back to 'unknown'.""" accession = "GCF_000001215.4" asm_dir = f"{accession}_Release_6" - key = f"{DEFAULT_PATH_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" + key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"data") manifest = tmp_path / "updated.txt" @@ -271,7 +271,7 @@ def test_unknown_release_fallback( assert count == 1 archive_key = ( - f"{DEFAULT_PATH_PREFIX}archive/unknown/" + f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/unknown/" f"raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" ) resp = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_key) diff --git a/tests/pipelines/test_ncbi_ftp_download.py b/tests/pipelines/test_ncbi_ftp_download.py index 1dfc0997..90df299f 100644 --- a/tests/pipelines/test_ncbi_ftp_download.py +++ b/tests/pipelines/test_ncbi_ftp_download.py @@ -92,9 +92,9 @@ def test_manifest_alias_m(self) -> None: s = make_settings(m="/data/m.txt") assert s.manifest == "/data/m.txt" - def test_output_dir_alias_o(self) -> None: - """Verify 'o' alias resolves to output_dir.""" - s = make_settings(o="/data/o") + def test_output_dir_alias(self) -> None: + """Verify 'output_dir' / 'output-dir' alias resolves to output_dir.""" + s = make_settings(output_dir="/data/o") assert s.output_dir == "/data/o" def test_threads_alias_t(self) -> None: From e5bc2f6b74a1c69a742b6edc3e83f0e235607a89 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Fri, 17 Apr 2026 14:56:19 -0700 Subject: [PATCH 005/128] cleanup and formatting --- cdm_ncbi_fto.prompt.md | 292 ------------------------------ notebooks/ncbi_ftp_manifest.ipynb | 15 +- notebooks/ncbi_ftp_promote.ipynb | 4 +- tests/ncbi_ftp/test_notebooks.py | 6 +- tests/ncbi_ftp/test_promote.py | 8 +- tests/utils/test_s3.py | 10 +- 6 files changed, 18 insertions(+), 317 deletions(-) delete mode 100644 cdm_ncbi_fto.prompt.md diff --git a/cdm_ncbi_fto.prompt.md b/cdm_ncbi_fto.prompt.md deleted file mode 100644 index a1e52592..00000000 --- a/cdm_ncbi_fto.prompt.md +++ /dev/null @@ -1,292 +0,0 @@ -# Plan: Port NCBI Assembly Sync to cdm-data-loaders - -Port the 3-phase NCBI assembly transfer pipeline from this repo -([kbase-transfers](https://github.com/kbase/kbase-transfers)) on the `develop-ncbi-automation` branchinto -[kbase/cdm-data-loaders](https://github.com/kbase/cdm-data-loaders/tree/develop) -(develop branch). - -- **Phase 2** (container download) becomes a new CTS entrypoint command. -- **Phases 1 and 3** become Jupyter notebooks in `notebooks/`. -- The existing cdm-data-loaders `utils/s3.py` gets new functions for metadata support - (existing functions are not modified). -- Tests use **moto** (matching cdm-data-loaders conventions). -- FTP logic stays as **ftplib**. -- New code lives in a dedicated `src/cdm_data_loaders/ncbi_ftp/` module, - separate from existing NCBI REST / refseq code. - -### Phase responsibilities - -Each phase has a deliberately narrow scope: - -| Phase | Input | Output | Responsibility | -|-------|-------|--------|----------------| -| 1 — Manifest | NCBI assembly summary + previous snapshot from S3 | `transfer_manifest.txt`, `removed_manifest.txt`, `diff_summary.json` | **All** filtering logic (prefix ranges, limits, diffing). Produces a final list of what to transfer and what to archive. | -| 2 — Download | `transfer_manifest.txt` (from input mount) | Downloaded files in output mount, preserving FTP directory structure (`GCF/000/001/.../assembly_dir/`) | Reads the manifest; downloads exactly those assemblies from NCBI FTP; verifies MD5; writes `.md5` sidecars. No filtering, no S3 access. | -| 3 — Promote | Downloaded files in S3 staging prefix + `removed_manifest.txt` | Files at final Lakehouse paths; archived replaced assemblies | Syncs staging → final location. Archives replaced/suppressed assemblies. **Removes successfully-promoted entries from `transfer_manifest.txt`** so an interrupted Phase 2 can resume from where it left off. | - ---- - -## Background: the 3-phase pipeline - -The pipeline is documented in this repo's -[scripts/ncbi/README.md](README.md#semi-automated-transfer-pipeline): - -| Phase | Script (this repo) | Purpose | -|-------|-------------------|---------| -| 1 — Manifest | [`generate_transfer_manifest.py`](generate_transfer_manifest.py) | Diff NCBI assembly summary against previous snapshot; produce download + removal manifests | -| 2 — Download | [`container_download.py`](container_download.py) | Download assemblies from NCBI FTP, verify MD5, write `.md5` sidecars (runs in CTS container) | -| 3 — Promote | [`promote_staged_files.py`](promote_staged_files.py) | Copy staged files to final Lakehouse paths; archive replaced/suppressed assemblies | - -Supporting code in this repo: - -| File | What to port | -|------|-------------| -| [`download_genomes.py`](download_genomes.py) | FTP resilience patterns (TCP keepalive, NOOP pings, thread-local connections), file filters, accession path construction | -| [`../../kbase_transfers/minio_client.py`](../../kbase_transfers/minio_client.py) | Metadata-aware upload pattern (MD5 as user metadata, CRC64/NVME checksums) | -| [`../../tests/test_sync.py`](../../tests/test_sync.py) | Unit tests for parsing, diffing — port to moto-based tests | -| [`../../tests/test_minio_client.py`](../../tests/test_minio_client.py) | Integration test patterns for S3 operations | - ---- - -## Phase A: Extend cdm-data-loaders `utils/s3.py` - -The promote step needs to attach user-metadata (MD5) to uploads and read -checksums via HEAD. The existing -[`s3.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/utils/s3.py) -doesn't support custom metadata on upload or `head_object` with checksum -retrieval. - -**Prefer adding new functions over modifying existing ones to minimise -impact on other scripts.** - -### Steps - -1. **Add `upload_file_with_metadata()`** — new function that accepts - `local_file_path`, `destination_dir`, `metadata: dict[str, str]`, - optional `object_name`. Passes `Metadata` in `ExtraArgs` alongside the - existing `ChecksumAlgorithm: CRC64NVME`. Same upload logic as - `upload_file()` but with metadata support. - -2. **Add `head_object(s3_path)`** — new function returning dict with `size`, - `metadata`, `checksum_crc64nvme` (from `ChecksumCRC64NVME` header), or - `None` if 404. Uses `ChecksumMode='ENABLED'`. - -3. **Add `copy_object_with_metadata()`** — new function wrapping - `s3.copy_object()` that accepts `metadata` + `MetadataDirective='REPLACE'` - for archiving replaced assemblies with tags (`archive_reason`, `archive_date`, - `ncbi_last_release`). - -4. **Add moto-based tests** following the existing `mock_s3_client` fixture pattern - in [`tests/utils/test_s3.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/tests/utils/test_s3.py). - Use the existing `strip_checksum_algorithm` workaround for moto's CRC64NVME limitation. - -**Files modified:** -- `src/cdm_data_loaders/utils/s3.py` — add 3 new functions (no changes to existing functions) -- `tests/utils/test_s3.py` — add corresponding tests - ---- - -## Phase B: Create the `ncbi_ftp` module - -New module at `src/cdm_data_loaders/ncbi_ftp/`, separate from the -existing `ncbi_rest_api.py` pipeline and `refseq_pipeline/`. - -``` -src/cdm_data_loaders/ncbi_ftp/ -├── __init__.py -├── ftp_client.py # FTP: connect, keepalive, list, download, retry -├── manifest.py # Phase 1: summary diffing, manifest generation, ALL filtering -├── download.py # Phase 2: CTS container download + MD5 verification -├── promote.py # Phase 3: sync staging → final, archive, trim manifest -├── checksums.py # MD5 verification, CRC64/NVME computation -└── settings.py # Pydantic settings (extends CtsDefaultSettings) -``` - -### Steps - -5. **`ftp_client.py`** — Port from [`download_genomes.py`](download_genomes.py) - and [`container_download.py`](container_download.py): - - `connect_ftp(host, timeout)` with TCP keepalive (`SO_KEEPALIVE`, `TCP_KEEPIDLE`, `TCP_KEEPINTVL`) - - `ftp_noop_keepalive(ftp, interval)` — NOOP sender for idle connections - - `ftp_list_dir(ftp, path)` — NLST wrapper with retry on `error_temp` - - `ftp_download_file(ftp, remote_path, local_path)` — `RETR` with retry - - Thread-local FTP connection management for parallel downloads - - Use `get_cdm_logger()` instead of print statements - -6. **`checksums.py`** — Port from [`download_genomes.py`](download_genomes.py): - - `compute_crc64nvme(file_path) -> str` — reads in 1MB chunks, returns base64-encoded 8-byte big-endian (uses `awscrt.checksums.crc64nvme`) - - `verify_md5(file_path, expected_md5) -> bool` - - `parse_md5_checksums_file(text) -> dict[str, str]` — parses NCBI `md5checksums.txt` - -7. **`settings.py`** — Pydantic settings following - [`cts_defaults.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/pipelines/cts_defaults.py) pattern: - - `DownloadSettings(CtsDefaultSettings)` — adds `manifest`, `threads`, `ftp_host` - - CLI-parseable with `AliasChoices`, `field_validator` for constraints - -8. **`manifest.py`** — Port from [`generate_transfer_manifest.py`](generate_transfer_manifest.py): - - `download_assembly_summary(ftp, database) -> str` - - `parse_assembly_summary(text) -> dict[str, AssemblyRecord]` - - `compute_diff(current, previous) -> DiffResult` — new/updated/replaced/suppressed/withdrawn - - `write_manifest(diff, output_dir)` — writes `transfer_manifest.txt`, `removed_manifest.txt`, `diff_summary.json` - - **All filtering logic lives here**: prefix-range filtering (`--prefix-from`, `--prefix-to`), `--limit`, any other subsetting - - The output `transfer_manifest.txt` is the final, filtered list — Phase 2 downloads exactly what's in it - -9. **`download.py`** — Port from [`container_download.py`](container_download.py). - **This phase is deliberately simple**: read manifest, download, verify. - - `run_download(settings: DownloadSettings)` — main CTS entry point - - Reads `transfer_manifest.txt` from input mount; each line is an FTP path - - `download_assembly(ftp, ftp_path, output_dir) -> DownloadResult` - - File filters: `_genomic.fna.gz`, `_genomic.gff.gz`, `_protein.faa.gz`, `_gene_ontology.gaf.gz`, `_assembly_report.txt`, `_assembly_stats.txt`, etc. - - MD5 verification (3 retries), `.md5` sidecar writing - - `ThreadPoolExecutor` for parallel downloads - - Output preserves FTP directory structure: `{GCF|GCA}/{000}/{001}/{215}/{assembly_dir}/` (same subfolder hierarchy as on the FTP server) - - Writes `download_report.json` summary - - `cli()` function using `run_cli(DownloadSettings, run_download)` from - [`core.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/pipelines/core.py) - - **No filtering or subsetting logic** — downloads exactly what's in the manifest - -10. **`promote.py`** — Port from [`promote_staged_files.py`](promote_staged_files.py): - - `run_promote(staging_prefix, removed_manifest, release_tag, manifest_path)` - - Walk staged files in S3 staging prefix, upload each to final Lakehouse path via `upload_file_with_metadata()` with MD5 from `.md5` sidecars - - Skip `.md5` and `.crc64nvme` sidecar files themselves - - Archive replaced/suppressed assemblies (from `removed_manifest.txt`): `copy_object_with_metadata()` to `archive/{release}/...` with metadata tags, then `delete_object()` - - **Manifest trimming for resumability**: after successfully promoting an assembly's files, remove that entry from `transfer_manifest.txt`. If Phase 2 is re-run after a partial failure, it only downloads the remaining entries. - - `--dry-run` support - ---- - -## Phase C: Notebooks for Phases 1 and 3 - -11. **`notebooks/ncbi_ftp_manifest.ipynb`** — Phase 1: - - Cell 1: imports + S3 client init (`get_s3_client()`) - - Cell 2: configure parameters (database, prefix-from/to, limit, dry-run) - - Cell 3: download current assembly summary from FTP - - Cell 4: load previous summary from S3 (or scan existing prefixes) - - Cell 5: compute diff, display summary - - Cell 6: apply filters (prefix range, limit), write manifest files, upload new summary to S3 - -12. **`notebooks/ncbi_ftp_promote.ipynb`** — Phase 3: - - Cell 1: imports + S3 client init - - Cell 2: configure parameters (staging prefix, removed manifest path, release tag, manifest path for trimming, dry-run) - - Cell 3: scan staged files, display summary - - Cell 4: promote files to final paths - - Cell 5: archive replaced/suppressed assemblies - - Cell 6: trim manifest (remove promoted entries), display promotion report - ---- - -## Phase D: Container integration (Phase 2) - -13. Register CLI entry point in `pyproject.toml`: - ```toml - [project.scripts] - ncbi_ftp_sync = "cdm_data_loaders.ncbi_ftp.download:cli" - ``` - -14. Add command to `scripts/entrypoint.sh`: - ```bash - ncbi_ftp_sync) - exec /usr/bin/tini -- uv run --no-sync ncbi_ftp_sync "$@" - ;; - ``` - -15. No Dockerfile changes needed (package installed via `uv sync`; entrypoint dispatches). - ---- - -## Phase E: Tests - -All tests use **moto** for S3 mocking. No live MinIO dependency in CI. - -``` -tests/ncbi_ftp/ -├── __init__.py -├── conftest.py # Mock FTP, sample manifests, assembly records -├── test_ftp_client.py # Mock ftplib: keepalive, retry, thread-local -├── test_checksums.py # MD5 verify, CRC64/NVME, md5checksums.txt parsing -├── test_manifest.py # Summary parsing, diff logic, filtering (port from test_sync.py) -├── test_download.py # Mock FTP + fs: filters, MD5 verify, sidecars, layout -├── test_promote.py # moto S3: upload with metadata, archive, manifest trimming, dry-run -└── test_settings.py # Pydantic validation (follow test_ncbi_rest_api.py) -``` - -### Steps - -16. **`test_checksums.py`** — `verify_md5` correct/incorrect, `parse_md5_checksums_file`, - `compute_crc64nvme` (skip if `awscrt` unavailable) - -17. **`test_manifest.py`** — Port relevant tests from this repo's - [`tests/test_sync.py`](../../tests/test_sync.py): `parse_assembly_summary`, - `compute_diff` (new/updated/replaced/suppressed/withdrawn), `write_manifest`, - prefix-range filtering, limit filtering - -18. **`test_ftp_client.py`** — Mock `ftplib.FTP`: keepalive options, retry on - `error_temp`, thread-local connections - -19. **`test_download.py`** — Mock FTP + filesystem: file filter logic, MD5 - verification, sidecar writing, directory layout preserves FTP structure, - `download_report.json` - -20. **`test_promote.py`** — moto `mock_s3_client` fixture: upload with metadata, - archive copy with tags, deletion of originals, **manifest trimming** (verify - promoted entries removed, remaining entries preserved), dry-run no side effects. - Use `strip_checksum_algorithm` workaround for CRC64NVME. - -21. **`test_settings.py`** — Follow - [`test_ncbi_rest_api.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/tests/pipelines/test_ncbi_rest_api.py) - pattern: defaults, all params, CLI variants, invalid values, boolean parsing - ---- - -## Phase F: Dependencies and CI - -22. Add `awscrt` to `pyproject.toml` if not already covered by `boto3[crt]`. - -23. All new tests run automatically in CI — no `requires_spark` marks needed. - ruff checks apply (120 char lines, py313 target). - ---- - -## Key reference patterns in cdm-data-loaders - -| Pattern | Where to find it | -|---------|-----------------| -| S3 utility functions + moto tests | [`utils/s3.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/utils/s3.py), [`tests/utils/test_s3.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/tests/utils/test_s3.py) | -| CTS settings base class | [`pipelines/cts_defaults.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/pipelines/cts_defaults.py) | -| `run_cli()` entry point | [`pipelines/core.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/pipelines/core.py) | -| Logger | [`utils/cdm_logger.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/utils/cdm_logger.py) | -| Settings test pattern | [`tests/pipelines/test_ncbi_rest_api.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/tests/pipelines/test_ncbi_rest_api.py) | -| Entrypoint dispatch | [`scripts/entrypoint.sh`](https://github.com/kbase/cdm-data-loaders/blob/develop/scripts/entrypoint.sh) | -| moto CRC64NVME workaround | `strip_checksum_algorithm()` in [`tests/utils/test_s3.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/tests/utils/test_s3.py) | - ---- - -## Verification - -1. `ruff check src/cdm_data_loaders/ncbi_ftp/ tests/ncbi_ftp/` — lint passes -2. `ruff format --check src/cdm_data_loaders/ncbi_ftp/ tests/ncbi_ftp/` — formatting passes -3. `uv run pytest tests/ncbi_ftp/ -v` — all unit tests pass -4. `uv run pytest tests/utils/test_s3.py -v` — new S3 function tests pass -5. Manual: build Docker image, run `ncbi_ftp_sync` with small manifest against local MinIO -6. Manual: run both notebooks against local MinIO for end-to-end verification - ---- - -## Decisions - -- **`ncbi_ftp` naming** — distinguishes bulk FTP file transfer from the existing NCBI REST API pipeline (`ncbi_rest_api.py`) and Spark-based refseq processing (`refseq_pipeline/`) -- **New functions in `s3.py`, not modified existing ones** — minimises impact on other scripts; avoids signature changes that could break callers -- **All filtering in Phase 1** — Phase 2 is a pure download-what's-in-the-manifest step; Phase 3 is a pure sync-and-archive step. Clean separation of concerns. -- **Manifest trimming in Phase 3** — enables resumable Phase 2 runs. After promoting files, Phase 3 removes those entries from `transfer_manifest.txt`. Re-running Phase 2 only downloads what's left. -- **Output preserves FTP directory structure** — Phase 2 writes files under the same `GCF/000/001/.../assembly_dir/` path used on the FTP server, making it trivial to correlate staged files with their NCBI source -- **moto for tests** — matches cdm-data-loaders conventions; fast, no Docker in CI. The `strip_checksum_algorithm` workaround handles the CRC64NVME gap. -- **ftplib over httpx** — NCBI FTP is the established bulk download protocol; existing keepalive/NOOP/retry patterns are proven -- **Notebooks for Phases 1 & 3** — interactive, judgement-requiring steps; natural fit for JupyterLab -- **Phase 2 as CTS command** — matches the entrypoint dispatch pattern and CTS mount contract - -## Excluded from scope - -- Frictionless `datapackage.json` descriptors (only in old monolithic `download_genomes.py`) -- `backfill_checksums.py` (legacy utility, not part of ongoing pipeline) -- `download_genomes.py` monolith (superseded by the 3-phase pipeline) -- Spark/Delta Lake integration (assembly sync is file-level, not data transformation) diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb index 20ee5016..e59c6043 100644 --- a/notebooks/ncbi_ftp_manifest.ipynb +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -201,9 +201,15 @@ "\n", " candidates = diff.new + diff.updated\n", " print(f\"Verifying {len(candidates)} candidates against s3://{STORE_BUCKET}/{STORE_KEY_PREFIX} ...\")\n", - " confirmed = set(verify_transfer_candidates(\n", - " candidates, filtered, STORE_BUCKET, STORE_KEY_PREFIX, ftp_host=FTP_HOST,\n", - " ))\n", + " confirmed = set(\n", + " verify_transfer_candidates(\n", + " candidates,\n", + " filtered,\n", + " STORE_BUCKET,\n", + " STORE_KEY_PREFIX,\n", + " ftp_host=FTP_HOST,\n", + " )\n", + " )\n", " before = len(diff.new) + len(diff.updated)\n", " diff.new = [a for a in diff.new if a in confirmed]\n", " diff.updated = [a for a in diff.updated if a in confirmed]\n", @@ -290,8 +296,7 @@ " bucket, prefix = split_s3_path(STAGING_URI)\n", " prefix = prefix.rstrip(\"/\") + \"/\"\n", "\n", - " for manifest in [\"transfer_manifest.txt\", \"removed_manifest.txt\",\n", - " \"updated_manifest.txt\", \"diff_summary.json\"]:\n", + " for manifest in [\"transfer_manifest.txt\", \"removed_manifest.txt\", \"updated_manifest.txt\", \"diff_summary.json\"]:\n", " local_path = OUTPUT_DIR / manifest\n", " if local_path.exists():\n", " key = f\"{prefix}{manifest}\"\n", diff --git a/notebooks/ncbi_ftp_promote.ipynb b/notebooks/ncbi_ftp_promote.ipynb index 5dd46312..1da31ab2 100644 --- a/notebooks/ncbi_ftp_promote.ipynb +++ b/notebooks/ncbi_ftp_promote.ipynb @@ -184,10 +184,10 @@ "print(f\"Dry-run: {report['dry_run']}\")\n", "print(f\"Timestamp: {report['timestamp']}\")\n", "\n", - "if report['failed'] > 0:\n", + "if report[\"failed\"] > 0:\n", " print(\"\\n⚠️ Some operations failed — check logs above for details.\")\n", "\n", - "if report['dry_run']:\n", + "if report[\"dry_run\"]:\n", " print(\"\\n📋 This was a dry-run. Set DRY_RUN = False and re-run to apply changes.\")" ] } diff --git a/tests/ncbi_ftp/test_notebooks.py b/tests/ncbi_ftp/test_notebooks.py index fdd4a372..ba19c2a9 100644 --- a/tests/ncbi_ftp/test_notebooks.py +++ b/tests/ncbi_ftp/test_notebooks.py @@ -40,11 +40,7 @@ def _extract_code_cells(notebook_path: Path) -> list[str]: """ with notebook_path.open() as f: nb = json.load(f) - return [ - "".join(cell.get("source", [])) - for cell in nb.get("cells", []) - if cell.get("cell_type") == "code" - ] + return ["".join(cell.get("source", [])) for cell in nb.get("cells", []) if cell.get("cell_type") == "code"] @pytest.mark.parametrize("notebook", NCBI_NOTEBOOKS) diff --git a/tests/ncbi_ftp/test_promote.py b/tests/ncbi_ftp/test_promote.py index a2a98e2b..fa9898d0 100644 --- a/tests/ncbi_ftp/test_promote.py +++ b/tests/ncbi_ftp/test_promote.py @@ -42,9 +42,7 @@ def test_dry_run_no_writes(self, mock_s3_client_no_checksum: botocore.client.Bas assert report["dry_run"] is True # Final path should NOT exist - final_key = ( - f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz" - ) + final_key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz" resp = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=final_key) assert resp.get("KeyCount", 0) == 0 @@ -61,9 +59,7 @@ def test_promotes_with_metadata(self, mock_s3_client_no_checksum: botocore.clien assert report["failed"] == 0 # Check final object exists with metadata - final_key = ( - f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz" - ) + final_key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz" resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=final_key) assert resp["Metadata"].get("md5") == "md5hash123" diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index 26292a16..cb544a4a 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -559,7 +559,7 @@ def mocked_s3_client_no_checksum(mock_s3_client: Any) -> Generator[Any, Any]: allowing copy_object calls that include ChecksumAlgorithm to succeed. """ mock_s3_client.copy_object = strip_checksum_algorithm(mock_s3_client.copy_object) - yield mock_s3_client + return mock_s3_client # copy_object @@ -656,9 +656,7 @@ def test_upload_file_with_metadata_accepts_str_and_path(sample_file: Path, path_ @pytest.mark.s3 def test_head_object_returns_info(mock_s3_client: Any) -> None: """Verify that head_object returns size, metadata, and checksum fields.""" - mock_s3_client.put_object( - Bucket=CDM_LAKE_BUCKET, Key="info/file.txt", Body=b"hello", Metadata={"md5": "abc123"} - ) + mock_s3_client.put_object(Bucket=CDM_LAKE_BUCKET, Key="info/file.txt", Body=b"hello", Metadata={"md5": "abc123"}) result = head_object(f"{CDM_LAKE_BUCKET}/info/file.txt") assert result is not None assert result["size"] == 5 @@ -687,9 +685,7 @@ def test_head_object_with_protocols(mock_s3_client: Any, protocol: str) -> None: # copy_object_with_metadata @pytest.mark.parametrize("destination", BUCKETS) @pytest.mark.s3 -def test_copy_object_with_metadata_replaces_metadata( - mocked_s3_client_no_checksum: Any, destination: str -) -> None: +def test_copy_object_with_metadata_replaces_metadata(mocked_s3_client_no_checksum: Any, destination: str) -> None: """Verify that copy_object_with_metadata copies and replaces metadata.""" mocked_s3_client_no_checksum.put_object( Bucket=CDM_LAKE_BUCKET, Key="src/file.txt", Body=b"archive me", Metadata={"old_key": "old_val"} From de8d325eabdbd314d565e1984fb11bee7c0812df Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Fri, 17 Apr 2026 15:06:19 -0700 Subject: [PATCH 006/128] formatting --- src/cdm_data_loaders/ncbi_ftp/manifest.py | 2 +- src/cdm_data_loaders/utils/s3.py | 28 ++++++++++---------- tests/utils/test_s3.py | 31 ++++++++++++++--------- 3 files changed, 33 insertions(+), 28 deletions(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/manifest.py b/src/cdm_data_loaders/ncbi_ftp/manifest.py index e67c2185..f6a9fce1 100644 --- a/src/cdm_data_loaders/ncbi_ftp/manifest.py +++ b/src/cdm_data_loaders/ncbi_ftp/manifest.py @@ -260,7 +260,7 @@ def _ftp_dir_from_url(ftp_url: str, ftp_host: str = FTP_HOST) -> str: # ── Checksum verification against S3 store ─────────────────────────────── -def verify_transfer_candidates( +def verify_transfer_candidates( # noqa: PLR0912, PLR0915 accessions: list[str], current_assemblies: dict[str, AssemblyRecord], bucket: str, diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index 6fc1696a..c3897904 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -35,7 +35,7 @@ def get_s3_client(args: dict[str, str] | None = None) -> botocore.client.BaseCli if not args: try: - from berdl_notebook_utils.berdl_settings import get_settings + from berdl_notebook_utils.berdl_settings import get_settings # noqa: PLC0415 settings = get_settings() args = { @@ -44,9 +44,7 @@ def get_s3_client(args: dict[str, str] | None = None) -> botocore.client.BaseCli "aws_secret_access_key": settings.MINIO_SECRET_KEY, } except (ModuleNotFoundError, ImportError, NameError) as e: - print(e) - raise - except Exception: + print(e) # noqa: T201 raise required_args = ["endpoint_url", "aws_access_key_id", "aws_secret_access_key"] @@ -139,10 +137,10 @@ def object_exists(s3_path: str) -> bool: (bucket, key) = split_s3_path(s3_path) try: s3.head_object(Bucket=bucket, Key=key) - except Exception as e: + except botocore.exceptions.ClientError as e: error_string = str(e) if not error_string.startswith("An error occurred (404) when calling the HeadObject operation: Not Found"): - print(f"Error performing head operation on s3 object: {e!s}") + print(f"Error performing head operation on s3 object: {e!s}") # noqa: T201 return False return True @@ -175,7 +173,7 @@ def upload_file( s3_path = f"{destination_dir.removesuffix('/')}/{object_name}" if object_exists(s3_path): - print(f"File already present: {s3_path}") + print(f"File already present: {s3_path}") # noqa: T201 return True s3 = get_s3_client() @@ -184,7 +182,7 @@ def upload_file( # Upload the file file_size = local_file_path.stat().st_size with tqdm.tqdm(total=file_size, unit="B", unit_scale=True, desc=str(local_file_path)) as pbar: - print(f"uploading {local_file_path!s} to {s3_path}") + print(f"uploading {local_file_path!s} to {s3_path}") # noqa: T201 try: s3.upload_file( Filename=str(local_file_path), @@ -193,8 +191,8 @@ def upload_file( Callback=pbar.update, ExtraArgs=DEFAULT_EXTRA_ARGS, ) - except Exception as e: - print(f"Error uploading to s3: {e!s}") + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + print(f"Error uploading to s3: {e!s}") # noqa: T201 return False return True @@ -219,8 +217,8 @@ def download_file(s3_path: str, local_file_path: str | Path, version_id: str | N if not parent_dir.is_dir(): try: parent_dir.mkdir(parents=True, exist_ok=False) - except Exception as e: - print(f"Could not save s3 file to {local_file_path}: {e!s}") + except OSError as e: + print(f"Could not save s3 file to {local_file_path}: {e!s}") # noqa: T201 raise s3 = get_s3_client() @@ -232,12 +230,12 @@ def download_file(s3_path: str, local_file_path: str | Path, version_id: str | N # Get the object size try: object_size = s3.head_object(**kwargs)["ContentLength"] - except Exception as e: + except botocore.exceptions.ClientError as e: error_string = str(e) if error_string.startswith("An error occurred (404) when calling the HeadObject operation: Not Found"): - print(f"File not found: {s3_path}") + print(f"File not found: {s3_path}") # noqa: T201 else: - print(f"Error downloading {s3_path}: {e!s}") + print(f"Error downloading {s3_path}: {e!s}") # noqa: T201 raise extra_args = {"VersionId": version_id} if version_id is not None else None diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index cb544a4a..9050448f 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -47,6 +47,11 @@ } BUCKETS = [CDM_LAKE_BUCKET, ALT_BUCKET] +HTTP_STATUS_OK = 200 +HTTP_STATUS_NO_CONTENT = 204 +SIZE_HELLO = 5 +SIZE_DATA = 4 + @pytest.fixture def mock_s3_client() -> Generator[Any, Any]: @@ -307,7 +312,7 @@ def test_list_matching_objects_empty_for_missing_prefix( } -# TODO: use a single fixture for all these tests +# NOTE: These tests currently compose multiple fixtures explicitly for readability. @pytest.mark.parametrize("dir_path", EXPECTED_FILE_LIST.keys()) @pytest.mark.s3 def test_list_matching_objects_returns_more_than_1000_entries( @@ -478,7 +483,8 @@ def test_download_file_does_not_clobber_existing_file_to_mkdir(mock_s3_client: A @pytest.mark.s3 -def test_download_file_does_not_exist(mock_s3_client: Any, tmp_path: Path, capsys: pytest.CaptureFixture) -> None: +@pytest.mark.usefixtures("mock_s3_client") +def test_download_file_does_not_exist(tmp_path: Path, capsys: pytest.CaptureFixture) -> None: """Ensure that attempting to download a file that does not exist raises an error.""" bucket = BUCKETS[0] key = "to/the/door.txt" @@ -533,8 +539,8 @@ def test_upload_dir_raises_on_empty_destination(sample_dir: Path) -> None: upload_dir(sample_dir, "") -# FIXME: once moto supports CRC64NVME, this can be removed -def strip_checksum_algorithm(method: Callable): +# NOTE: Moto currently does not support CRC64NVME; remove this helper when it does. +def strip_checksum_algorithm(method: Callable[..., Any]) -> Callable[..., Any]: """Wrap a boto3 S3 method to remove the ChecksumAlgorithm argument before calling moto. Moto does not implement CRC64NVME checksums, so any call that includes @@ -543,7 +549,7 @@ def strip_checksum_algorithm(method: Callable): """ @functools.wraps(method) - def wrapper(*args, **kwargs): + def wrapper(*args: Any, **kwargs: Any) -> Any: """Remove the ChecksumAlgorithm argument from the call.""" kwargs.pop("ChecksumAlgorithm", None) return method(*args, **kwargs) @@ -577,7 +583,7 @@ def test_copy_file(mocked_s3_client_no_checksum: Any, destination: str) -> None: obj = mocked_s3_client_no_checksum.get_object(Bucket=destination, Key="dst/path/to/file.txt") assert obj["Body"].read() == b"copy me" - assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert response["ResponseMetadata"]["HTTPStatusCode"] == HTTP_STATUS_OK # delete_object @@ -592,12 +598,12 @@ def test_delete_object_removes_object(mock_s3_client: Any, bucket: str, protocol resp = delete_object(s3_path) assert object_exists(s3_path) is False - assert resp.get("ResponseMetadata", {}).get("HTTPStatusCode") == 204 + assert resp.get("ResponseMetadata", {}).get("HTTPStatusCode") == HTTP_STATUS_NO_CONTENT # retry the deletion resp = delete_object(s3_path) assert object_exists(s3_path) is False - assert resp.get("ResponseMetadata", {}).get("HTTPStatusCode") == 204 + assert resp.get("ResponseMetadata", {}).get("HTTPStatusCode") == HTTP_STATUS_NO_CONTENT # upload_file_with_metadata @@ -659,14 +665,15 @@ def test_head_object_returns_info(mock_s3_client: Any) -> None: mock_s3_client.put_object(Bucket=CDM_LAKE_BUCKET, Key="info/file.txt", Body=b"hello", Metadata={"md5": "abc123"}) result = head_object(f"{CDM_LAKE_BUCKET}/info/file.txt") assert result is not None - assert result["size"] == 5 + assert result["size"] == SIZE_HELLO assert result["metadata"]["md5"] == "abc123" # moto may not populate CRC64NVME, but the key should be present assert "checksum_crc64nvme" in result @pytest.mark.s3 -def test_head_object_returns_none_for_missing(mock_s3_client: Any) -> None: +@pytest.mark.usefixtures("mock_s3_client") +def test_head_object_returns_none_for_missing() -> None: """Verify that head_object returns None for a non-existent object.""" result = head_object(f"{CDM_LAKE_BUCKET}/does/not/exist.txt") assert result is None @@ -679,7 +686,7 @@ def test_head_object_with_protocols(mock_s3_client: Any, protocol: str) -> None: mock_s3_client.put_object(Bucket=CDM_LAKE_BUCKET, Key="proto/file.txt", Body=b"data") result = head_object(f"{protocol}{CDM_LAKE_BUCKET}/proto/file.txt") assert result is not None - assert result["size"] == 4 + assert result["size"] == SIZE_DATA # copy_object_with_metadata @@ -696,7 +703,7 @@ def test_copy_object_with_metadata_replaces_metadata(mocked_s3_client_no_checksu f"{destination}/archive/file.txt", metadata=new_metadata, ) - assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert response["ResponseMetadata"]["HTTPStatusCode"] == HTTP_STATUS_OK # verify the destination has the new metadata, not the old resp = mocked_s3_client_no_checksum.head_object(Bucket=destination, Key="archive/file.txt") From 3f0e6040ecdf3ca0d983f6c8209cb5e7b0582b50 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 20 Apr 2026 11:50:06 -0700 Subject: [PATCH 007/128] Potential fix for pull request finding 'Unused import' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- tests/ncbi_ftp/test_notebooks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/ncbi_ftp/test_notebooks.py b/tests/ncbi_ftp/test_notebooks.py index ba19c2a9..07b764e1 100644 --- a/tests/ncbi_ftp/test_notebooks.py +++ b/tests/ncbi_ftp/test_notebooks.py @@ -72,6 +72,8 @@ class TestManifestNotebookImports: def test_imports_resolve(self) -> None: """All manifest notebook imports are verified at module load time above.""" + assert isinstance(FTP_HOST, str) + assert FTP_HOST assert callable(download_assembly_summary) assert callable(write_updated_manifest) From 4f93d5f0685b3e72522fa905d3576e9f028da24d Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 20 Apr 2026 11:51:14 -0700 Subject: [PATCH 008/128] Potential fix for pull request finding 'Unused import' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- tests/ncbi_ftp/test_notebooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ncbi_ftp/test_notebooks.py b/tests/ncbi_ftp/test_notebooks.py index 07b764e1..d73b850c 100644 --- a/tests/ncbi_ftp/test_notebooks.py +++ b/tests/ncbi_ftp/test_notebooks.py @@ -22,7 +22,7 @@ DEFAULT_LAKEHOUSE_KEY_PREFIX, promote_from_s3, ) -from cdm_data_loaders.utils.s3 import get_s3_client, split_s3_path # noqa: F401 +from cdm_data_loaders.utils.s3 import split_s3_path # noqa: F401 NOTEBOOKS_DIR = Path(__file__).resolve().parents[2] / "notebooks" From 399a00d0ba69dc4838b2a1e75e888b7e4f765ba7 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 20 Apr 2026 11:51:50 -0700 Subject: [PATCH 009/128] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- scripts/entrypoint.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index ee087591..b5409887 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -1,9 +1,11 @@ #!/usr/bin/env bash set -euo pipefail +supported_commands="xml_split|uniref|uniprot|ncbi_rest_api|ncbi_ftp_sync|test|bash" + # Ensure at least one argument is provided if [ "$#" -eq 0 ]; then - echo "Usage: $0 {uniref|uniprot|ncbi_rest_api|ncbi_ftp_sync|xml_split|test} [args...]" + echo "Usage: $0 {$supported_commands} [args...]" exit 1 fi @@ -39,7 +41,7 @@ case "$cmd" in exec /usr/bin/tini -- /bin/bash ;; *) - echo "Error: unknown command '$cmd'; valid commands are 'uniref', 'uniprot', 'ncbi_rest_api', 'ncbi_ftp_sync', or 'xml_split'." >&2 + echo "Error: unknown command '$cmd'; valid commands are {$supported_commands}." >&2 exit 1 ;; esac From fee2ab4acb380179838b2ac94b663cb9583ade52 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 20 Apr 2026 11:52:52 -0700 Subject: [PATCH 010/128] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/cdm_data_loaders/ncbi_ftp/manifest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/manifest.py b/src/cdm_data_loaders/ncbi_ftp/manifest.py index f6a9fce1..ae587c28 100644 --- a/src/cdm_data_loaders/ncbi_ftp/manifest.py +++ b/src/cdm_data_loaders/ncbi_ftp/manifest.py @@ -251,7 +251,7 @@ def compute_diff( # noqa: PLR0912 def _ftp_dir_from_url(ftp_url: str, ftp_host: str = FTP_HOST) -> str: """Convert an FTP URL from the assembly summary to an FTP directory path.""" if ftp_url.startswith("https://"): - return ftp_url.replace("https://ftp.ncbi.nlm.nih.gov", "") + return ftp_url.replace(f"https://{ftp_host}", "") if ftp_url.startswith("ftp://"): return ftp_url.replace(f"ftp://{ftp_host}", "") return ftp_url From 5233d9dd9ca30b74c525a9dbe8d6a7dc96f055e7 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 20 Apr 2026 11:53:44 -0700 Subject: [PATCH 011/128] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/cdm_data_loaders/ncbi_ftp/promote.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index 52d9e57f..ccf6bc92 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -58,13 +58,14 @@ def promote_from_s3( # noqa: PLR0913 """ s3 = get_s3_client() paginator = s3.get_paginator("list_objects_v2") + normalized_staging_key_prefix = staging_key_prefix.rstrip("/") + "/" promoted = 0 failed = 0 # Collect all objects under the staging prefix staged_objects: list[str] = [] - for page in paginator.paginate(Bucket=bucket, Prefix=staging_key_prefix): + for page in paginator.paginate(Bucket=bucket, Prefix=normalized_staging_key_prefix): staged_objects.extend(obj["Key"] for obj in page.get("Contents", [])) # Separate data files from sidecars @@ -96,7 +97,7 @@ def promote_from_s3( # noqa: PLR0913 if staged_key.endswith("download_report.json"): continue - rel_path = staged_key[len(staging_key_prefix) :] + rel_path = staged_key[len(normalized_staging_key_prefix) :] if not rel_path.startswith("raw_data/"): continue final_key = lakehouse_key_prefix + rel_path From e5b58f24ff7f9e474867a014c2ffd0493a2112c1 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 20 Apr 2026 11:54:29 -0700 Subject: [PATCH 012/128] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/cdm_data_loaders/ncbi_ftp/promote.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index ccf6bc92..1763aba3 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -9,7 +9,7 @@ import re import tempfile from datetime import UTC, datetime -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any import botocore.exceptions @@ -120,11 +120,12 @@ def promote_from_s3( # noqa: PLR0913 md5_obj = s3.get_object(Bucket=bucket, Key=md5_key) metadata["md5"] = md5_obj["Body"].read().decode().strip() + final_key_path = PurePosixPath(final_key) upload_file_with_metadata( tmp_path, - f"{bucket}/{Path(final_key).parent}", + f"{bucket}/{final_key_path.parent}", metadata=metadata, - object_name=Path(final_key).name, + object_name=final_key_path.name, ) promoted += 1 From 64ead4c4d39889e7b26706df804ae7caf747de81 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 20 Apr 2026 11:55:18 -0700 Subject: [PATCH 013/128] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/cdm_data_loaders/utils/s3.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index c3897904..65942b20 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -387,7 +387,7 @@ def upload_file_with_metadata( :type metadata: dict[str, str] :param object_name: S3 object name; defaults to the local filename :type object_name: str | None - :return: True if the upload succeeded + :return: True if the upload succeeded, otherwise False :rtype: bool """ if isinstance(local_file_path, str): @@ -407,14 +407,17 @@ def upload_file_with_metadata( extra_args = {**DEFAULT_EXTRA_ARGS, "Metadata": metadata} file_size = local_file_path.stat().st_size - with tqdm.tqdm(total=file_size, unit="B", unit_scale=True, desc=str(local_file_path)) as pbar: - s3.upload_file( - Filename=str(local_file_path), - Bucket=bucket, - Key=key, - Callback=pbar.update, - ExtraArgs=extra_args, - ) + try: + with tqdm.tqdm(total=file_size, unit="B", unit_scale=True, desc=str(local_file_path)) as pbar: + s3.upload_file( + Filename=str(local_file_path), + Bucket=bucket, + Key=key, + Callback=pbar.update, + ExtraArgs=extra_args, + ) + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError): + return False return True From 9fba2c5ad4f4d0d6a148843b8135c553fff50a48 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 20 Apr 2026 11:55:45 -0700 Subject: [PATCH 014/128] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- scripts/s3_local.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/s3_local.py b/scripts/s3_local.py index 65f80396..5802a133 100755 --- a/scripts/s3_local.py +++ b/scripts/s3_local.py @@ -24,9 +24,10 @@ from pathlib import Path import boto3 +from botocore.client import BaseClient -def _client() -> boto3.client: +def _client() -> BaseClient: return boto3.client( "s3", endpoint_url=os.environ.get("MINIO_ENDPOINT_URL", "http://localhost:9000"), From c89ad37603da7a483b8d7048665e67f0eca47ef1 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 20 Apr 2026 12:02:16 -0700 Subject: [PATCH 015/128] address copilot comments --- tests/ncbi_ftp/test_notebooks.py | 1 + tests/utils/test_s3.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/ncbi_ftp/test_notebooks.py b/tests/ncbi_ftp/test_notebooks.py index d73b850c..1ff8f0e4 100644 --- a/tests/ncbi_ftp/test_notebooks.py +++ b/tests/ncbi_ftp/test_notebooks.py @@ -74,6 +74,7 @@ def test_imports_resolve(self) -> None: """All manifest notebook imports are verified at module load time above.""" assert isinstance(FTP_HOST, str) assert FTP_HOST + assert AssemblyRecord is not None assert callable(download_assembly_summary) assert callable(write_updated_manifest) diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index 9050448f..83a6efc6 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -558,8 +558,8 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: @pytest.fixture -def mocked_s3_client_no_checksum(mock_s3_client: Any) -> Generator[Any, Any]: - """Yield the mocked S3 client with copy_object patched to strip ChecksumAlgorithm. +def mocked_s3_client_no_checksum(mock_s3_client: Any) -> Any: + """Return the mocked S3 client with copy_object patched to strip ChecksumAlgorithm. This works around the moto limitation of not supporting CRC64NVME checksums, allowing copy_object calls that include ChecksumAlgorithm to succeed. From 9a5e2fe213e803ff02098101e6317e7eb13e058b Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 20 Apr 2026 12:03:41 -0700 Subject: [PATCH 016/128] increased timeout for trivy action --- .github/workflows/trivy.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index aa6735f4..ef7e0a1c 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -49,6 +49,7 @@ jobs: template: "@/contrib/sarif.tpl" output: "trivy-results.sarif" severity: "CRITICAL,HIGH" + timeout: "15m" - name: Upload Trivy scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@v3 From de77b144fff4b1e1fc32cf484c3afc44d0fa2e9e Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 20 Apr 2026 12:04:14 -0700 Subject: [PATCH 017/128] Potential fix for pull request finding 'Unused import' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- tests/ncbi_ftp/test_notebooks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ncbi_ftp/test_notebooks.py b/tests/ncbi_ftp/test_notebooks.py index 1ff8f0e4..3a8c1cb2 100644 --- a/tests/ncbi_ftp/test_notebooks.py +++ b/tests/ncbi_ftp/test_notebooks.py @@ -76,6 +76,7 @@ def test_imports_resolve(self) -> None: assert FTP_HOST assert AssemblyRecord is not None assert callable(download_assembly_summary) + assert callable(compute_diff) assert callable(write_updated_manifest) From 7d9ced1faff4d8ec041a22aa7b0b8e574cd6d855 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 20 Apr 2026 12:04:46 -0700 Subject: [PATCH 018/128] Potential fix for pull request finding 'Unused import' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- tests/ncbi_ftp/test_notebooks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ncbi_ftp/test_notebooks.py b/tests/ncbi_ftp/test_notebooks.py index 3a8c1cb2..6d2e3758 100644 --- a/tests/ncbi_ftp/test_notebooks.py +++ b/tests/ncbi_ftp/test_notebooks.py @@ -87,3 +87,4 @@ def test_imports_resolve(self) -> None: """All promote notebook imports are verified at module load time above.""" assert callable(promote_from_s3) assert isinstance(DEFAULT_LAKEHOUSE_KEY_PREFIX, str) + assert callable(split_s3_path) From 16dfda3b18432c07aff78c0c9ab9c773d50ef7f0 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 20 Apr 2026 12:11:36 -0700 Subject: [PATCH 019/128] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/cdm_data_loaders/ncbi_ftp/promote.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index 1763aba3..6faf8c79 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -121,12 +121,17 @@ def promote_from_s3( # noqa: PLR0913 metadata["md5"] = md5_obj["Body"].read().decode().strip() final_key_path = PurePosixPath(final_key) - upload_file_with_metadata( + upload_succeeded = upload_file_with_metadata( tmp_path, f"{bucket}/{final_key_path.parent}", metadata=metadata, object_name=final_key_path.name, ) + if not upload_succeeded: + logger.error("Failed to upload promoted file %s to %s", staged_key, final_key) + failed += 1 + continue + promoted += 1 # Track promoted accession for manifest trimming From 521978d0855e7d4052f9a740743ed5269dc52f43 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 20 Apr 2026 12:12:24 -0700 Subject: [PATCH 020/128] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tests/integration/conftest.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 33e53519..95e60008 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -17,6 +17,7 @@ import boto3 import botocore.client +import botocore.config import pytest import cdm_data_loaders.ncbi_ftp.manifest as manifest_mod @@ -48,6 +49,11 @@ def _minio_reachable() -> bool: endpoint_url=MINIO_ENDPOINT_URL, aws_access_key_id=MINIO_ACCESS_KEY, aws_secret_access_key=MINIO_SECRET_KEY, + config=botocore.config.Config( + connect_timeout=1, + read_timeout=1, + retries={"max_attempts": 1}, + ), ) client.list_buckets() except Exception: # noqa: BLE001 From 8f73674106da5a1fde8cde0dd0861bb293007c26 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 20 Apr 2026 12:13:15 -0700 Subject: [PATCH 021/128] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/cdm_data_loaders/pipelines/ncbi_ftp_download.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py index 616afee4..30a4b61b 100644 --- a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py +++ b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py @@ -15,7 +15,6 @@ from typing import Any from pydantic import AliasChoices, Field, field_validator -from pydantic_settings import CliSuppress from cdm_data_loaders.ncbi_ftp.assembly import FTP_HOST, download_assembly_to_local from cdm_data_loaders.pipelines.core import run_cli @@ -54,7 +53,7 @@ class DownloadSettings(CtsDefaultSettings): description="NCBI FTP hostname", validation_alias=AliasChoices("ftp-host", "ftp_host"), ) - limit: CliSuppress[int | None] = Field( + limit: int | None = Field( default=None, ge=1, description="Limit to first N assemblies (for testing)", From 1e8f363d7dc93b02e3645c0ce105c77eb41c19c5 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 20 Apr 2026 12:42:48 -0700 Subject: [PATCH 022/128] add progress bar to manifest verification --- notebooks/ncbi_ftp_manifest.ipynb | 16 ++++++++++++++-- src/cdm_data_loaders/ncbi_ftp/manifest.py | 19 +++++++++++++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb index e59c6043..1608f4e2 100644 --- a/notebooks/ncbi_ftp_manifest.ipynb +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -198,9 +198,18 @@ "# ── Verify against Lakehouse ──\n", "if STORE_BUCKET:\n", " from cdm_data_loaders.ncbi_ftp.manifest import verify_transfer_candidates\n", + " from tqdm.notebook import tqdm\n", "\n", " candidates = diff.new + diff.updated\n", - " print(f\"Verifying {len(candidates)} candidates against s3://{STORE_BUCKET}/{STORE_KEY_PREFIX} ...\")\n", + " total = len(candidates)\n", + " print(f\"Verifying {total} candidates against s3://{STORE_BUCKET}/{STORE_KEY_PREFIX} ...\")\n", + "\n", + " progress = tqdm(total=total, unit=\"assembly\", desc=\"Verifying checksums\")\n", + "\n", + " def _update_progress(done: int, _total: int, acc: str) -> None:\n", + " progress.update(1)\n", + " progress.set_postfix(acc=acc, refresh=False)\n", + "\n", " confirmed = set(\n", " verify_transfer_candidates(\n", " candidates,\n", @@ -208,8 +217,11 @@ " STORE_BUCKET,\n", " STORE_KEY_PREFIX,\n", " ftp_host=FTP_HOST,\n", + " progress_callback=_update_progress,\n", " )\n", " )\n", + " progress.close()\n", + "\n", " before = len(diff.new) + len(diff.updated)\n", " diff.new = [a for a in diff.new if a in confirmed]\n", " diff.updated = [a for a in diff.updated if a in confirmed]\n", @@ -228,7 +240,7 @@ " diff.new = [a for a in diff.new if a in limited_set]\n", " diff.updated = [a for a in diff.updated if a in limited_set]\n", " print(f\"After limit ({LIMIT}): {len(diff.new)} new, {len(diff.updated)} updated\")\n", - " print(f\" (was {original_new} new, {original_updated} updated)\")" + " print(f\" (was {original_new} new, {original_updated} updated)\")\n" ] }, { diff --git a/src/cdm_data_loaders/ncbi_ftp/manifest.py b/src/cdm_data_loaders/ncbi_ftp/manifest.py index ae587c28..62960ad1 100644 --- a/src/cdm_data_loaders/ncbi_ftp/manifest.py +++ b/src/cdm_data_loaders/ncbi_ftp/manifest.py @@ -12,7 +12,7 @@ import json import re import time -from collections.abc import Iterable +from collections.abc import Callable, Iterable from dataclasses import dataclass, field from datetime import UTC, datetime from pathlib import Path @@ -266,6 +266,7 @@ def verify_transfer_candidates( # noqa: PLR0912, PLR0915 bucket: str, key_prefix: str, ftp_host: str = FTP_HOST, + progress_callback: Callable[[int, int, str], None] | None = None, ) -> list[str]: """Verify which transfer candidates actually need downloading. @@ -282,6 +283,9 @@ def verify_transfer_candidates( # noqa: PLR0912, PLR0915 :param bucket: S3 bucket name :param key_prefix: S3 key prefix for the Lakehouse dataset root :param ftp_host: NCBI FTP hostname + :param progress_callback: optional callable invoked after each accession is + processed with ``(done, total, accession)`` so callers can display a + progress bar. ``done`` is the 1-based count of completed accessions. :return: filtered list of accessions that actually need downloading """ if not accessions: @@ -295,10 +299,12 @@ def verify_transfer_candidates( # noqa: PLR0912, PLR0915 last_activity = time.monotonic() try: - for acc in accessions: + for done, acc in enumerate(accessions, start=1): rec = current_assemblies.get(acc) if not rec: confirmed.append(acc) + if progress_callback is not None: + progress_callback(done, len(accessions), acc) continue # Build S3 prefix for this assembly @@ -311,6 +317,8 @@ def verify_transfer_candidates( # noqa: PLR0912, PLR0915 # Nothing in the store — definitely needs downloading confirmed.append(acc) skipped_missing += 1 + if progress_callback is not None: + progress_callback(done, len(accessions), acc) continue # Objects exist — need FTP md5 checksums to decide @@ -327,6 +335,8 @@ def verify_transfer_candidates( # noqa: PLR0912, PLR0915 except Exception: # noqa: BLE001 logger.warning("Cannot fetch md5checksums.txt for %s, keeping in transfer list", acc) confirmed.append(acc) + if progress_callback is not None: + progress_callback(done, len(accessions), acc) continue # Filter to files we'd actually download @@ -338,6 +348,8 @@ def verify_transfer_candidates( # noqa: PLR0912, PLR0915 if not target_checksums: confirmed.append(acc) + if progress_callback is not None: + progress_callback(done, len(accessions), acc) continue # Short-circuit: if any file differs or is missing, keep the assembly @@ -361,6 +373,9 @@ def verify_transfer_candidates( # noqa: PLR0912, PLR0915 else: pruned += 1 logger.debug("Pruned %s — all files match S3 checksums", acc) + + if progress_callback is not None: + progress_callback(done, len(accessions), acc) finally: if ftp is not None: with contextlib.suppress(Exception): From 7326641a34555676d9a0a07af9e6a2bbff8b17ea Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 20 Apr 2026 14:58:34 -0700 Subject: [PATCH 023/128] add synthetic assembly summary generation option --- docs/ncbi_ftp_e2e_walkthrough.md | 38 +++++ notebooks/ncbi_ftp_manifest.ipynb | 86 +++++++++-- src/cdm_data_loaders/ncbi_ftp/manifest.py | 106 +++++++++++++ tests/integration/test_manifest_e2e.py | 87 +++++++++++ tests/ncbi_ftp/test_manifest.py | 180 ++++++++++++++++++++++ 5 files changed, 486 insertions(+), 11 deletions(-) diff --git a/docs/ncbi_ftp_e2e_walkthrough.md b/docs/ncbi_ftp_e2e_walkthrough.md index 40b467a7..ffce0e3e 100644 --- a/docs/ncbi_ftp_e2e_walkthrough.md +++ b/docs/ncbi_ftp_e2e_walkthrough.md @@ -140,6 +140,44 @@ be skipped — though on repeat runs you should set `STORE_BUCKET` so assemblies already promoted to the Lakehouse are pruned from the transfer manifest. +### Optional: Bootstrap from existing store (Cell 5) + +If you have a pre-populated S3 store but lack a baseline assembly summary, +you can scan the store to generate a synthetic baseline. This is especially +useful for large stores (100K+ assemblies) where verifying against FTP +checksums would take days. + +**When to use this:** +- First run against an existing, pre-populated store +- You want to start diffing without waiting for checksum verification +- You don't have a previous assembly summary snapshot to compare against + +**How it works:** +1. Set `SCAN_STORE = True` in Cell 5 +2. The notebook scans all objects under `s3://{STORE_BUCKET}/{STORE_KEY_PREFIX}` +3. For each unique assembly found, it extracts the accession and uses the + earliest object `LastModified` as a conservative `seq_rel_date` +4. It saves the synthetic summary to `LOCAL_SYNTHETIC_SUMMARY` (default: + `output/synthetic_summary_from_store.txt`) +5. This becomes the baseline for diffing; subsequent runs can load this + file as `PREVIOUS_SUMMARY_URI` + +**Example (for a 500K-assembly store):** +```python +SCAN_STORE = True +STORE_BUCKET = "cdm-lake" +STORE_KEY_PREFIX = "tenant-general-warehouse/kbase/datasets/ncbi/" +LOCAL_SYNTHETIC_SUMMARY = Path("output/synthetic_summary_from_store.txt") + +# After running Cell 5, upload the result to S3 for future runs: +# s3 cp output/synthetic_summary_from_store.txt s3://cdm-lake/assembly_summaries/synthetic_base.txt +# Then in future runs, set: +# PREVIOUS_SUMMARY_URI = "s3://cdm-lake/assembly_summaries/synthetic_base.txt" +``` + +**Performance:** Scanning typically takes 5–10 minutes for 500K assemblies +(vs. ~6 days of checksum verification). + ### Run the notebook Execute all cells in order. After Cell 7 finishes you should see files in diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb index 1608f4e2..34e42355 100644 --- a/notebooks/ncbi_ftp_manifest.ipynb +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -132,23 +132,87 @@ { "cell_type": "code", "execution_count": null, - "id": "88954378", + "id": "ceb5af5a", "metadata": {}, "outputs": [], "source": [ - "\"\"\"Load previous summary from S3 (or start fresh).\"\"\"\n", + "\"\"\"Optional: Bootstrap baseline by scanning current store (if no previous summary available).\n", "\n", - "previous: dict[str, AssemblyRecord] | None = None\n", + "If you have a pre-populated S3 store but no previous assembly summary snapshot,\n", + "you can scan the store to generate a synthetic summary. This becomes the baseline\n", + "for the diff.\n", "\n", - "if PREVIOUS_SUMMARY_URI:\n", - " s3 = get_s3_client()\n", - " bucket, key = split_s3_path(PREVIOUS_SUMMARY_URI)\n", - " resp = s3.get_object(Bucket=bucket, Key=key)\n", - " prev_text = resp[\"Body\"].read().decode(\"utf-8\")\n", - " previous = parse_assembly_summary(prev_text)\n", - " print(f\"Loaded {len(previous)} assemblies from previous snapshot\")\n", + "Set SCAN_STORE=True below to enable. The scan will:\n", + " 1. List all objects under STORE_BUCKET/STORE_KEY_PREFIX\n", + " 2. Extract accessions and use earliest LastModified as seq_rel_date (conservative)\n", + " 3. Build AssemblyRecord for each assembly found\n", + " 4. Save to LOCAL_SYNTHETIC_SUMMARY for re-use in future runs\n", + "\n", + "Typical use case: First run against 500K+ existing assemblies. Scanning takes\n", + "~5 minutes instead of ~6 days of checksum verification.\n", + "\"\"\"\n", + "\n", + "SCAN_STORE = False # Set to True to scan your store\n", + "LOCAL_SYNTHETIC_SUMMARY = Path(\"output/synthetic_summary_from_store.txt\")\n", + "\n", + "if SCAN_STORE and STORE_BUCKET:\n", + " from cdm_data_loaders.ncbi_ftp.manifest import scan_store_to_synthetic_summary\n", + " from tqdm.notebook import tqdm\n", + "\n", + " print(f\"Scanning s3://{STORE_BUCKET}/{STORE_KEY_PREFIX} for existing assemblies ...\")\n", + " progress = tqdm(unit=\"assembly\", desc=\"Scanning store\")\n", + "\n", + " def _track_scan(count: int, acc: str) -> None:\n", + " progress.update(1)\n", + " progress.set_postfix(acc=acc, refresh=False)\n", + "\n", + " synthetic = scan_store_to_synthetic_summary(STORE_BUCKET, STORE_KEY_PREFIX, progress_callback=_track_scan)\n", + " progress.close()\n", + "\n", + " print(f\"Found {len(synthetic)} assemblies in store\")\n", + "\n", + " # Save synthetic summary to file for future runs\n", + " with LOCAL_SYNTHETIC_SUMMARY.open(\"w\") as f:\n", + " for acc in sorted(synthetic.keys()):\n", + " rec = synthetic[acc]\n", + " f.write(\n", + " f\"{rec.accession}\\t.\\t.\\t.\\t.\\t.\\t.\\t.\\t.\\t.\\t{rec.status}\\t.\\t.\\t.\\t{rec.seq_rel_date}\\t.\\t.\\t.\\t.\\t{rec.ftp_path}\\t.\\n\"\n", + " )\n", + " print(f\"Saved synthetic summary to {LOCAL_SYNTHETIC_SUMMARY}\")\n", + "\n", + " # Use it as the previous baseline\n", + " previous = synthetic\n", "else:\n", - " print(\"No previous snapshot — all current 'latest' assemblies will be marked as new\")" + " if SCAN_STORE:\n", + " print(\"SCAN_STORE=True but STORE_BUCKET not set. Skipping.\")\n", + " print(\"Skipping store scan (SCAN_STORE=False). Will load/use PREVIOUS_SUMMARY_URI instead.\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "88954378", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Load previous summary from S3 (or from synthetic scan, or start fresh).\n", + "\n", + "If you ran the store scan in the previous cell, SCAN_STORE=True above will have\n", + "set `previous` already. Otherwise, try to load from PREVIOUS_SUMMARY_URI.\n", + "\"\"\"\n", + "\n", + "if \"previous\" not in locals() or previous is None:\n", + " # Store scan didn't run, or was skipped. Try to load from S3.\n", + " if PREVIOUS_SUMMARY_URI:\n", + " s3 = get_s3_client()\n", + " bucket, key = split_s3_path(PREVIOUS_SUMMARY_URI)\n", + " resp = s3.get_object(Bucket=bucket, Key=key)\n", + " prev_text = resp[\"Body\"].read().decode(\"utf-8\")\n", + " previous = parse_assembly_summary(prev_text)\n", + " print(f\"Loaded {len(previous)} assemblies from previous snapshot\")\n", + " else:\n", + " print(\"No previous snapshot and SCAN_STORE=False — all current 'latest' assemblies will be marked as new\")\n", + " previous = None\n" ] }, { diff --git a/src/cdm_data_loaders/ncbi_ftp/manifest.py b/src/cdm_data_loaders/ncbi_ftp/manifest.py index 62960ad1..ebce8dce 100644 --- a/src/cdm_data_loaders/ncbi_ftp/manifest.py +++ b/src/cdm_data_loaders/ncbi_ftp/manifest.py @@ -257,6 +257,112 @@ def _ftp_dir_from_url(ftp_url: str, ftp_host: str = FTP_HOST) -> str: return ftp_url +# ── Synthetic summary from S3 store scan ──────────────────────────────── + + +def _extract_accession_from_s3_key(key: str) -> str | None: + """Extract the assembly accession from an S3 object key. + + Looks for the pattern GCF_######.# or GCA_######.# in the key path. + + :param key: S3 object key + :return: accession (e.g. "GCF_000001215.4") or None if not found + """ + m = re.search(r"(GC[AF]_\d{3}\d{6}\.\d+)", key) + return m.group(1) if m else None + + +def _extract_assembly_dir_from_s3_key(key: str) -> str | None: + """Extract the assembly directory name from an S3 object key. + + The assembly directory is the path component that follows the accession + and contains assembly metadata (e.g. "GCF_000001215.4_Release_6_plus_ISO1_MT"). + + :param key: S3 object key + :return: assembly directory name or None if not found + """ + # Match accession followed by underscore and then capture until next / + m = re.search(r"(GC[AF]_\d{3}\d{6}\.\d+[^/]*)/", key) + return m.group(1) if m else None + + +def scan_store_to_synthetic_summary( + bucket: str, + key_prefix: str, + progress_callback: Callable[[int, str], None] | None = None, +) -> dict[str, AssemblyRecord]: + """Scan S3 store and build a synthetic assembly summary from existing objects. + + This function is useful when bootstrapping a diffs against an existing, + pre-populated S3 store that lacks a baseline assembly summary. + + For each assembly found in the store: + - Extracts the accession and assembly directory name from S3 paths + - Uses the earliest ``LastModified`` timestamp across all files in that + assembly as the synthetic ``seq_rel_date`` (conservative estimate) + - Creates an ``AssemblyRecord`` with ``status="latest"`` + + The function paginates through S3 to handle large stores efficiently. + + :param bucket: S3 bucket name + :param key_prefix: S3 key prefix (all objects under this prefix are scanned) + :param progress_callback: optional callable invoked after each accession is + processed with ``(count, accession)`` where count is the running total + of unique accessions found + :return: dict mapping accession to ``AssemblyRecord`` + """ + s3 = get_s3_client() + assemblies: dict[str, AssemblyRecord] = {} + processed_count = 0 + + try: + paginator = s3.get_paginator("list_objects_v2") + pages = paginator.paginate(Bucket=bucket, Prefix=key_prefix) + + for page in pages: + for obj in page.get("Contents", []): + acc = _extract_accession_from_s3_key(obj["Key"]) + assembly_dir = _extract_assembly_dir_from_s3_key(obj["Key"]) + + if not acc or not assembly_dir: + continue + + # Convert LastModified to NCBI date format (YYYY/MM/DD) + last_modified = obj["LastModified"] + # Handle both aware and naive datetimes + if last_modified.tzinfo is None: + last_modified = last_modified.replace(tzinfo=UTC) + obj_date_str = last_modified.strftime("%Y/%m/%d") + + if acc not in assemblies: + # First object for this accession; store it + assemblies[acc] = AssemblyRecord( + accession=acc, + status="latest", + seq_rel_date=obj_date_str, + ftp_path="", # synthesized; empty as it's not from FTP + assembly_dir=assembly_dir, + ) + processed_count += 1 + if progress_callback is not None: + progress_callback(processed_count, acc) + else: + # Update to earliest timestamp (conservative) + existing_record = assemblies[acc] + existing_date = datetime.strptime(existing_record.seq_rel_date, "%Y/%m/%d").replace( + tzinfo=UTC + ) + if last_modified < existing_date: + existing_record.seq_rel_date = obj_date_str + + except Exception as e: # noqa: BLE001 + logger.error("Error scanning store: %s", e) + raise + + logger.info("Scanned S3 store: found %d unique assemblies", len(assemblies)) + return assemblies + + # ── Checksum verification against S3 store ─────────────────────────────── diff --git a/tests/integration/test_manifest_e2e.py b/tests/integration/test_manifest_e2e.py index df25d410..4ab208f0 100644 --- a/tests/integration/test_manifest_e2e.py +++ b/tests/integration/test_manifest_e2e.py @@ -18,6 +18,7 @@ download_assembly_summary, filter_by_prefix_range, parse_assembly_summary, + scan_store_to_synthetic_summary, verify_transfer_candidates, write_diff_summary, write_removed_manifest, @@ -209,3 +210,89 @@ def test_prunes_existing_matching_md5( remaining_candidates = [c for c in candidates if c != acc] for c in remaining_candidates: assert c in result, f"Expected {c} to remain (not seeded)" + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestScanStoreToSyntheticSummary: + """Test synthetic assembly summary generation from MinIO store.""" + + def test_builds_summary_from_minio_store( + self, + minio_s3_client: object, + test_bucket: str, + ) -> None: + """Verify synthetic summary captures assemblies from MinIO.""" + s3 = minio_s3_client + path_prefix = DEFAULT_LAKEHOUSE_KEY_PREFIX + + # Seed MinIO with a couple of assemblies + assemblies = { + "GCF_000001215.4_v1": ["_genomic.fna.gz", "_protein.faa.gz"], + "GCF_000005845.2_v2": ["_genomic.fna.gz"], + } + + for assembly_dir, files in assemblies.items(): + for fname in files: + key = f"{path_prefix}refseq/{assembly_dir}/{assembly_dir}{fname}" + s3.put_object( + Bucket=test_bucket, + Key=key, + Body=b"placeholder", + ) + + # Scan the store + result = scan_store_to_synthetic_summary(test_bucket, path_prefix) + + # Should have found both assemblies + assert "GCF_000001215.4" in result + assert "GCF_000005845.2" in result + + # Verify basic record structure + rec1 = result["GCF_000001215.4"] + assert rec1.accession == "GCF_000001215.4" + assert rec1.status == "latest" + assert rec1.assembly_dir == "GCF_000001215.4_v1" + + def test_synthetic_summary_diff_against_current( + self, + minio_s3_client: object, + test_bucket: str, + ) -> None: + """Verify synthetic summary can be used as baseline for diffing.""" + s3 = minio_s3_client + path_prefix = DEFAULT_LAKEHOUSE_KEY_PREFIX + + # Seed MinIO with one assembly + key1 = f"{path_prefix}refseq/GCF_000001215.4_old/GCF_000001215.4_old_genomic.fna.gz" + s3.put_object(Bucket=test_bucket, Key=key1, Body=b"data") + + # Build synthetic summary from store + synthetic = scan_store_to_synthetic_summary(test_bucket, path_prefix) + assert "GCF_000001215.4" in synthetic + + # Simulate current NCBI summary with one new and one existing + current = { + "GCF_000001215.4": AssemblyRecord( + accession="GCF_000001215.4", + status="latest", + seq_rel_date=synthetic["GCF_000001215.4"].seq_rel_date, + ftp_path="", + assembly_dir="GCF_000001215.4_old", + ), + "GCF_000005845.2": AssemblyRecord( + accession="GCF_000005845.2", + status="latest", + seq_rel_date="2024/04/20", + ftp_path="", + assembly_dir="GCF_000005845.2_new", + ), + } + + # Compute diff + diff = compute_diff(current, previous_assemblies=synthetic) + + # Should find one new and zero updated + assert "GCF_000005845.2" in diff.new + assert "GCF_000001215.4" not in diff.new # Already in store + assert len(diff.updated) == 0 # Same date, same dir diff --git a/tests/ncbi_ftp/test_manifest.py b/tests/ncbi_ftp/test_manifest.py index dac0400f..82ed14c0 100644 --- a/tests/ncbi_ftp/test_manifest.py +++ b/tests/ncbi_ftp/test_manifest.py @@ -6,12 +6,15 @@ from cdm_data_loaders.ncbi_ftp.manifest import ( DiffResult, + _extract_accession_from_s3_key, + _extract_assembly_dir_from_s3_key, _ftp_dir_from_url, accession_prefix, compute_diff, filter_by_prefix_range, get_latest_assembly_paths, parse_assembly_summary, + scan_store_to_synthetic_summary, verify_transfer_candidates, write_diff_summary, write_removed_manifest, @@ -559,3 +562,180 @@ def test_skips_ftp_when_folder_missing_from_store( assert result == ["GCF_000001215.4"] # FTP should never have been connected (lazy init) mock_connect.assert_not_called() + + +# ── Synthetic summary from S3 store scan ──────────────────────────────── + + +class TestExtractAccessionFromS3Key: + """Test accession extraction from S3 paths.""" + + @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") + def test_extracts_accession_from_path(self, _mock_s3: MagicMock) -> None: + """Verify accession is extracted correctly from S3 keys.""" + assert _extract_accession_from_s3_key( + "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6_plus_ISO1_MT/file.gz" + ) == "GCF_000001215.4" + assert _extract_accession_from_s3_key( + "some/path/GCA_999999999.1_whatever/data.txt" + ) == "GCA_999999999.1" + + def test_returns_none_for_invalid_path(self) -> None: + """Verify None is returned when no accession is found.""" + assert _extract_accession_from_s3_key("some/random/path") is None + assert _extract_accession_from_s3_key("") is None + + +class TestExtractAssemblyDirFromS3Key: + """Test assembly directory extraction from S3 paths.""" + + def test_extracts_assembly_dir(self) -> None: + """Verify assembly directory is extracted correctly from S3 keys.""" + assert _extract_assembly_dir_from_s3_key( + "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6_plus_ISO1_MT/file.gz" + ) == "GCF_000001215.4_Release_6_plus_ISO1_MT" + assert _extract_assembly_dir_from_s3_key( + "prefix/GCA_999999999.1_assembly_name/subdir/data.txt" + ) == "GCA_999999999.1_assembly_name" + + def test_returns_none_for_invalid_path(self) -> None: + """Verify None is returned when no assembly directory is found.""" + assert _extract_assembly_dir_from_s3_key("some/random/path") is None + assert _extract_assembly_dir_from_s3_key("") is None + + +class TestScanStoreToSyntheticSummary: + """Test synthetic assembly summary generation from S3 store scan.""" + + def _mock_s3_with_objects(self) -> MagicMock: + """Return a mock S3 client with assembly objects.""" + from datetime import datetime, timezone + + mock = MagicMock() + mock_paginator = MagicMock() + mock.get_paginator.return_value = mock_paginator + + # Mock objects from two assemblies + page_contents = [ + { + "Key": "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6/file1.gz", + "LastModified": datetime(2024, 1, 15, tzinfo=timezone.utc), + }, + { + "Key": "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6/file2.gz", + "LastModified": datetime(2024, 1, 16, tzinfo=timezone.utc), + }, + { + "Key": "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000005845.2_Assembly/file.gz", + "LastModified": datetime(2024, 2, 20, tzinfo=timezone.utc), + }, + ] + mock_paginator.paginate.return_value = [{"Contents": page_contents}] + return mock + + @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") + def test_builds_summary_from_store(self, mock_get_s3: MagicMock) -> None: + """Verify synthetic summary is built correctly from S3 objects.""" + mock_get_s3.return_value = self._mock_s3_with_objects() + + result = scan_store_to_synthetic_summary("test-bucket", "prefix/") + + assert len(result) == 2 + assert "GCF_000001215.4" in result + assert "GCF_000005845.2" in result + + # Should use earliest date (2024-01-15) + rec1 = result["GCF_000001215.4"] + assert rec1.accession == "GCF_000001215.4" + assert rec1.status == "latest" + assert rec1.seq_rel_date == "2024/01/15" + assert rec1.assembly_dir == "GCF_000001215.4_Release_6" + + # Other assembly uses its single date + rec2 = result["GCF_000005845.2"] + assert rec2.seq_rel_date == "2024/02/20" + + @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") + def test_uses_earliest_date_per_assembly(self, mock_get_s3: MagicMock) -> None: + """Verify earliest LastModified is used when assembly has multiple files.""" + mock = MagicMock() + mock_paginator = MagicMock() + mock.get_paginator.return_value = mock_paginator + + from datetime import datetime, timezone + + # Files from same assembly with different dates + page_contents = [ + { + "Key": "prefix/GCF_000001215.4_v1/file_newer.gz", + "LastModified": datetime(2024, 3, 20, tzinfo=timezone.utc), + }, + { + "Key": "prefix/GCF_000001215.4_v1/file_older.gz", + "LastModified": datetime(2024, 1, 10, tzinfo=timezone.utc), + }, + ] + mock_paginator.paginate.return_value = [{"Contents": page_contents}] + mock_get_s3.return_value = mock + + result = scan_store_to_synthetic_summary("test-bucket", "prefix/") + + # Should use the earliest date + assert result["GCF_000001215.4"].seq_rel_date == "2024/01/10" + + @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") + def test_invokes_progress_callback(self, mock_get_s3: MagicMock) -> None: + """Verify progress callback is called for each unique assembly.""" + mock_get_s3.return_value = self._mock_s3_with_objects() + callback_calls = [] + + def track_progress(count: int, acc: str) -> None: + callback_calls.append((count, acc)) + + scan_store_to_synthetic_summary("test-bucket", "prefix/", progress_callback=track_progress) + + # Should have 2 calls (one per assembly discovered) + assert len(callback_calls) == 2 + assert callback_calls[0][0] == 1 # first assembly + assert callback_calls[1][0] == 2 # second assembly + + @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") + def test_handles_empty_store(self, mock_get_s3: MagicMock) -> None: + """Verify function handles empty store gracefully.""" + mock = MagicMock() + mock_paginator = MagicMock() + mock.get_paginator.return_value = mock_paginator + mock_paginator.paginate.return_value = [{"Contents": []}] + mock_get_s3.return_value = mock + + result = scan_store_to_synthetic_summary("test-bucket", "prefix/") + + assert result == {} + + @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") + def test_skips_objects_without_accession(self, mock_get_s3: MagicMock) -> None: + """Verify objects without valid accessions are skipped.""" + from datetime import datetime, timezone + + mock = MagicMock() + mock_paginator = MagicMock() + mock.get_paginator.return_value = mock_paginator + + page_contents = [ + { + "Key": "prefix/some/random/file.txt", # No accession + "LastModified": datetime(2024, 1, 1, tzinfo=timezone.utc), + }, + { + "Key": "prefix/GCF_000001215.4_Assembly/valid_file.gz", + "LastModified": datetime(2024, 2, 1, tzinfo=timezone.utc), + }, + ] + mock_paginator.paginate.return_value = [{"Contents": page_contents}] + mock_get_s3.return_value = mock + + result = scan_store_to_synthetic_summary("test-bucket", "prefix/") + + # Only one valid assembly should be found + assert len(result) == 1 + assert "GCF_000001215.4" in result From 4cf60d6f23805ecf0ebc7aa8aa364445188650b4 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 20 Apr 2026 15:48:51 -0700 Subject: [PATCH 024/128] debug synthetic manifest --- notebooks/ncbi_ftp_manifest.ipynb | 47 ++++++++++++------- src/cdm_data_loaders/ncbi_ftp/manifest.py | 11 +++-- tests/ncbi_ftp/test_manifest.py | 57 ++++++++++++++++++++++- 3 files changed, 92 insertions(+), 23 deletions(-) diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb index 34e42355..1d822445 100644 --- a/notebooks/ncbi_ftp_manifest.ipynb +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -152,22 +152,25 @@ "~5 minutes instead of ~6 days of checksum verification.\n", "\"\"\"\n", "\n", - "SCAN_STORE = False # Set to True to scan your store\n", + "SCAN_STORE = True # Set to True to scan your store\n", "LOCAL_SYNTHETIC_SUMMARY = Path(\"output/synthetic_summary_from_store.txt\")\n", "\n", "if SCAN_STORE and STORE_BUCKET:\n", " from cdm_data_loaders.ncbi_ftp.manifest import scan_store_to_synthetic_summary\n", + " from IPython.display import display\n", " from tqdm.notebook import tqdm\n", "\n", " print(f\"Scanning s3://{STORE_BUCKET}/{STORE_KEY_PREFIX} for existing assemblies ...\")\n", - " progress = tqdm(unit=\"assembly\", desc=\"Scanning store\")\n", + " progress = tqdm(unit=\"assembly\", desc=\"Scanning store\", leave=True)\n", + " display(progress)\n", "\n", " def _track_scan(count: int, acc: str) -> None:\n", - " progress.update(1)\n", + " progress.n = count\n", " progress.set_postfix(acc=acc, refresh=False)\n", + " progress.refresh()\n", "\n", " synthetic = scan_store_to_synthetic_summary(STORE_BUCKET, STORE_KEY_PREFIX, progress_callback=_track_scan)\n", - " progress.close()\n", + " progress.refresh()\n", "\n", " print(f\"Found {len(synthetic)} assemblies in store\")\n", "\n", @@ -259,32 +262,40 @@ "that genuinely need downloading.\n", "\"\"\"\n", "\n", - "# ── Verify against Lakehouse ──\n", + "# -- Verify against Lakehouse --\n", "if STORE_BUCKET:\n", " from cdm_data_loaders.ncbi_ftp.manifest import verify_transfer_candidates\n", + " from IPython.display import display\n", " from tqdm.notebook import tqdm\n", "\n", " candidates = diff.new + diff.updated\n", " total = len(candidates)\n", " print(f\"Verifying {total} candidates against s3://{STORE_BUCKET}/{STORE_KEY_PREFIX} ...\")\n", "\n", - " progress = tqdm(total=total, unit=\"assembly\", desc=\"Verifying checksums\")\n", + " progress = tqdm(total=total, unit=\"assembly\", desc=\"Verifying checksums\", leave=True)\n", + " display(progress)\n", "\n", " def _update_progress(done: int, _total: int, acc: str) -> None:\n", - " progress.update(1)\n", + " progress.n = done\n", " progress.set_postfix(acc=acc, refresh=False)\n", + " progress.refresh()\n", "\n", - " confirmed = set(\n", - " verify_transfer_candidates(\n", - " candidates,\n", - " filtered,\n", - " STORE_BUCKET,\n", - " STORE_KEY_PREFIX,\n", - " ftp_host=FTP_HOST,\n", - " progress_callback=_update_progress,\n", + " if total == 0:\n", + " print(\"No candidates to verify; skipping checksum checks.\")\n", + " confirmed = set()\n", + " else:\n", + " confirmed = set(\n", + " verify_transfer_candidates(\n", + " candidates,\n", + " filtered,\n", + " STORE_BUCKET,\n", + " STORE_KEY_PREFIX,\n", + " ftp_host=FTP_HOST,\n", + " progress_callback=_update_progress,\n", + " )\n", " )\n", - " )\n", - " progress.close()\n", + "\n", + " progress.refresh()\n", "\n", " before = len(diff.new) + len(diff.updated)\n", " diff.new = [a for a in diff.new if a in confirmed]\n", @@ -294,7 +305,7 @@ "else:\n", " print(\"Skipping S3 verification (STORE_BUCKET not set)\")\n", "\n", - "# ── Apply LIMIT ──\n", + "# -- Apply LIMIT --\n", "if LIMIT is not None:\n", " original_new = len(diff.new)\n", " original_updated = len(diff.updated)\n", diff --git a/src/cdm_data_loaders/ncbi_ftp/manifest.py b/src/cdm_data_loaders/ncbi_ftp/manifest.py index ebce8dce..ca7596c1 100644 --- a/src/cdm_data_loaders/ncbi_ftp/manifest.py +++ b/src/cdm_data_loaders/ncbi_ftp/manifest.py @@ -229,7 +229,7 @@ def compute_diff( # noqa: PLR0912 diff.new.append(acc) elif previous_assemblies is not None: prev = previous_assemblies.get(acc) - if prev and (rec.seq_rel_date != prev.seq_rel_date or rec.assembly_dir != prev.assembly_dir): + if prev and (rec.seq_rel_date > prev.seq_rel_date or rec.assembly_dir != prev.assembly_dir): diff.updated.append(acc) # Accessions in previous but entirely absent from current (withdrawn) @@ -335,12 +335,17 @@ def scan_store_to_synthetic_summary( obj_date_str = last_modified.strftime("%Y/%m/%d") if acc not in assemblies: - # First object for this accession; store it + # First object for this accession; store it. + # Construct a fake FTP path that ends with assembly_dir so + # that round-tripping through parse_assembly_summary (which + # derives assembly_dir via ftp_path.rstrip("/").split("/")[-1]) + # yields the correct assembly_dir and therefore correct diffs. + fake_ftp_path = f"https://ftp.ncbi.nlm.nih.gov/synthetic/{assembly_dir}" assemblies[acc] = AssemblyRecord( accession=acc, status="latest", seq_rel_date=obj_date_str, - ftp_path="", # synthesized; empty as it's not from FTP + ftp_path=fake_ftp_path, assembly_dir=assembly_dir, ) processed_count += 1 diff --git a/tests/ncbi_ftp/test_manifest.py b/tests/ncbi_ftp/test_manifest.py index 82ed14c0..af7f5eb1 100644 --- a/tests/ncbi_ftp/test_manifest.py +++ b/tests/ncbi_ftp/test_manifest.py @@ -154,14 +154,22 @@ def test_nothing_new_when_all_known(self) -> None: diff = compute_diff(current, previous_accessions=known) assert len(diff.new) == 0 - def test_detects_updated_seq_rel_date(self) -> None: - """Verify assemblies with changed seq_rel_date are marked updated.""" + def test_detects_updated_seq_rel_date_newer(self) -> None: + """Assemblies whose seq_rel_date moved forward are marked updated.""" current = parse_assembly_summary(SAMPLE_SUMMARY) previous = parse_assembly_summary(SAMPLE_SUMMARY) previous["GCF_000001215.4"].seq_rel_date = "2010/01/01" diff = compute_diff(current, previous_assemblies=previous) assert "GCF_000001215.4" in diff.updated + def test_does_not_flag_updated_when_seq_rel_date_older(self) -> None: + """Assemblies whose seq_rel_date in current is older (e.g. synthetic baseline) are not flagged.""" + current = parse_assembly_summary(SAMPLE_SUMMARY) + previous = parse_assembly_summary(SAMPLE_SUMMARY) + previous["GCF_000001215.4"].seq_rel_date = "2099/12/31" + diff = compute_diff(current, previous_assemblies=previous) + assert "GCF_000001215.4" not in diff.updated + def test_detects_replaced(self) -> None: """Verify assemblies with status 'replaced' are detected.""" current = parse_assembly_summary(SAMPLE_SUMMARY) @@ -739,3 +747,48 @@ def test_skips_objects_without_accession(self, mock_get_s3: MagicMock) -> None: # Only one valid assembly should be found assert len(result) == 1 assert "GCF_000001215.4" in result + + @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") + def test_assembly_dir_survives_file_round_trip(self, mock_get_s3: MagicMock, tmp_path: Path) -> None: + """Verify assembly_dir is preserved when saving to file and parsing back. + + Regression test: previously ftp_path was written as "" which caused + parse_assembly_summary to recover assembly_dir="" for all records, + making compute_diff flag every assembly as updated. + """ + from datetime import datetime, timezone + + mock = MagicMock() + mock_paginator = MagicMock() + page_contents = [ + { + "Key": "prefix/GCF_000001215.4_Release_6_plus_ISO1_MT/file.gz", + "LastModified": datetime(2024, 3, 10, tzinfo=timezone.utc), + }, + ] + mock_paginator.paginate.return_value = [{"Contents": page_contents}] + mock.get_paginator.return_value = mock_paginator + mock_get_s3.return_value = mock + + synthetic = scan_store_to_synthetic_summary("test-bucket", "prefix/") + + # Simulate the notebook's save logic + out_file = tmp_path / "synthetic_summary.txt" + with out_file.open("w") as f: + for acc in sorted(synthetic.keys()): + rec = synthetic[acc] + f.write( + f"{rec.accession}\t.\t.\t.\t.\t.\t.\t.\t.\t.\t{rec.status}\t.\t.\t.\t{rec.seq_rel_date}\t.\t.\t.\t.\t{rec.ftp_path}\t.\n" + ) + + # Parse the file back + reparsed = parse_assembly_summary(out_file) + + assert "GCF_000001215.4" in reparsed + reparsed_rec = reparsed["GCF_000001215.4"] + original_rec = synthetic["GCF_000001215.4"] + + # assembly_dir must survive the round-trip so diffs are accurate + assert reparsed_rec.assembly_dir == original_rec.assembly_dir + assert reparsed_rec.seq_rel_date == original_rec.seq_rel_date + assert reparsed_rec.status == original_rec.status From 9a1d1e55fcc61995f42e0661304ba98c4bde7077 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Tue, 21 Apr 2026 08:36:30 -0700 Subject: [PATCH 025/128] add custom release date to synthetic manifest generation --- notebooks/ncbi_ftp_manifest.ipynb | 10 ++++-- src/cdm_data_loaders/ncbi_ftp/manifest.py | 32 ++++++++------------ tests/integration/test_manifest_e2e.py | 4 +-- tests/ncbi_ftp/test_manifest.py | 37 ++++++++++++++--------- 4 files changed, 46 insertions(+), 37 deletions(-) diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb index 1d822445..38b2c982 100644 --- a/notebooks/ncbi_ftp_manifest.ipynb +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -144,7 +144,7 @@ "\n", "Set SCAN_STORE=True below to enable. The scan will:\n", " 1. List all objects under STORE_BUCKET/STORE_KEY_PREFIX\n", - " 2. Extract accessions and use earliest LastModified as seq_rel_date (conservative)\n", + " 2. Extract accessions and apply user-provided SYNTHETIC_RELEASE_DATE to all records\n", " 3. Build AssemblyRecord for each assembly found\n", " 4. Save to LOCAL_SYNTHETIC_SUMMARY for re-use in future runs\n", "\n", @@ -153,6 +153,7 @@ "\"\"\"\n", "\n", "SCAN_STORE = True # Set to True to scan your store\n", + "SYNTHETIC_RELEASE_DATE = \"2025/10/31\" # YYYY/MM/DD applied to all synthetic records\n", "LOCAL_SYNTHETIC_SUMMARY = Path(\"output/synthetic_summary_from_store.txt\")\n", "\n", "if SCAN_STORE and STORE_BUCKET:\n", @@ -169,7 +170,12 @@ " progress.set_postfix(acc=acc, refresh=False)\n", " progress.refresh()\n", "\n", - " synthetic = scan_store_to_synthetic_summary(STORE_BUCKET, STORE_KEY_PREFIX, progress_callback=_track_scan)\n", + " synthetic = scan_store_to_synthetic_summary(\n", + " STORE_BUCKET,\n", + " STORE_KEY_PREFIX,\n", + " SYNTHETIC_RELEASE_DATE,\n", + " progress_callback=_track_scan,\n", + " )\n", " progress.refresh()\n", "\n", " print(f\"Found {len(synthetic)} assemblies in store\")\n", diff --git a/src/cdm_data_loaders/ncbi_ftp/manifest.py b/src/cdm_data_loaders/ncbi_ftp/manifest.py index ca7596c1..42fd99d2 100644 --- a/src/cdm_data_loaders/ncbi_ftp/manifest.py +++ b/src/cdm_data_loaders/ncbi_ftp/manifest.py @@ -289,6 +289,7 @@ def _extract_assembly_dir_from_s3_key(key: str) -> str | None: def scan_store_to_synthetic_summary( bucket: str, key_prefix: str, + release_date: str, progress_callback: Callable[[int, str], None] | None = None, ) -> dict[str, AssemblyRecord]: """Scan S3 store and build a synthetic assembly summary from existing objects. @@ -296,21 +297,29 @@ def scan_store_to_synthetic_summary( This function is useful when bootstrapping a diffs against an existing, pre-populated S3 store that lacks a baseline assembly summary. - For each assembly found in the store: + For each assembly found in the store: - Extracts the accession and assembly directory name from S3 paths - - Uses the earliest ``LastModified`` timestamp across all files in that - assembly as the synthetic ``seq_rel_date`` (conservative estimate) + - Applies the provided ``release_date`` as synthetic ``seq_rel_date`` for + all assemblies - Creates an ``AssemblyRecord`` with ``status="latest"`` The function paginates through S3 to handle large stores efficiently. :param bucket: S3 bucket name :param key_prefix: S3 key prefix (all objects under this prefix are scanned) + :param release_date: release date string in ``YYYY/MM/DD`` format used for + all synthetic records :param progress_callback: optional callable invoked after each accession is processed with ``(count, accession)`` where count is the running total of unique accessions found :return: dict mapping accession to ``AssemblyRecord`` """ + try: + datetime.strptime(release_date, "%Y/%m/%d") + except ValueError as exc: + msg = f"Invalid release_date '{release_date}'. Expected format YYYY/MM/DD." + raise ValueError(msg) from exc + s3 = get_s3_client() assemblies: dict[str, AssemblyRecord] = {} processed_count = 0 @@ -327,13 +336,6 @@ def scan_store_to_synthetic_summary( if not acc or not assembly_dir: continue - # Convert LastModified to NCBI date format (YYYY/MM/DD) - last_modified = obj["LastModified"] - # Handle both aware and naive datetimes - if last_modified.tzinfo is None: - last_modified = last_modified.replace(tzinfo=UTC) - obj_date_str = last_modified.strftime("%Y/%m/%d") - if acc not in assemblies: # First object for this accession; store it. # Construct a fake FTP path that ends with assembly_dir so @@ -344,21 +346,13 @@ def scan_store_to_synthetic_summary( assemblies[acc] = AssemblyRecord( accession=acc, status="latest", - seq_rel_date=obj_date_str, + seq_rel_date=release_date, ftp_path=fake_ftp_path, assembly_dir=assembly_dir, ) processed_count += 1 if progress_callback is not None: progress_callback(processed_count, acc) - else: - # Update to earliest timestamp (conservative) - existing_record = assemblies[acc] - existing_date = datetime.strptime(existing_record.seq_rel_date, "%Y/%m/%d").replace( - tzinfo=UTC - ) - if last_modified < existing_date: - existing_record.seq_rel_date = obj_date_str except Exception as e: # noqa: BLE001 logger.error("Error scanning store: %s", e) diff --git a/tests/integration/test_manifest_e2e.py b/tests/integration/test_manifest_e2e.py index 4ab208f0..a0357a51 100644 --- a/tests/integration/test_manifest_e2e.py +++ b/tests/integration/test_manifest_e2e.py @@ -242,7 +242,7 @@ def test_builds_summary_from_minio_store( ) # Scan the store - result = scan_store_to_synthetic_summary(test_bucket, path_prefix) + result = scan_store_to_synthetic_summary(test_bucket, path_prefix, "2024/04/01") # Should have found both assemblies assert "GCF_000001215.4" in result @@ -268,7 +268,7 @@ def test_synthetic_summary_diff_against_current( s3.put_object(Bucket=test_bucket, Key=key1, Body=b"data") # Build synthetic summary from store - synthetic = scan_store_to_synthetic_summary(test_bucket, path_prefix) + synthetic = scan_store_to_synthetic_summary(test_bucket, path_prefix, "2024/04/20") assert "GCF_000001215.4" in synthetic # Simulate current NCBI summary with one new and one existing diff --git a/tests/ncbi_ftp/test_manifest.py b/tests/ncbi_ftp/test_manifest.py index af7f5eb1..e3ade2e3 100644 --- a/tests/ncbi_ftp/test_manifest.py +++ b/tests/ncbi_ftp/test_manifest.py @@ -4,6 +4,8 @@ from pathlib import Path from unittest.mock import MagicMock, patch +import pytest + from cdm_data_loaders.ncbi_ftp.manifest import ( DiffResult, _extract_accession_from_s3_key, @@ -646,26 +648,26 @@ def test_builds_summary_from_store(self, mock_get_s3: MagicMock) -> None: """Verify synthetic summary is built correctly from S3 objects.""" mock_get_s3.return_value = self._mock_s3_with_objects() - result = scan_store_to_synthetic_summary("test-bucket", "prefix/") + result = scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/01/31") assert len(result) == 2 assert "GCF_000001215.4" in result assert "GCF_000005845.2" in result - # Should use earliest date (2024-01-15) + # Should use provided release date for all records rec1 = result["GCF_000001215.4"] assert rec1.accession == "GCF_000001215.4" assert rec1.status == "latest" - assert rec1.seq_rel_date == "2024/01/15" + assert rec1.seq_rel_date == "2024/01/31" assert rec1.assembly_dir == "GCF_000001215.4_Release_6" - # Other assembly uses its single date + # Other assembly uses the same provided date rec2 = result["GCF_000005845.2"] - assert rec2.seq_rel_date == "2024/02/20" + assert rec2.seq_rel_date == "2024/01/31" @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") - def test_uses_earliest_date_per_assembly(self, mock_get_s3: MagicMock) -> None: - """Verify earliest LastModified is used when assembly has multiple files.""" + def test_applies_release_date_to_all_assemblies(self, mock_get_s3: MagicMock) -> None: + """Verify provided release_date is used for all assemblies.""" mock = MagicMock() mock_paginator = MagicMock() mock.get_paginator.return_value = mock_paginator @@ -686,10 +688,17 @@ def test_uses_earliest_date_per_assembly(self, mock_get_s3: MagicMock) -> None: mock_paginator.paginate.return_value = [{"Contents": page_contents}] mock_get_s3.return_value = mock - result = scan_store_to_synthetic_summary("test-bucket", "prefix/") + result = scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/03/31") + + assert result["GCF_000001215.4"].seq_rel_date == "2024/03/31" + + @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") + def test_raises_for_invalid_release_date(self, mock_get_s3: MagicMock) -> None: + """Verify invalid release_date format is rejected.""" + mock_get_s3.return_value = self._mock_s3_with_objects() - # Should use the earliest date - assert result["GCF_000001215.4"].seq_rel_date == "2024/01/10" + with pytest.raises(ValueError, match="Invalid release_date"): + scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024-03-31") @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") def test_invokes_progress_callback(self, mock_get_s3: MagicMock) -> None: @@ -700,7 +709,7 @@ def test_invokes_progress_callback(self, mock_get_s3: MagicMock) -> None: def track_progress(count: int, acc: str) -> None: callback_calls.append((count, acc)) - scan_store_to_synthetic_summary("test-bucket", "prefix/", progress_callback=track_progress) + scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/01/31", progress_callback=track_progress) # Should have 2 calls (one per assembly discovered) assert len(callback_calls) == 2 @@ -716,7 +725,7 @@ def test_handles_empty_store(self, mock_get_s3: MagicMock) -> None: mock_paginator.paginate.return_value = [{"Contents": []}] mock_get_s3.return_value = mock - result = scan_store_to_synthetic_summary("test-bucket", "prefix/") + result = scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/01/31") assert result == {} @@ -742,7 +751,7 @@ def test_skips_objects_without_accession(self, mock_get_s3: MagicMock) -> None: mock_paginator.paginate.return_value = [{"Contents": page_contents}] mock_get_s3.return_value = mock - result = scan_store_to_synthetic_summary("test-bucket", "prefix/") + result = scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/01/31") # Only one valid assembly should be found assert len(result) == 1 @@ -770,7 +779,7 @@ def test_assembly_dir_survives_file_round_trip(self, mock_get_s3: MagicMock, tmp mock.get_paginator.return_value = mock_paginator mock_get_s3.return_value = mock - synthetic = scan_store_to_synthetic_summary("test-bucket", "prefix/") + synthetic = scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/03/10") # Simulate the notebook's save logic out_file = tmp_path / "synthetic_summary.txt" From a4ee1866a8cbb691deceab3ca641f4753866ebb6 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Tue, 21 Apr 2026 08:39:04 -0700 Subject: [PATCH 026/128] fix style --- notebooks/ncbi_ftp_manifest.ipynb | 6 +++--- tests/ncbi_ftp/test_manifest.py | 29 +++++++++++++++++------------ 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb index 38b2c982..0c12060e 100644 --- a/notebooks/ncbi_ftp_manifest.ipynb +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -194,7 +194,7 @@ "else:\n", " if SCAN_STORE:\n", " print(\"SCAN_STORE=True but STORE_BUCKET not set. Skipping.\")\n", - " print(\"Skipping store scan (SCAN_STORE=False). Will load/use PREVIOUS_SUMMARY_URI instead.\")\n" + " print(\"Skipping store scan (SCAN_STORE=False). Will load/use PREVIOUS_SUMMARY_URI instead.\")" ] }, { @@ -221,7 +221,7 @@ " print(f\"Loaded {len(previous)} assemblies from previous snapshot\")\n", " else:\n", " print(\"No previous snapshot and SCAN_STORE=False — all current 'latest' assemblies will be marked as new\")\n", - " previous = None\n" + " previous = None" ] }, { @@ -321,7 +321,7 @@ " diff.new = [a for a in diff.new if a in limited_set]\n", " diff.updated = [a for a in diff.updated if a in limited_set]\n", " print(f\"After limit ({LIMIT}): {len(diff.new)} new, {len(diff.updated)} updated\")\n", - " print(f\" (was {original_new} new, {original_updated} updated)\")\n" + " print(f\" (was {original_new} new, {original_updated} updated)\")" ] }, { diff --git a/tests/ncbi_ftp/test_manifest.py b/tests/ncbi_ftp/test_manifest.py index e3ade2e3..9a3add97 100644 --- a/tests/ncbi_ftp/test_manifest.py +++ b/tests/ncbi_ftp/test_manifest.py @@ -583,12 +583,13 @@ class TestExtractAccessionFromS3Key: @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") def test_extracts_accession_from_path(self, _mock_s3: MagicMock) -> None: """Verify accession is extracted correctly from S3 keys.""" - assert _extract_accession_from_s3_key( - "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6_plus_ISO1_MT/file.gz" - ) == "GCF_000001215.4" - assert _extract_accession_from_s3_key( - "some/path/GCA_999999999.1_whatever/data.txt" - ) == "GCA_999999999.1" + assert ( + _extract_accession_from_s3_key( + "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6_plus_ISO1_MT/file.gz" + ) + == "GCF_000001215.4" + ) + assert _extract_accession_from_s3_key("some/path/GCA_999999999.1_whatever/data.txt") == "GCA_999999999.1" def test_returns_none_for_invalid_path(self) -> None: """Verify None is returned when no accession is found.""" @@ -601,12 +602,16 @@ class TestExtractAssemblyDirFromS3Key: def test_extracts_assembly_dir(self) -> None: """Verify assembly directory is extracted correctly from S3 keys.""" - assert _extract_assembly_dir_from_s3_key( - "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6_plus_ISO1_MT/file.gz" - ) == "GCF_000001215.4_Release_6_plus_ISO1_MT" - assert _extract_assembly_dir_from_s3_key( - "prefix/GCA_999999999.1_assembly_name/subdir/data.txt" - ) == "GCA_999999999.1_assembly_name" + assert ( + _extract_assembly_dir_from_s3_key( + "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6_plus_ISO1_MT/file.gz" + ) + == "GCF_000001215.4_Release_6_plus_ISO1_MT" + ) + assert ( + _extract_assembly_dir_from_s3_key("prefix/GCA_999999999.1_assembly_name/subdir/data.txt") + == "GCA_999999999.1_assembly_name" + ) def test_returns_none_for_invalid_path(self) -> None: """Verify None is returned when no assembly directory is found.""" From 6e458797642aac7e900442753cf3e888df7ada67 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Tue, 21 Apr 2026 12:22:32 -0700 Subject: [PATCH 027/128] update docs and progress bar --- docs/ncbi_ftp_e2e_walkthrough.md | 22 +++++++++++++++++++++- notebooks/ncbi_ftp_manifest.ipynb | 8 ++------ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/docs/ncbi_ftp_e2e_walkthrough.md b/docs/ncbi_ftp_e2e_walkthrough.md index ffce0e3e..b4420658 100644 --- a/docs/ncbi_ftp_e2e_walkthrough.md +++ b/docs/ncbi_ftp_e2e_walkthrough.md @@ -76,7 +76,11 @@ s3://{STORE_BUCKET}/{STAGING_KEY_PREFIX}raw_data/{GCF|GCA}/{nnn}/{nnn}/{nnn}/{as --- -## 1. Start MinIO +## 1. Setup + +### Local testing + +### Start MinIO ```sh docker run -d \ @@ -97,6 +101,22 @@ included `scripts/s3_local.py` helper (requires no extra installs — only uv run python scripts/s3_local.py mb s3://cdm-lake ``` +### Lakehouse + +#### Build `cdm-data-loaders` + +First, clone the `cdm-data-loaders` repo in your Lakehouse user space. Then, build the package +in a virtual environment and register it as a Jupyter kernel: +``` +cd cdm-data-loaders +uv sync +source .venv/bin/activate +uv pip install -e . +uv pip install ipykernel +uv run python -m ipykernel install --user --name cdm-data-loaders --display-name "cdm-data-loaders" +``` +Then, when you open the manifest or promote notebooks, choose the `cdm-data-loaders` kernel. + --- ## 2. Phase 1 — Generate manifests (notebook) diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb index 0c12060e..22516ae2 100644 --- a/notebooks/ncbi_ftp_manifest.ipynb +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -158,12 +158,10 @@ "\n", "if SCAN_STORE and STORE_BUCKET:\n", " from cdm_data_loaders.ncbi_ftp.manifest import scan_store_to_synthetic_summary\n", - " from IPython.display import display\n", " from tqdm.notebook import tqdm\n", "\n", " print(f\"Scanning s3://{STORE_BUCKET}/{STORE_KEY_PREFIX} for existing assemblies ...\")\n", " progress = tqdm(unit=\"assembly\", desc=\"Scanning store\", leave=True)\n", - " display(progress)\n", "\n", " def _track_scan(count: int, acc: str) -> None:\n", " progress.n = count\n", @@ -194,7 +192,7 @@ "else:\n", " if SCAN_STORE:\n", " print(\"SCAN_STORE=True but STORE_BUCKET not set. Skipping.\")\n", - " print(\"Skipping store scan (SCAN_STORE=False). Will load/use PREVIOUS_SUMMARY_URI instead.\")" + " print(\"Skipping store scan (SCAN_STORE=False). Will load/use PREVIOUS_SUMMARY_URI instead.\")\n" ] }, { @@ -271,7 +269,6 @@ "# -- Verify against Lakehouse --\n", "if STORE_BUCKET:\n", " from cdm_data_loaders.ncbi_ftp.manifest import verify_transfer_candidates\n", - " from IPython.display import display\n", " from tqdm.notebook import tqdm\n", "\n", " candidates = diff.new + diff.updated\n", @@ -279,7 +276,6 @@ " print(f\"Verifying {total} candidates against s3://{STORE_BUCKET}/{STORE_KEY_PREFIX} ...\")\n", "\n", " progress = tqdm(total=total, unit=\"assembly\", desc=\"Verifying checksums\", leave=True)\n", - " display(progress)\n", "\n", " def _update_progress(done: int, _total: int, acc: str) -> None:\n", " progress.n = done\n", @@ -321,7 +317,7 @@ " diff.new = [a for a in diff.new if a in limited_set]\n", " diff.updated = [a for a in diff.updated if a in limited_set]\n", " print(f\"After limit ({LIMIT}): {len(diff.new)} new, {len(diff.updated)} updated\")\n", - " print(f\" (was {original_new} new, {original_updated} updated)\")" + " print(f\" (was {original_new} new, {original_updated} updated)\")\n" ] }, { From adfda18ca5f5e241290195a4ce434cfdac9cd40e Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Tue, 21 Apr 2026 12:29:02 -0700 Subject: [PATCH 028/128] add checks for synthetic summary creation --- notebooks/ncbi_ftp_manifest.ipynb | 67 ++++++++++++++++++------------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb index 22516ae2..e1b915d8 100644 --- a/notebooks/ncbi_ftp_manifest.ipynb +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -149,46 +149,55 @@ " 4. Save to LOCAL_SYNTHETIC_SUMMARY for re-use in future runs\n", "\n", "Typical use case: First run against 500K+ existing assemblies. Scanning takes\n", - "~5 minutes instead of ~6 days of checksum verification.\n", + "significant time (potentially 15-30+ min for large stores with many files per assembly).\n", + "On subsequent runs the saved file is loaded directly — set FORCE_RESCAN=True to override.\n", "\"\"\"\n", "\n", "SCAN_STORE = True # Set to True to scan your store\n", + "FORCE_RESCAN = False # Set to True to ignore an existing LOCAL_SYNTHETIC_SUMMARY and rescan\n", "SYNTHETIC_RELEASE_DATE = \"2025/10/31\" # YYYY/MM/DD applied to all synthetic records\n", "LOCAL_SYNTHETIC_SUMMARY = Path(\"output/synthetic_summary_from_store.txt\")\n", "\n", "if SCAN_STORE and STORE_BUCKET:\n", - " from cdm_data_loaders.ncbi_ftp.manifest import scan_store_to_synthetic_summary\n", - " from tqdm.notebook import tqdm\n", - "\n", - " print(f\"Scanning s3://{STORE_BUCKET}/{STORE_KEY_PREFIX} for existing assemblies ...\")\n", - " progress = tqdm(unit=\"assembly\", desc=\"Scanning store\", leave=True)\n", - "\n", - " def _track_scan(count: int, acc: str) -> None:\n", - " progress.n = count\n", - " progress.set_postfix(acc=acc, refresh=False)\n", + " if LOCAL_SYNTHETIC_SUMMARY.exists() and not FORCE_RESCAN:\n", + " print(f\"Loading existing synthetic summary from {LOCAL_SYNTHETIC_SUMMARY} (set FORCE_RESCAN=True to rescan)\")\n", + " previous = parse_assembly_summary(LOCAL_SYNTHETIC_SUMMARY)\n", + " print(f\"Loaded {len(previous)} assemblies from synthetic summary\")\n", + " else:\n", + " from cdm_data_loaders.ncbi_ftp.manifest import scan_store_to_synthetic_summary\n", + " from tqdm.notebook import tqdm\n", + "\n", + " print(f\"Scanning s3://{STORE_BUCKET}/{STORE_KEY_PREFIX} for existing assemblies ...\")\n", + " print(\"Note: large stores (500K+ assemblies) may take 15-30+ minutes.\")\n", + " progress = tqdm(unit=\"assembly\", desc=\"Scanning store\", leave=True)\n", + "\n", + " def _track_scan(count: int, acc: str) -> None:\n", + " progress.n = count\n", + " progress.set_postfix(acc=acc, refresh=False)\n", + " progress.refresh()\n", + "\n", + " synthetic = scan_store_to_synthetic_summary(\n", + " STORE_BUCKET,\n", + " STORE_KEY_PREFIX,\n", + " SYNTHETIC_RELEASE_DATE,\n", + " progress_callback=_track_scan,\n", + " )\n", " progress.refresh()\n", "\n", - " synthetic = scan_store_to_synthetic_summary(\n", - " STORE_BUCKET,\n", - " STORE_KEY_PREFIX,\n", - " SYNTHETIC_RELEASE_DATE,\n", - " progress_callback=_track_scan,\n", - " )\n", - " progress.refresh()\n", - "\n", - " print(f\"Found {len(synthetic)} assemblies in store\")\n", + " print(f\"Found {len(synthetic)} assemblies in store\")\n", "\n", - " # Save synthetic summary to file for future runs\n", - " with LOCAL_SYNTHETIC_SUMMARY.open(\"w\") as f:\n", - " for acc in sorted(synthetic.keys()):\n", - " rec = synthetic[acc]\n", - " f.write(\n", - " f\"{rec.accession}\\t.\\t.\\t.\\t.\\t.\\t.\\t.\\t.\\t.\\t{rec.status}\\t.\\t.\\t.\\t{rec.seq_rel_date}\\t.\\t.\\t.\\t.\\t{rec.ftp_path}\\t.\\n\"\n", - " )\n", - " print(f\"Saved synthetic summary to {LOCAL_SYNTHETIC_SUMMARY}\")\n", + " # Save synthetic summary to file for future runs\n", + " LOCAL_SYNTHETIC_SUMMARY.parent.mkdir(parents=True, exist_ok=True)\n", + " with LOCAL_SYNTHETIC_SUMMARY.open(\"w\") as f:\n", + " for acc in sorted(synthetic.keys()):\n", + " rec = synthetic[acc]\n", + " f.write(\n", + " f\"{rec.accession}\\t.\\t.\\t.\\t.\\t.\\t.\\t.\\t.\\t.\\t{rec.status}\\t.\\t.\\t.\\t{rec.seq_rel_date}\\t.\\t.\\t.\\t.\\t{rec.ftp_path}\\t.\\n\"\n", + " )\n", + " print(f\"Saved synthetic summary to {LOCAL_SYNTHETIC_SUMMARY}\")\n", "\n", - " # Use it as the previous baseline\n", - " previous = synthetic\n", + " # Use it as the previous baseline\n", + " previous = synthetic\n", "else:\n", " if SCAN_STORE:\n", " print(\"SCAN_STORE=True but STORE_BUCKET not set. Skipping.\")\n", From fdd67c4f002ea29495ba7ce028040cf230748609 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Tue, 21 Apr 2026 12:35:05 -0700 Subject: [PATCH 029/128] add connection check to S3 store --- notebooks/ncbi_ftp_manifest.ipynb | 39 +++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb index e1b915d8..c0807086 100644 --- a/notebooks/ncbi_ftp_manifest.ipynb +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -115,6 +115,45 @@ "print(f\"Output dir: {OUTPUT_DIR}\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "be1fcf1c", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Validate S3 connectivity and bucket/prefix configuration.\"\"\"\n", + "\n", + "s3 = get_s3_client()\n", + "\n", + "# Check bucket is accessible\n", + "try:\n", + " s3.head_bucket(Bucket=\"cdm-lake\" if not STORE_BUCKET else STORE_BUCKET)\n", + " print(f\"✓ Bucket accessible: {STORE_BUCKET or 'cdm-lake'}\")\n", + "except Exception as e:\n", + " print(f\"✗ Bucket not accessible: {e}\")\n", + " raise\n", + "\n", + "# Check that objects exist under the key prefix (i.e. prefix is non-empty)\n", + "if STORE_BUCKET:\n", + " resp = s3.list_objects_v2(Bucket=STORE_BUCKET, Prefix=STORE_KEY_PREFIX, MaxKeys=1)\n", + " if resp.get(\"KeyCount\", 0) > 0:\n", + " print(f\"✓ Prefix has objects: s3://{STORE_BUCKET}/{STORE_KEY_PREFIX}\")\n", + " else:\n", + " print(f\"⚠ No objects found under s3://{STORE_BUCKET}/{STORE_KEY_PREFIX} — check STORE_KEY_PREFIX\")\n", + "else:\n", + " print(\"Skipping prefix check (STORE_BUCKET not set)\")\n", + "\n", + "# Check STAGING_URI bucket if set\n", + "if \"STAGING_URI\" in dir() and STAGING_URI:\n", + " staging_bucket, _ = split_s3_path(STAGING_URI)\n", + " try:\n", + " s3.head_bucket(Bucket=staging_bucket)\n", + " print(f\"✓ Staging bucket accessible: {staging_bucket}\")\n", + " except Exception as e:\n", + " print(f\"✗ Staging bucket not accessible: {e}\")\n" + ] + }, { "cell_type": "code", "execution_count": null, From d4f868d817d5536275b33b1e1dd260c5822a3c30 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Tue, 21 Apr 2026 13:13:22 -0700 Subject: [PATCH 030/128] add s3 connection check --- docs/ncbi_ftp_e2e_walkthrough.md | 27 ++++++++++++++++- notebooks/ncbi_ftp_manifest.ipynb | 50 +++++++++++++++++-------------- 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/docs/ncbi_ftp_e2e_walkthrough.md b/docs/ncbi_ftp_e2e_walkthrough.md index b4420658..8b2f1ab5 100644 --- a/docs/ncbi_ftp_e2e_walkthrough.md +++ b/docs/ncbi_ftp_e2e_walkthrough.md @@ -107,7 +107,7 @@ uv run python scripts/s3_local.py mb s3://cdm-lake First, clone the `cdm-data-loaders` repo in your Lakehouse user space. Then, build the package in a virtual environment and register it as a Jupyter kernel: -``` +```bash cd cdm-data-loaders uv sync source .venv/bin/activate @@ -117,6 +117,31 @@ uv run python -m ipykernel install --user --name cdm-data-loaders --display-name ``` Then, when you open the manifest or promote notebooks, choose the `cdm-data-loaders` kernel. +#### Add the S3 Credentials to the Kernel + +Open a new Jupyter Notebook with the default kernel and run this in a new cell: +```python +import os +for k, v in sorted(os.environ.items()): + if "AWS" in k or "S3" in k or "MINIO" in k: + print(f"{k}={v}") +``` +Take the output and add the environment vars to the `kernel.json` for your new kernel (e.g., in `cdm-data-loaders/.venv/share/jupyter/kernels/python3/kernel.json`): +```json +{ + "argv": ["..."], + "display_name": "cdm-data-loaders", + "language": "python", + "env": { + "AWS_ACCESS_KEY_ID": "...", + "AWS_SECRET_ACCESS_KEY": "...", + "AWS_DEFAULT_REGION": "...", + ... + } +} +``` + + --- ## 2. Phase 1 — Generate manifests (notebook) diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb index c0807086..85f73b13 100644 --- a/notebooks/ncbi_ftp_manifest.ipynb +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -124,34 +124,40 @@ "source": [ "\"\"\"Validate S3 connectivity and bucket/prefix configuration.\"\"\"\n", "\n", - "s3 = get_s3_client()\n", + "import boto3\n", + "from botocore.exceptions import ClientError, NoCredentialsError\n", "\n", - "# Check bucket is accessible\n", + "s3 = boto3.client(\"s3\")\n", + "\n", + "# Check credentials are present\n", "try:\n", - " s3.head_bucket(Bucket=\"cdm-lake\" if not STORE_BUCKET else STORE_BUCKET)\n", - " print(f\"✓ Bucket accessible: {STORE_BUCKET or 'cdm-lake'}\")\n", - "except Exception as e:\n", - " print(f\"✗ Bucket not accessible: {e}\")\n", + " sts = boto3.client(\"sts\")\n", + " identity = sts.get_caller_identity()\n", + " print(f\"✓ Credentials valid — account: {identity['Account']}, arn: {identity['Arn']}\")\n", + "except NoCredentialsError:\n", + " print(\"✗ No AWS credentials found\")\n", " raise\n", + "except ClientError as e:\n", + " if e.response[\"Error\"][\"Code\"] == \"InvalidParameterValue\":\n", + " print(\"✓ Credentials present (STS GetCallerIdentity not supported on this endpoint — skipping identity check)\")\n", + " else:\n", + " print(f\"✗ Credential check failed: {e}\")\n", + " raise\n", "\n", - "# Check that objects exist under the key prefix (i.e. prefix is non-empty)\n", - "if STORE_BUCKET:\n", - " resp = s3.list_objects_v2(Bucket=STORE_BUCKET, Prefix=STORE_KEY_PREFIX, MaxKeys=1)\n", + "# Check bucket access and prefix in one step — list_objects_v2 requires only\n", + "# s3:ListBucket on the prefix, which is less restrictive than HeadBucket.\n", + "_check_bucket = STORE_BUCKET if \"STORE_BUCKET\" in dir() and STORE_BUCKET else \"cdm-lake\"\n", + "_check_prefix = STORE_KEY_PREFIX if \"STORE_KEY_PREFIX\" in dir() else \"tenant-general-warehouse/kbase/datasets/ncbi/\"\n", + "try:\n", + " resp = s3.list_objects_v2(Bucket=_check_bucket, Prefix=_check_prefix, MaxKeys=1)\n", " if resp.get(\"KeyCount\", 0) > 0:\n", - " print(f\"✓ Prefix has objects: s3://{STORE_BUCKET}/{STORE_KEY_PREFIX}\")\n", + " print(f\"✓ Bucket accessible and prefix has objects: s3://{_check_bucket}/{_check_prefix}\")\n", " else:\n", - " print(f\"⚠ No objects found under s3://{STORE_BUCKET}/{STORE_KEY_PREFIX} — check STORE_KEY_PREFIX\")\n", - "else:\n", - " print(\"Skipping prefix check (STORE_BUCKET not set)\")\n", - "\n", - "# Check STAGING_URI bucket if set\n", - "if \"STAGING_URI\" in dir() and STAGING_URI:\n", - " staging_bucket, _ = split_s3_path(STAGING_URI)\n", - " try:\n", - " s3.head_bucket(Bucket=staging_bucket)\n", - " print(f\"✓ Staging bucket accessible: {staging_bucket}\")\n", - " except Exception as e:\n", - " print(f\"✗ Staging bucket not accessible: {e}\")\n" + " print(f\"✓ Bucket accessible but no objects found under s3://{_check_bucket}/{_check_prefix} — check STORE_KEY_PREFIX\")\n", + "except ClientError as e:\n", + " code = e.response[\"Error\"][\"Code\"]\n", + " print(f\"✗ S3 access check failed (HTTP {code}): {e}\")\n", + " raise\n" ] }, { From 55cbf9e296df9dadb33bba52692f97fa20dc8c37 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Tue, 21 Apr 2026 13:21:54 -0700 Subject: [PATCH 031/128] reduce progress bar update rate --- notebooks/ncbi_ftp_manifest.ipynb | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb index 85f73b13..a25031d5 100644 --- a/notebooks/ncbi_ftp_manifest.ipynb +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -209,17 +209,24 @@ " previous = parse_assembly_summary(LOCAL_SYNTHETIC_SUMMARY)\n", " print(f\"Loaded {len(previous)} assemblies from synthetic summary\")\n", " else:\n", + " import time as _time\n", " from cdm_data_loaders.ncbi_ftp.manifest import scan_store_to_synthetic_summary\n", " from tqdm.notebook import tqdm\n", "\n", " print(f\"Scanning s3://{STORE_BUCKET}/{STORE_KEY_PREFIX} for existing assemblies ...\")\n", " print(\"Note: large stores (500K+ assemblies) may take 15-30+ minutes.\")\n", - " progress = tqdm(unit=\"assembly\", desc=\"Scanning store\", leave=True)\n", + " progress = tqdm(unit=\"assembly\", desc=\"Scanning store\", leave=True, mininterval=2.0)\n", + "\n", + " _last_refresh = _time.monotonic()\n", + " _REFRESH_INTERVAL = 2.0 # seconds between display updates\n", "\n", " def _track_scan(count: int, acc: str) -> None:\n", + " global _last_refresh\n", " progress.n = count\n", - " progress.set_postfix(acc=acc, refresh=False)\n", - " progress.refresh()\n", + " now = _time.monotonic()\n", + " if now - _last_refresh >= _REFRESH_INTERVAL:\n", + " progress.set_postfix(acc=acc, refresh=True)\n", + " _last_refresh = now\n", "\n", " synthetic = scan_store_to_synthetic_summary(\n", " STORE_BUCKET,\n", @@ -227,6 +234,7 @@ " SYNTHETIC_RELEASE_DATE,\n", " progress_callback=_track_scan,\n", " )\n", + " progress.n = len(synthetic)\n", " progress.refresh()\n", "\n", " print(f\"Found {len(synthetic)} assemblies in store\")\n", From c1ee7e3e1afae2e8a2fbc9301c96555fc5dbe3e9 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Tue, 21 Apr 2026 13:42:06 -0700 Subject: [PATCH 032/128] select database for synthetic summary --- notebooks/ncbi_ftp_manifest.ipynb | 12 +++++++----- src/cdm_data_loaders/ncbi_ftp/manifest.py | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb index a25031d5..4762f768 100644 --- a/notebooks/ncbi_ftp_manifest.ipynb +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -189,9 +189,10 @@ "\n", "Set SCAN_STORE=True below to enable. The scan will:\n", " 1. List all objects under STORE_BUCKET/STORE_KEY_PREFIX\n", - " 2. Extract accessions and apply user-provided SYNTHETIC_RELEASE_DATE to all records\n", - " 3. Build AssemblyRecord for each assembly found\n", - " 4. Save to LOCAL_SYNTHETIC_SUMMARY for re-use in future runs\n", + " 2. Extract accessions matching the DATABASE type (GCF_ for refseq, GCA_ for genbank)\n", + " 3. Apply user-provided SYNTHETIC_RELEASE_DATE to all records\n", + " 4. Build AssemblyRecord for each assembly found\n", + " 5. Save to LOCAL_SYNTHETIC_SUMMARY for re-use in future runs\n", "\n", "Typical use case: First run against 500K+ existing assemblies. Scanning takes\n", "significant time (potentially 15-30+ min for large stores with many files per assembly).\n", @@ -213,7 +214,7 @@ " from cdm_data_loaders.ncbi_ftp.manifest import scan_store_to_synthetic_summary\n", " from tqdm.notebook import tqdm\n", "\n", - " print(f\"Scanning s3://{STORE_BUCKET}/{STORE_KEY_PREFIX} for existing assemblies ...\")\n", + " print(f\"Scanning s3://{STORE_BUCKET}/{STORE_KEY_PREFIX} for existing {DATABASE} assemblies ...\")\n", " print(\"Note: large stores (500K+ assemblies) may take 15-30+ minutes.\")\n", " progress = tqdm(unit=\"assembly\", desc=\"Scanning store\", leave=True, mininterval=2.0)\n", "\n", @@ -232,12 +233,13 @@ " STORE_BUCKET,\n", " STORE_KEY_PREFIX,\n", " SYNTHETIC_RELEASE_DATE,\n", + " database=DATABASE,\n", " progress_callback=_track_scan,\n", " )\n", " progress.n = len(synthetic)\n", " progress.refresh()\n", "\n", - " print(f\"Found {len(synthetic)} assemblies in store\")\n", + " print(f\"Found {len(synthetic)} {DATABASE} assemblies in store\")\n", "\n", " # Save synthetic summary to file for future runs\n", " LOCAL_SYNTHETIC_SUMMARY.parent.mkdir(parents=True, exist_ok=True)\n", diff --git a/src/cdm_data_loaders/ncbi_ftp/manifest.py b/src/cdm_data_loaders/ncbi_ftp/manifest.py index 42fd99d2..b14d2923 100644 --- a/src/cdm_data_loaders/ncbi_ftp/manifest.py +++ b/src/cdm_data_loaders/ncbi_ftp/manifest.py @@ -286,10 +286,17 @@ def _extract_assembly_dir_from_s3_key(key: str) -> str | None: return m.group(1) if m else None +_DATABASE_ACC_PREFIX: dict[str, str] = { + "refseq": "GCF_", + "genbank": "GCA_", +} + + def scan_store_to_synthetic_summary( bucket: str, key_prefix: str, release_date: str, + database: str = "refseq", progress_callback: Callable[[int, str], None] | None = None, ) -> dict[str, AssemblyRecord]: """Scan S3 store and build a synthetic assembly summary from existing objects. @@ -302,6 +309,8 @@ def scan_store_to_synthetic_summary( - Applies the provided ``release_date`` as synthetic ``seq_rel_date`` for all assemblies - Creates an ``AssemblyRecord`` with ``status="latest"`` + - Filters to accessions matching the expected prefix for ``database`` + (``GCF_`` for ``"refseq"``, ``GCA_`` for ``"genbank"``) The function paginates through S3 to handle large stores efficiently. @@ -309,6 +318,8 @@ def scan_store_to_synthetic_summary( :param key_prefix: S3 key prefix (all objects under this prefix are scanned) :param release_date: release date string in ``YYYY/MM/DD`` format used for all synthetic records + :param database: ``"refseq"`` or ``"genbank"`` — controls which accession + prefix is included (``GCF_`` or ``GCA_`` respectively) :param progress_callback: optional callable invoked after each accession is processed with ``(count, accession)`` where count is the running total of unique accessions found @@ -320,6 +331,11 @@ def scan_store_to_synthetic_summary( msg = f"Invalid release_date '{release_date}'. Expected format YYYY/MM/DD." raise ValueError(msg) from exc + acc_prefix = _DATABASE_ACC_PREFIX.get(database) + if acc_prefix is None: + msg = f"Unknown database: {database!r}. Expected 'refseq' or 'genbank'." + raise ValueError(msg) + s3 = get_s3_client() assemblies: dict[str, AssemblyRecord] = {} processed_count = 0 @@ -331,6 +347,8 @@ def scan_store_to_synthetic_summary( for page in pages: for obj in page.get("Contents", []): acc = _extract_accession_from_s3_key(obj["Key"]) + if not acc or not acc.startswith(acc_prefix): + continue assembly_dir = _extract_assembly_dir_from_s3_key(obj["Key"]) if not acc or not assembly_dir: From 76349c1f48fc9ab45dea20c1dc3f627fbf182fb2 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 22 Apr 2026 07:59:12 -0700 Subject: [PATCH 033/128] formatting --- notebooks/ncbi_ftp_manifest.ipynb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb index 4762f768..4d51c5b3 100644 --- a/notebooks/ncbi_ftp_manifest.ipynb +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -153,11 +153,13 @@ " if resp.get(\"KeyCount\", 0) > 0:\n", " print(f\"✓ Bucket accessible and prefix has objects: s3://{_check_bucket}/{_check_prefix}\")\n", " else:\n", - " print(f\"✓ Bucket accessible but no objects found under s3://{_check_bucket}/{_check_prefix} — check STORE_KEY_PREFIX\")\n", + " print(\n", + " f\"✓ Bucket accessible but no objects found under s3://{_check_bucket}/{_check_prefix} — check STORE_KEY_PREFIX\"\n", + " )\n", "except ClientError as e:\n", " code = e.response[\"Error\"][\"Code\"]\n", " print(f\"✗ S3 access check failed (HTTP {code}): {e}\")\n", - " raise\n" + " raise" ] }, { @@ -256,7 +258,7 @@ "else:\n", " if SCAN_STORE:\n", " print(\"SCAN_STORE=True but STORE_BUCKET not set. Skipping.\")\n", - " print(\"Skipping store scan (SCAN_STORE=False). Will load/use PREVIOUS_SUMMARY_URI instead.\")\n" + " print(\"Skipping store scan (SCAN_STORE=False). Will load/use PREVIOUS_SUMMARY_URI instead.\")" ] }, { @@ -381,7 +383,7 @@ " diff.new = [a for a in diff.new if a in limited_set]\n", " diff.updated = [a for a in diff.updated if a in limited_set]\n", " print(f\"After limit ({LIMIT}): {len(diff.new)} new, {len(diff.updated)} updated\")\n", - " print(f\" (was {original_new} new, {original_updated} updated)\")\n" + " print(f\" (was {original_new} new, {original_updated} updated)\")" ] }, { From 3369e11299cc9ebf19db604c648cb249e280aeda Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 22 Apr 2026 11:56:22 -0700 Subject: [PATCH 034/128] add frictionless descriptors --- docs/ncbi_ftp_e2e_walkthrough.md | 40 +++ notebooks/ncbi_ftp_promote.ipynb | 44 ++- src/cdm_data_loaders/ncbi_ftp/metadata.py | 259 +++++++++++++++ src/cdm_data_loaders/ncbi_ftp/promote.py | 138 ++++++-- tests/integration/test_promote_e2e.py | 173 ++++++++++ tests/ncbi_ftp/test_metadata.py | 369 ++++++++++++++++++++++ 6 files changed, 993 insertions(+), 30 deletions(-) create mode 100644 src/cdm_data_loaders/ncbi_ftp/metadata.py create mode 100644 tests/ncbi_ftp/test_metadata.py diff --git a/docs/ncbi_ftp_e2e_walkthrough.md b/docs/ncbi_ftp_e2e_walkthrough.md index 8b2f1ab5..d814b73b 100644 --- a/docs/ncbi_ftp_e2e_walkthrough.md +++ b/docs/ncbi_ftp_e2e_walkthrough.md @@ -428,6 +428,46 @@ uv run python scripts/s3_local.py head \ s3://cdm-lake/tenant-general-warehouse/kbase/datasets/ncbi/raw_data/GCF/900/000/615/GCF_900000615.1_PRJEB7657_assembly/GCF_900000615.1_PRJEB7657_assembly_genomic.fna.gz ``` +### Frictionless metadata descriptors + +Each promoted assembly gets a [frictionless](https://framework.frictionlessdata.io/) data package descriptor stored at: + +``` +s3://{STORE_BUCKET}/{LAKEHOUSE_KEY_PREFIX}metadata/{assembly_dir}_datapackage.json +``` + +For example: + +``` +s3://cdm-lake/tenant-general-warehouse/kbase/datasets/ncbi/metadata/GCF_900000615.1_PRJEB7657_assembly_datapackage.json +``` + +The descriptor follows the KBase credit metadata schema (v1.0) and records: + +- **identifier** — `NCBI:{accession}`, e.g. `NCBI:GCF_900000615.1` +- **resource_type** — always `"dataset"` +- **resources** — list of promoted files with their final S3 key, byte size, + file format, and MD5 hash (when available) +- **contributors / publisher** — NCBI organizational metadata +- **meta.saved_by** — `"cdm-data-loaders-ncbi-ftp"` + +When an assembly is archived (updated or removed), its live descriptor is +copied to: + +``` +s3://{STORE_BUCKET}/{LAKEHOUSE_KEY_PREFIX}archive/{release_tag}/metadata/{assembly_dir}_datapackage.json +``` + +Use the last cell of `notebooks/ncbi_ftp_promote.ipynb` to list and preview +all descriptors written in a promote run. + +To inspect a descriptor directly: + +```sh +uv run python scripts/s3_local.py cat \ + s3://cdm-lake/tenant-general-warehouse/kbase/datasets/ncbi/metadata/GCF_900000615.1_PRJEB7657_assembly_datapackage.json +``` + --- ## 6. Incremental run (second sync) diff --git a/notebooks/ncbi_ftp_promote.ipynb b/notebooks/ncbi_ftp_promote.ipynb index 1da31ab2..81239f39 100644 --- a/notebooks/ncbi_ftp_promote.ipynb +++ b/notebooks/ncbi_ftp_promote.ipynb @@ -34,8 +34,8 @@ "| `_S3_KEY` | S3 object key (no scheme/bucket) | `staging/transfer_manifest.txt` |\n", "| `_PATH` | local filesystem path | `output/removed_manifest.txt` |\n", "\n", - "Lakehouse object: `s3://{STORE_BUCKET}/{LAKEHOUSE_KEY_PREFIX}raw_data/…/{filename}`\n", - "Staging object: `s3://{STORE_BUCKET}/{STAGING_KEY_PREFIX}raw_data/…/{filename}`" + "Lakehouse object: `s3://{STORE_BUCKET}/{LAKEHOUSE_KEY_PREFIX}raw_data/\u2026/{filename}`\n", + "Staging object: `s3://{STORE_BUCKET}/{STAGING_KEY_PREFIX}raw_data/\u2026/{filename}`" ] }, { @@ -100,7 +100,7 @@ "# format: S3 key prefix within STORE_BUCKET (no scheme, no bucket)\n", "LAKEHOUSE_KEY_PREFIX = DEFAULT_LAKEHOUSE_KEY_PREFIX\n", "\n", - "# Dry-run mode — log actions without making changes\n", + "# Dry-run mode \u2014 log actions without making changes\n", "DRY_RUN = False\n", "\n", "print(f\"Bucket: {STORE_BUCKET}\")\n", @@ -185,10 +185,42 @@ "print(f\"Timestamp: {report['timestamp']}\")\n", "\n", "if report[\"failed\"] > 0:\n", - " print(\"\\n⚠️ Some operations failed — check logs above for details.\")\n", + " print(\"\\n\u26a0\ufe0f Some operations failed \u2014 check logs above for details.\")\n", "\n", "if report[\"dry_run\"]:\n", - " print(\"\\n📋 This was a dry-run. Set DRY_RUN = False and re-run to apply changes.\")" + " print(\"\\n\ud83d\udccb This was a dry-run. Set DRY_RUN = False and re-run to apply changes.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Inspect frictionless descriptors written to metadata/.\"\"\"\n", + "\n", + "from cdm_data_loaders.ncbi_ftp.metadata import build_descriptor_key\n", + "\n", + "s3 = get_s3_client()\n", + "paginator = s3.get_paginator(\"list_objects_v2\")\n", + "\n", + "descriptor_keys: list[str] = []\n", + "for page in paginator.paginate(Bucket=STORE_BUCKET, Prefix=LAKEHOUSE_KEY_PREFIX + \"metadata/\"):\n", + " descriptor_keys.extend(obj[\"Key\"] for obj in page.get(\"Contents\", []))\n", + "\n", + "print(f\"Found {len(descriptor_keys)} descriptor(s) in metadata/\")\n", + "\n", + "for key in descriptor_keys[:5]: # preview first 5\n", + " obj = s3.get_object(Bucket=STORE_BUCKET, Key=key)\n", + " descriptor = json.loads(obj[\"Body\"].read())\n", + " print()\n", + " print(f\" Key: {key}\")\n", + " print(f\" Identifier: {descriptor.get('identifier')}\")\n", + " print(f\" Version: {descriptor.get('version')}\")\n", + " print(f\" Resources: {len(descriptor.get('resources', []))} file(s)\")\n", + "\n", + "if len(descriptor_keys) > 5:\n", + " print(f\" ... and {len(descriptor_keys) - 5} more\")" ] } ], @@ -213,4 +245,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file diff --git a/src/cdm_data_loaders/ncbi_ftp/metadata.py b/src/cdm_data_loaders/ncbi_ftp/metadata.py new file mode 100644 index 00000000..356ee0d6 --- /dev/null +++ b/src/cdm_data_loaders/ncbi_ftp/metadata.py @@ -0,0 +1,259 @@ +"""Frictionless data package descriptor creation for NCBI FTP assemblies. + +Creates KBase credit metadata descriptors for each promoted assembly, +matching the schema produced by ``kbase-transfers/scripts/ncbi/download_genomes.py``. + +Each descriptor is a frictionless ``Package``-compatible JSON document +describing the assembly's data files, stored at:: + + {key_prefix}metadata/{assembly_dir}_datapackage.json + +and archived alongside raw data at:: + + {key_prefix}archive/{release_tag}/metadata/{assembly_dir}_datapackage.json + +The descriptor ``resources`` list records the final Lakehouse S3 key, byte +size, file format, and MD5 hash of each promoted data file. +""" + +from __future__ import annotations + +import json +import tempfile +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, TypedDict + +from frictionless import Package + +from cdm_data_loaders.utils.cdm_logger import get_cdm_logger +from cdm_data_loaders.utils.s3 import copy_object_with_metadata, get_s3_client + +logger = get_cdm_logger() + +_NCBI_CONTRIBUTOR = { + "contributor_type": "Organization", + "name": "National Center for Biotechnology Information", + "contributor_id": "ROR:02meqm098", + "contributor_roles": "DataCurator", +} +_NCBI_PUBLISHER = { + "organization_name": "National Center for Biotechnology Information", + "organization_id": "ROR:02meqm098", +} +_SAVED_BY = "cdm-data-loaders-ncbi-ftp" +_SCHEMA_VERSION = "1.0" + + +class DescriptorResource(TypedDict, total=False): + """A single resource entry in the frictionless descriptor ``resources`` list.""" + + name: str + path: str + format: str + bytes: int | None + hash: str | None + + +# ── Public helpers ──────────────────────────────────────────────────────── + + +def build_descriptor_key(assembly_dir: str, key_prefix: str) -> str: + """Return the S3 key for the live descriptor of *assembly_dir*. + + :param assembly_dir: full assembly directory name, e.g. ``GCF_000001215.4_Release_6_plus_ISO1_MT`` + :param key_prefix: Lakehouse key prefix (trailing slash optional) + :return: S3 key, e.g. ``tenant-general-warehouse/.../ncbi/metadata/GCF_..._datapackage.json`` + """ + prefix = key_prefix.rstrip("/") + "/" + return f"{prefix}metadata/{assembly_dir}_datapackage.json" + + +def build_archive_descriptor_key(assembly_dir: str, release_tag: str, key_prefix: str) -> str: + """Return the S3 key for the archived descriptor of *assembly_dir*. + + :param assembly_dir: full assembly directory name + :param release_tag: NCBI release tag used in the archive path, e.g. ``"2024-01"`` + :param key_prefix: Lakehouse key prefix + :return: S3 key under ``archive/{release_tag}/metadata/`` + """ + prefix = key_prefix.rstrip("/") + "/" + return f"{prefix}archive/{release_tag}/metadata/{assembly_dir}_datapackage.json" + + +def create_descriptor( + assembly_dir: str, + accession_full: str, + resources: list[DescriptorResource], + *, + timestamp: int | None = None, +) -> dict[str, Any]: + """Build a KBase credit metadata descriptor for an NCBI assembly. + + Matches the schema produced by + ``kbase-transfers/scripts/ncbi/download_genomes.py::create_frictionless_descriptor()``. + + Resource names are lowercased. Resources whose ``hash`` value is ``None`` + have the ``hash`` key removed entirely (frictionless does not accept null + hash values). + + :param assembly_dir: full assembly directory name (includes the accession + suffix, e.g. ``GCF_000001215.4_Release_6_plus_ISO1_MT``) + :param accession_full: accession without suffix, e.g. ``GCF_000001215.4`` + :param resources: list of :class:`DescriptorResource` dicts + :param timestamp: Unix timestamp to embed; defaults to ``datetime.now(UTC)`` + :return: descriptor dict ready for serialisation and frictionless validation + """ + ts = timestamp if timestamp is not None else int(datetime.now(UTC).timestamp()) + version = accession_full.rsplit(".", 1)[-1] # e.g. "4" from "GCF_000001215.4" + + # Normalise resources: lowercase name, drop null hash + normalised: list[dict[str, Any]] = [] + for res in resources: + entry: dict[str, Any] = { + "name": res["name"].lower(), + "path": res["path"], + "format": res.get("format", ""), + } + if res.get("bytes") is not None: + entry["bytes"] = res["bytes"] + if res.get("hash") is not None: + entry["hash"] = res["hash"] + normalised.append(entry) + + return { + "identifier": f"NCBI:{accession_full}", + "resource_type": "dataset", + "version": version, + "titles": [{"title": f"NCBI Genome Assembly {assembly_dir}"}], + "descriptions": [ + {"description_text": (f"Genome assembly files for {accession_full} downloaded from NCBI Datasets")} + ], + "url": f"https://www.ncbi.nlm.nih.gov/datasets/genome/{accession_full}/", + "contributors": [_NCBI_CONTRIBUTOR], + "publisher": _NCBI_PUBLISHER, + "license": {}, + "meta": { + "credit_metadata_schema_version": _SCHEMA_VERSION, + "credit_metadata_source": [ + { + "source_name": "NCBI Genomes FTP", + "source_url": "ftp.ncbi.nlm.nih.gov/genomes/all/", + "access_timestamp": ts, + } + ], + "saved_by": _SAVED_BY, + "timestamp": ts, + }, + "resources": normalised, + } + + +def validate_descriptor(descriptor: dict[str, Any], accession_full: str) -> None: + """Validate a descriptor with frictionless. + + :param descriptor: descriptor dict from :func:`create_descriptor` + :param accession_full: accession (used only in error messages) + :raises ValueError: if frictionless reports any metadata errors + """ + errors = list(Package.metadata_validate(descriptor)) + if errors: + error_details = "; ".join(str(e) for e in errors) + msg = f"Frictionless validation failed for {accession_full}: {error_details}" + raise ValueError(msg) + logger.debug("Frictionless descriptor valid for %s", accession_full) + + +def upload_descriptor( + descriptor: dict[str, Any], + assembly_dir: str, + bucket: str, + key_prefix: str, + *, + dry_run: bool = False, +) -> str: + """Serialise and upload a descriptor to the live ``metadata/`` path. + + :param descriptor: descriptor dict from :func:`create_descriptor` + :param assembly_dir: full assembly directory name + :param bucket: S3 bucket name + :param key_prefix: Lakehouse key prefix + :param dry_run: if True, log without uploading + :return: S3 key the descriptor was (or would be) written to + """ + key = build_descriptor_key(assembly_dir, key_prefix) + + if dry_run: + logger.info("[dry-run] would upload descriptor: s3://%s/%s", bucket, key) + return key + + s3 = get_s3_client() + body = json.dumps(descriptor, indent=2).encode() + + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as tmp: + tmp_path = tmp.name + tmp.write(body) + + try: + s3.upload_file(Filename=tmp_path, Bucket=bucket, Key=key) + logger.info("Uploaded descriptor: s3://%s/%s", bucket, key) + finally: + Path(tmp_path).unlink() + + return key + + +def archive_descriptor( # noqa: PLR0913 + assembly_dir: str, + bucket: str, + key_prefix: str, + release_tag: str, + *, + archive_reason: str = "unknown", + dry_run: bool = False, +) -> bool: + """Copy the live descriptor to the archive path. + + If the live descriptor does not yet exist (e.g. archival is triggered + before the first promote), logs a warning and returns ``False``. + + :param assembly_dir: full assembly directory name + :param bucket: S3 bucket name + :param key_prefix: Lakehouse key prefix + :param release_tag: NCBI release tag for the archive path + :param archive_reason: metadata value describing why archived (matches raw data metadata) + :param dry_run: if True, log without copying + :return: ``True`` if the descriptor was (or would be) archived; ``False`` if not found + """ + source_key = build_descriptor_key(assembly_dir, key_prefix) + archive_key = build_archive_descriptor_key(assembly_dir, release_tag, key_prefix) + + if dry_run: + logger.info("[dry-run] would archive descriptor: s3://%s/%s -> %s", bucket, source_key, archive_key) + return True + + s3 = get_s3_client() + try: + s3.head_object(Bucket=bucket, Key=source_key) + except s3.exceptions.NoSuchKey: + logger.warning("Descriptor not found, skipping archive: s3://%s/%s", bucket, source_key) + return False + except Exception as e: + # head_object raises ClientError with 404 when key is absent + if hasattr(e, "response") and e.response.get("Error", {}).get("Code") in ("404", "NoSuchKey"): # type: ignore[union-attr] + logger.warning("Descriptor not found, skipping archive: s3://%s/%s", bucket, source_key) + return False + raise + + datestamp = datetime.now(UTC).strftime("%Y-%m-%d") + copy_object_with_metadata( + f"{bucket}/{source_key}", + f"{bucket}/{archive_key}", + metadata={ + "ncbi_last_release": release_tag, + "archive_reason": archive_reason, + "archive_date": datestamp, + }, + ) + logger.debug("Archived descriptor: s3://%s/%s -> %s", bucket, source_key, archive_key) + return True diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index 6faf8c79..7767ccb2 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -8,12 +8,19 @@ import re import tempfile +from collections import defaultdict from datetime import UTC, datetime from pathlib import Path, PurePosixPath from typing import Any import botocore.exceptions +from cdm_data_loaders.ncbi_ftp.metadata import ( + DescriptorResource, + archive_descriptor, + create_descriptor, + upload_descriptor, +) from cdm_data_loaders.utils.cdm_logger import get_cdm_logger from cdm_data_loaders.utils.s3 import ( copy_object_with_metadata, @@ -60,9 +67,6 @@ def promote_from_s3( # noqa: PLR0913 paginator = s3.get_paginator("list_objects_v2") normalized_staging_key_prefix = staging_key_prefix.rstrip("/") + "/" - promoted = 0 - failed = 0 - # Collect all objects under the staging prefix staged_objects: list[str] = [] for page in paginator.paginate(Bucket=bucket, Prefix=normalized_staging_key_prefix): @@ -91,13 +95,79 @@ def promote_from_s3( # noqa: PLR0913 dry_run=dry_run, ) + promoted, failed, promoted_accessions, assembly_resources = _promote_data_files( + data_files, + sidecars, + normalized_staging_key_prefix, + lakehouse_key_prefix, + bucket, + dry_run=dry_run, + ) + + # Trim manifest for resumability + if manifest_s3_key and promoted_accessions and not dry_run: + _trim_manifest(manifest_s3_key, bucket, promoted_accessions) + + # Upload frictionless descriptors for each promoted assembly + descriptors_written = 0 + for (adir, acc), resources in assembly_resources.items(): + if not resources: + continue + try: + descriptor = create_descriptor(adir, acc, resources) + upload_descriptor(descriptor, adir, bucket, lakehouse_key_prefix, dry_run=dry_run) + descriptors_written += 1 + except Exception: + logger.exception("Failed to write descriptor for %s", adir) + + if descriptors_written: + logger.info("Wrote %d frictionless descriptor(s)", descriptors_written) + + report: dict[str, Any] = { + "timestamp": datetime.now(UTC).isoformat(), + "promoted": promoted, + "archived": archived, + "failed": failed, + "dry_run": dry_run, + } + + logger.info( + "PROMOTE SUMMARY: %d promoted, %d archived, %d failed%s", + promoted, + archived, + failed, + " (dry-run)" if dry_run else "", + ) + return report + + +# ── Promote data files (per-file loop) ────────────────────────────────── + + +def _promote_data_files( # noqa: PLR0913, PLR0915 + data_files: list[str], + sidecars: set[str], + normalized_staging_prefix: str, + lakehouse_key_prefix: str, + bucket: str, + *, + dry_run: bool, +) -> tuple[int, int, set[str], defaultdict[tuple[str, str], list[DescriptorResource]]]: + """Promote each data file from staging to the final Lakehouse path. + + :return: (promoted_count, failed_count, promoted_accessions, assembly_resources) + """ + s3 = get_s3_client() + promoted = 0 + failed = 0 promoted_accessions: set[str] = set() + assembly_resources: defaultdict[tuple[str, str], list[DescriptorResource]] = defaultdict(list) for staged_key in data_files: if staged_key.endswith("download_report.json"): continue - rel_path = staged_key[len(normalized_staging_key_prefix) :] + rel_path = staged_key[len(normalized_staging_prefix) :] if not rel_path.startswith("raw_data/"): continue final_key = lakehouse_key_prefix + rel_path @@ -139,32 +209,30 @@ def promote_from_s3( # noqa: PLR0913 if acc_match: promoted_accessions.add(acc_match.group(1)) + # Track resources for frictionless descriptor creation + adir_match = re.search(r"raw_data/GC[AF]/\d+/\d+/\d+/([^/]+)/", final_key) + if adir_match and acc_match: + adir = adir_match.group(1) + acc = acc_match.group(1) + fname = final_key_path.name + ext = fname.rsplit(".", 1)[-1] if "." in fname else "" + md5_hash = metadata.get("md5") + resource: DescriptorResource = { + "name": fname.lower(), + "path": final_key, + "format": ext, + "bytes": Path(tmp_path).stat().st_size, + "hash": md5_hash, + } + assembly_resources[(adir, acc)].append(resource) + finally: Path(tmp_path).unlink() except Exception: logger.exception("Failed to promote %s", staged_key) failed += 1 - # Trim manifest for resumability - if manifest_s3_key and promoted_accessions and not dry_run: - _trim_manifest(manifest_s3_key, bucket, promoted_accessions) - - report: dict[str, Any] = { - "timestamp": datetime.now(UTC).isoformat(), - "promoted": promoted, - "archived": archived, - "failed": failed, - "dry_run": dry_run, - } - - logger.info( - "PROMOTE SUMMARY: %d promoted, %d archived, %d failed%s", - promoted, - archived, - failed, - " (dry-run)" if dry_run else "", - ) - return report + return promoted, failed, promoted_accessions, assembly_resources # ── Archive assemblies ────────────────────────────────────────────────── @@ -223,6 +291,14 @@ def _archive_assemblies( # noqa: PLR0913 logger.debug("No objects found for %s, skipping archive", accession) continue + # Infer assembly_dir from key paths for descriptor archival + assembly_dir: str | None = None + for key in matching_keys: + adir_match = re.search(r"raw_data/GC[AF]/\d+/\d+/\d+/([^/]+)/", key) + if adir_match: + assembly_dir = adir_match.group(1) + break + for source_key in matching_keys: rel = source_key[len(lakehouse_key_prefix) :] archive_key = f"{lakehouse_key_prefix}archive/{release_tag}/{rel}" @@ -249,6 +325,20 @@ def _archive_assemblies( # noqa: PLR0913 except Exception: logger.exception("Failed to archive %s", source_key) + # Archive the frictionless descriptor alongside raw data + if assembly_dir: + try: + archive_descriptor( + assembly_dir, + bucket, + lakehouse_key_prefix, + release_tag, + archive_reason=archive_reason, + dry_run=dry_run, + ) + except Exception: + logger.exception("Failed to archive descriptor for %s", assembly_dir) + logger.info("Archived %d objects for %d accessions (%s)", archived, len(accessions), archive_reason) return archived diff --git a/tests/integration/test_promote_e2e.py b/tests/integration/test_promote_e2e.py index 69fa3999..b97aa542 100644 --- a/tests/integration/test_promote_e2e.py +++ b/tests/integration/test_promote_e2e.py @@ -11,11 +11,17 @@ from __future__ import annotations import hashlib +import json from typing import TYPE_CHECKING import pytest from cdm_data_loaders.ncbi_ftp.assembly import build_accession_path +from cdm_data_loaders.ncbi_ftp.metadata import ( + build_archive_descriptor_key, + build_descriptor_key, + create_descriptor, +) from cdm_data_loaders.ncbi_ftp.promote import DEFAULT_LAKEHOUSE_KEY_PREFIX, promote_from_s3 from .conftest import get_object_metadata, list_all_keys, seed_lakehouse @@ -318,3 +324,170 @@ def test_incomplete_staging(self, minio_s3_client: object, test_bucket: str) -> # No objects at final path final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") assert len(final_keys) == 0 + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestPromoteCreatesDescriptor: + """Promote step writes a frictionless descriptor for each promoted assembly.""" + + def test_descriptor_created(self, minio_s3_client: object, test_bucket: str) -> None: + """After promote, a JSON descriptor exists under ``metadata/``.""" + s3 = minio_s3_client + _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) + + promote_from_s3( + staging_key_prefix=STAGING_PREFIX, + bucket=test_bucket, + lakehouse_key_prefix=PATH_PREFIX, + ) + + descriptor_key = build_descriptor_key(ASSEMBLY_DIR_A, PATH_PREFIX) + obj = s3.get_object(Bucket=test_bucket, Key=descriptor_key) + body = json.loads(obj["Body"].read()) + + assert body["identifier"] == f"NCBI:{ACCESSION_A}" + assert body["resource_type"] == "dataset" + + def test_descriptor_resources_include_promoted_files(self, minio_s3_client: object, test_bucket: str) -> None: + """Descriptor's ``resources`` list references the final Lakehouse key.""" + s3 = minio_s3_client + _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) + + promote_from_s3( + staging_key_prefix=STAGING_PREFIX, + bucket=test_bucket, + lakehouse_key_prefix=PATH_PREFIX, + ) + + descriptor_key = build_descriptor_key(ASSEMBLY_DIR_A, PATH_PREFIX) + obj = s3.get_object(Bucket=test_bucket, Key=descriptor_key) + body = json.loads(obj["Body"].read()) + + resource_paths = [r["path"] for r in body["resources"]] + assert any(PATH_PREFIX + "raw_data/" in p for p in resource_paths) + + def test_descriptor_resources_have_md5(self, minio_s3_client: object, test_bucket: str) -> None: + """Resources with .md5 sidecars include the hash value.""" + s3 = minio_s3_client + _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) + + promote_from_s3( + staging_key_prefix=STAGING_PREFIX, + bucket=test_bucket, + lakehouse_key_prefix=PATH_PREFIX, + ) + + descriptor_key = build_descriptor_key(ASSEMBLY_DIR_A, PATH_PREFIX) + obj = s3.get_object(Bucket=test_bucket, Key=descriptor_key) + body = json.loads(obj["Body"].read()) + + # Both staged files have .md5 sidecars + for resource in body["resources"]: + assert "hash" in resource, f"Expected hash in resource: {resource}" + + def test_multiple_assemblies_get_separate_descriptors(self, minio_s3_client: object, test_bucket: str) -> None: + """Each assembly gets its own descriptor file.""" + s3 = minio_s3_client + _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) + _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_B) + + promote_from_s3( + staging_key_prefix=STAGING_PREFIX, + bucket=test_bucket, + lakehouse_key_prefix=PATH_PREFIX, + ) + + for assembly_dir, accession in [(ASSEMBLY_DIR_A, ACCESSION_A), (ASSEMBLY_DIR_B, ACCESSION_B)]: + key = build_descriptor_key(assembly_dir, PATH_PREFIX) + obj = s3.get_object(Bucket=test_bucket, Key=key) + body = json.loads(obj["Body"].read()) + assert body["identifier"] == f"NCBI:{accession}" + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestPromoteArchiveUpdatedIncludesDescriptor: + """Archiving updated assemblies also archives the descriptor.""" + + def test_archive_copies_descriptor(self, minio_s3_client: object, test_bucket: str, tmp_path: Path) -> None: + """After archiving an updated assembly, the descriptor appears under archive/.""" + s3 = minio_s3_client + + # Seed old version at Lakehouse path *including* a live descriptor + old_files = {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": "old content"} + seed_lakehouse(s3, test_bucket, ACCESSION_A, old_files, PATH_PREFIX, ASSEMBLY_DIR_A) + # Pre-upload a descriptor so archive_descriptor can find it + descriptor = create_descriptor(ASSEMBLY_DIR_A, ACCESSION_A, []) + # Upload directly to MinIO (not via promote) + descriptor_key = build_descriptor_key(ASSEMBLY_DIR_A, PATH_PREFIX) + s3.put_object(Bucket=test_bucket, Key=descriptor_key, Body=json.dumps(descriptor).encode()) + + _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) + updated_manifest = _write_manifest(tmp_path, [ACCESSION_A], "updated_manifest.txt") + + promote_from_s3( + staging_key_prefix=STAGING_PREFIX, + bucket=test_bucket, + updated_manifest_path=str(updated_manifest), + ncbi_release="2024-01", + lakehouse_key_prefix=PATH_PREFIX, + ) + + archive_key = build_archive_descriptor_key(ASSEMBLY_DIR_A, "2024-01", PATH_PREFIX) + # Confirm the archive descriptor object exists + resp = s3.head_object(Bucket=test_bucket, Key=archive_key) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestPromoteArchiveRemovedIncludesDescriptor: + """Archiving removed assemblies also archives the descriptor.""" + + def test_archive_removed_copies_descriptor(self, minio_s3_client: object, test_bucket: str, tmp_path: Path) -> None: + """After archiving a removed assembly, the descriptor is under archive/.""" + s3 = minio_s3_client + + # Seed the assembly at final Lakehouse path + files = {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": "content"} + seed_lakehouse(s3, test_bucket, ACCESSION_A, files, PATH_PREFIX, ASSEMBLY_DIR_A) + # Pre-upload a descriptor + descriptor = create_descriptor(ASSEMBLY_DIR_A, ACCESSION_A, []) + descriptor_key = build_descriptor_key(ASSEMBLY_DIR_A, PATH_PREFIX) + s3.put_object(Bucket=test_bucket, Key=descriptor_key, Body=json.dumps(descriptor).encode()) + + removed_manifest = _write_manifest(tmp_path, [ACCESSION_A], "removed_manifest.txt") + + promote_from_s3( + staging_key_prefix=STAGING_PREFIX, + bucket=test_bucket, + removed_manifest_path=str(removed_manifest), + ncbi_release="2024-01", + lakehouse_key_prefix=PATH_PREFIX, + ) + + archive_key = build_archive_descriptor_key(ASSEMBLY_DIR_A, "2024-01", PATH_PREFIX) + resp = s3.head_object(Bucket=test_bucket, Key=archive_key) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestPromoteDryRunNoDescriptor: + """Dry-run must not write any descriptor files.""" + + def test_dry_run_no_descriptor(self, minio_s3_client: object, test_bucket: str) -> None: + """Dry-run does not upload a descriptor to the metadata/ prefix.""" + s3 = minio_s3_client + _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) + + promote_from_s3( + staging_key_prefix=STAGING_PREFIX, + bucket=test_bucket, + lakehouse_key_prefix=PATH_PREFIX, + dry_run=True, + ) + + metadata_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "metadata/") + assert len(metadata_keys) == 0, f"Dry-run should not create descriptor files, found: {metadata_keys}" diff --git a/tests/ncbi_ftp/test_metadata.py b/tests/ncbi_ftp/test_metadata.py new file mode 100644 index 00000000..821ac81d --- /dev/null +++ b/tests/ncbi_ftp/test_metadata.py @@ -0,0 +1,369 @@ +"""Unit tests for cdm_data_loaders.ncbi_ftp.metadata.""" + +from __future__ import annotations + +import json +import time +from typing import TYPE_CHECKING +from unittest.mock import MagicMock, patch + +import boto3 +import pytest +from moto import mock_aws + +if TYPE_CHECKING: + from collections.abc import Generator + + import botocore.client + +import cdm_data_loaders.ncbi_ftp.metadata as metadata_mod +import cdm_data_loaders.utils.s3 as s3_utils +from cdm_data_loaders.ncbi_ftp.metadata import ( + DescriptorResource, + archive_descriptor, + build_archive_descriptor_key, + build_descriptor_key, + create_descriptor, + upload_descriptor, + validate_descriptor, +) +from cdm_data_loaders.utils.s3 import reset_s3_client +from tests.ncbi_ftp.conftest import TEST_BUCKET + +AWS_REGION = "us-east-1" + +_ACCESSION = "GCF_000001215.4" +_ASSEMBLY_DIR = "GCF_000001215.4_Release_6_plus_ISO1_MT" +_RELEASE_TAG = "2024-01" +_KEY_PREFIX = "tenant-general-warehouse/kbase/datasets/ncbi/" +_TIMESTAMP = 1_700_000_000 + +_SAMPLE_RESOURCES: list[DescriptorResource] = [ + { + "name": "GCF_000001215.4_genomic.fna.gz", + "path": f"{_KEY_PREFIX}raw_data/GCF/000/001/215/{_ASSEMBLY_DIR}/GCF_000001215.4_genomic.fna.gz", + "format": "gz", + "bytes": 1024, + "hash": "abc123", + }, + { + "name": "GCF_000001215.4_assembly_report.txt", + "path": f"{_KEY_PREFIX}raw_data/GCF/000/001/215/{_ASSEMBLY_DIR}/GCF_000001215.4_assembly_report.txt", + "format": "txt", + "bytes": 512, + "hash": None, # no md5 sidecar for this one + }, +] + + +# ── build_descriptor_key ───────────────────────────────────────────────── + + +class TestBuildDescriptorKey: + """Tests for build_descriptor_key path helper.""" + + def test_produces_metadata_path(self) -> None: + """Key is located under metadata/ with _datapackage.json suffix.""" + key = build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX) + assert key == f"{_KEY_PREFIX}metadata/{_ASSEMBLY_DIR}_datapackage.json" + + def test_trailing_slash_normalised(self) -> None: + """Key is the same whether key_prefix ends with a slash or not.""" + key_no_slash = build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX.rstrip("/")) + key_with_slash = build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX) + assert key_no_slash == key_with_slash + + def test_no_double_slash(self) -> None: + """Key never contains a double slash.""" + key = build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX) + assert "//" not in key + + +# ── build_archive_descriptor_key ───────────────────────────────────────── + + +class TestBuildArchiveDescriptorKey: + """Tests for build_archive_descriptor_key path helper.""" + + def test_produces_archive_path(self) -> None: + """Key is located under archive/{release_tag}/metadata/.""" + key = build_archive_descriptor_key(_ASSEMBLY_DIR, _RELEASE_TAG, _KEY_PREFIX) + expected = f"{_KEY_PREFIX}archive/{_RELEASE_TAG}/metadata/{_ASSEMBLY_DIR}_datapackage.json" + assert key == expected + + def test_trailing_slash_normalised(self) -> None: + """Key is the same whether key_prefix ends with a slash or not.""" + a = build_archive_descriptor_key(_ASSEMBLY_DIR, _RELEASE_TAG, _KEY_PREFIX.rstrip("/")) + b = build_archive_descriptor_key(_ASSEMBLY_DIR, _RELEASE_TAG, _KEY_PREFIX) + assert a == b + + def test_release_tag_in_path(self) -> None: + """Release tag appears in the archive key path.""" + key = build_archive_descriptor_key(_ASSEMBLY_DIR, "2025-06", _KEY_PREFIX) + assert "2025-06" in key + + +# ── create_descriptor ──────────────────────────────────────────────────── + + +class TestCreateDescriptor: + """Tests for create_descriptor().""" + + def test_identifier(self) -> None: + """Identifier field is prefixed with NCBI:.""" + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + assert d["identifier"] == f"NCBI:{_ACCESSION}" + + def test_version_extracted_from_accession(self) -> None: + """Version is the suffix after the last dot in the accession.""" + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + assert d["version"] == "4" # last segment of GCF_000001215.4 + + def test_title_includes_assembly_dir(self) -> None: + """Title includes the full assembly directory name.""" + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + assert _ASSEMBLY_DIR in d["titles"][0]["title"] + + def test_description_includes_accession(self) -> None: + """Description text includes the accession.""" + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + assert _ACCESSION in d["descriptions"][0]["description_text"] + + def test_url_references_accession(self) -> None: + """URL points to the NCBI genome page for the accession.""" + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + assert _ACCESSION in d["url"] + assert "ncbi.nlm.nih.gov" in d["url"] + + def test_ncbi_contributor(self) -> None: + """Contributor is NCBI with the correct ROR ID.""" + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + assert d["contributors"][0]["name"] == "National Center for Biotechnology Information" + assert d["contributors"][0]["contributor_id"] == "ROR:02meqm098" + + def test_saved_by(self) -> None: + """meta.saved_by is the cdm-data-loaders identifier.""" + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + assert d["meta"]["saved_by"] == "cdm-data-loaders-ncbi-ftp" + + def test_timestamp_propagated(self) -> None: + """Explicit timestamp is used for both meta.timestamp and access_timestamp.""" + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + assert d["meta"]["timestamp"] == _TIMESTAMP + assert d["meta"]["credit_metadata_source"][0]["access_timestamp"] == _TIMESTAMP + + def test_default_timestamp_is_recent(self) -> None: + """Default timestamp is close to current time when not specified.""" + before = int(time.time()) + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES) + after = int(time.time()) + ts = d["meta"]["timestamp"] + assert before <= ts <= after + 1 + + def test_resource_names_lowercased(self) -> None: + """Resource names are converted to lowercase.""" + resources: list[DescriptorResource] = [ + {"name": "FILE_UPPER.FNA.GZ", "path": "s3://bucket/a", "format": "gz", "bytes": 100, "hash": "x"}, + ] + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, resources, timestamp=_TIMESTAMP) + assert d["resources"][0]["name"] == "file_upper.fna.gz" + + def test_null_hash_omitted(self) -> None: + """Resources with hash=None must not include the 'hash' key.""" + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + resources = d["resources"] + # Second resource has hash=None → key absent + assert "hash" not in resources[1] + + def test_non_null_hash_present(self) -> None: + """Non-null hash is retained in the resource entry.""" + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + assert d["resources"][0]["hash"] == _SAMPLE_RESOURCES[0]["hash"] + + def test_resource_count(self) -> None: + """Resource list length matches the number of input resources.""" + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + assert len(d["resources"]) == len(_SAMPLE_RESOURCES) + + def test_resource_bytes(self) -> None: + """Resource bytes matches the input bytes value.""" + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + assert d["resources"][0]["bytes"] == _SAMPLE_RESOURCES[0]["bytes"] + + def test_resource_path(self) -> None: + """Resource path matches the input path value.""" + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + assert d["resources"][0]["path"] == _SAMPLE_RESOURCES[0]["path"] + + def test_license_is_empty_dict(self) -> None: + """License field is an empty dict.""" + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + assert d["license"] == {} + + def test_resource_type_is_dataset(self) -> None: + """resource_type is 'dataset'.""" + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + assert d["resource_type"] == "dataset" + + def test_schema_version(self) -> None: + """credit_metadata_schema_version is '1.0'.""" + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + assert d["meta"]["credit_metadata_schema_version"] == "1.0" + + def test_empty_resources_allowed(self) -> None: + """Empty resources list produces a valid descriptor.""" + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, [], timestamp=_TIMESTAMP) + assert d["resources"] == [] + + def test_null_bytes_omitted(self) -> None: + """Resources with bytes=None have the 'bytes' key removed from the output.""" + resources: list[DescriptorResource] = [ + {"name": "f.txt", "path": "s3://b/f.txt", "format": "txt", "bytes": None, "hash": "x"}, + ] + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, resources, timestamp=_TIMESTAMP) + assert "bytes" not in d["resources"][0] + + +# ── validate_descriptor ────────────────────────────────────────────────── + + +class TestValidateDescriptor: + """Tests for validate_descriptor().""" + + def test_valid_descriptor_passes(self) -> None: + """Valid descriptor does not raise.""" + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + # Should not raise + validate_descriptor(d, _ACCESSION) + + def test_empty_descriptor_raises(self) -> None: + """Empty dict fails frictionless validation and raises.""" + with pytest.raises((ValueError, Exception)): + validate_descriptor({}, _ACCESSION) + + +# ── upload_descriptor ──────────────────────────────────────────────────── + + +@pytest.mark.s3 +class TestUploadDescriptor: + """Tests for upload_descriptor() using moto-mocked S3.""" + + @pytest.fixture + def mock_s3(self) -> Generator[botocore.client.BaseClient]: + """Yield a mocked S3 client with the CDM Lake bucket pre-created.""" + with mock_aws(): + client = boto3.client("s3", region_name=AWS_REGION) + client.create_bucket(Bucket=TEST_BUCKET) + reset_s3_client() + with ( + patch.object(s3_utils, "get_s3_client", return_value=client), + patch.object(metadata_mod, "get_s3_client", return_value=client), + ): + yield client + reset_s3_client() + + def test_uploads_json(self, mock_s3: botocore.client.BaseClient) -> None: + """Uploaded object is valid JSON with the expected identifier.""" + descriptor = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + key = upload_descriptor(descriptor, _ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX) + assert key == build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX) + obj = mock_s3.get_object(Bucket=TEST_BUCKET, Key=key) + body = json.loads(obj["Body"].read()) + assert body["identifier"] == f"NCBI:{_ACCESSION}" + + def test_returns_expected_key(self, mock_s3: botocore.client.BaseClient) -> None: + """Return value is the metadata/ S3 key for the assembly.""" + descriptor = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + key = upload_descriptor(descriptor, _ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX) + assert key.startswith(_KEY_PREFIX) + assert key.endswith("_datapackage.json") + + def test_dry_run_skips_upload(self, mock_s3: botocore.client.BaseClient) -> None: + """Dry-run returns the key but does not create any S3 object.""" + descriptor = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + key = upload_descriptor(descriptor, _ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX, dry_run=True) + # No object in S3 + objs = mock_s3.list_objects_v2(Bucket=TEST_BUCKET).get("Contents", []) + assert not any(o["Key"] == key for o in objs) + + def test_dry_run_returns_key(self, mock_s3: botocore.client.BaseClient) -> None: + """Dry-run returns the same key as a real upload would.""" + descriptor = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + key = upload_descriptor(descriptor, _ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX, dry_run=True) + assert key == build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX) + + +# ── archive_descriptor ─────────────────────────────────────────────────── + + +@pytest.mark.s3 +class TestArchiveDescriptor: + """Tests for archive_descriptor() using moto-mocked S3.""" + + @pytest.fixture + def mock_s3_with_descriptor(self) -> Generator[tuple[botocore.client.BaseClient, MagicMock]]: + """S3 with a live descriptor already uploaded.""" + with mock_aws(): + client = boto3.client("s3", region_name=AWS_REGION) + client.create_bucket(Bucket=TEST_BUCKET) + # Pre-upload a descriptor + descriptor = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + live_key = build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX) + client.put_object( + Bucket=TEST_BUCKET, + Key=live_key, + Body=json.dumps(descriptor).encode(), + ) + reset_s3_client() + with ( + patch.object(s3_utils, "get_s3_client", return_value=client), + patch.object(metadata_mod, "get_s3_client", return_value=client), + patch.object(metadata_mod, "copy_object_with_metadata") as mock_copy, + ): + yield client, mock_copy + reset_s3_client() + + def test_returns_true_when_descriptor_exists( + self, mock_s3_with_descriptor: tuple[botocore.client.BaseClient, MagicMock] + ) -> None: + """Returns True when the live descriptor object exists in S3.""" + _, _ = mock_s3_with_descriptor + result = archive_descriptor(_ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX, _RELEASE_TAG) + assert result is True + + def test_calls_copy_with_correct_keys( + self, mock_s3_with_descriptor: tuple[botocore.client.BaseClient, MagicMock] + ) -> None: + """copy_object_with_metadata is called with the live and archive keys.""" + _, mock_copy = mock_s3_with_descriptor + archive_descriptor(_ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX, _RELEASE_TAG) + live_key = build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX) + archive_key = build_archive_descriptor_key(_ASSEMBLY_DIR, _RELEASE_TAG, _KEY_PREFIX) + mock_copy.assert_called_once() + args = mock_copy.call_args + assert f"{TEST_BUCKET}/{live_key}" in args[0] + assert f"{TEST_BUCKET}/{archive_key}" in args[0] + + def test_dry_run_returns_true_without_copy( + self, mock_s3_with_descriptor: tuple[botocore.client.BaseClient, MagicMock] + ) -> None: + """Dry-run returns True but does not call copy_object_with_metadata.""" + _, mock_copy = mock_s3_with_descriptor + result = archive_descriptor(_ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX, _RELEASE_TAG, dry_run=True) + assert result is True + mock_copy.assert_not_called() + + def test_missing_descriptor_returns_false(self) -> None: + """Returns False when no descriptor exists at the live key.""" + with mock_aws(): + client = boto3.client("s3", region_name=AWS_REGION) + client.create_bucket(Bucket=TEST_BUCKET) + reset_s3_client() + with ( + patch.object(s3_utils, "get_s3_client", return_value=client), + patch.object(metadata_mod, "get_s3_client", return_value=client), + ): + result = archive_descriptor(_ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX, _RELEASE_TAG) + reset_s3_client() + assert result is False From 21dda1cf6f834f5db320e2b71f03591f3957c318 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 22 Apr 2026 11:59:52 -0700 Subject: [PATCH 035/128] Potential fix for pull request finding 'CodeQL / Incomplete URL substring sanitization' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- tests/ncbi_ftp/test_metadata.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/ncbi_ftp/test_metadata.py b/tests/ncbi_ftp/test_metadata.py index 821ac81d..446378f3 100644 --- a/tests/ncbi_ftp/test_metadata.py +++ b/tests/ncbi_ftp/test_metadata.py @@ -6,6 +6,7 @@ import time from typing import TYPE_CHECKING from unittest.mock import MagicMock, patch +from urllib.parse import urlparse import boto3 import pytest @@ -133,7 +134,8 @@ def test_url_references_accession(self) -> None: """URL points to the NCBI genome page for the accession.""" d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) assert _ACCESSION in d["url"] - assert "ncbi.nlm.nih.gov" in d["url"] + parsed = urlparse(d["url"]) + assert parsed.hostname == "ncbi.nlm.nih.gov" def test_ncbi_contributor(self) -> None: """Contributor is NCBI with the correct ROR ID.""" From 803d8c4989947f0cb8052277b261f7f6393b3f0e Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 22 Apr 2026 12:14:00 -0700 Subject: [PATCH 036/128] update url parsing test --- tests/ncbi_ftp/test_metadata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/ncbi_ftp/test_metadata.py b/tests/ncbi_ftp/test_metadata.py index 446378f3..d9c307bb 100644 --- a/tests/ncbi_ftp/test_metadata.py +++ b/tests/ncbi_ftp/test_metadata.py @@ -135,7 +135,8 @@ def test_url_references_accession(self) -> None: d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) assert _ACCESSION in d["url"] parsed = urlparse(d["url"]) - assert parsed.hostname == "ncbi.nlm.nih.gov" + assert parsed.hostname is not None + assert parsed.hostname.endswith("ncbi.nlm.nih.gov") def test_ncbi_contributor(self) -> None: """Contributor is NCBI with the correct ROR ID.""" From 280bb635b6cbd83fce180bf4a1f765778b687ba2 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 22 Apr 2026 12:16:05 -0700 Subject: [PATCH 037/128] Potential fix for pull request finding 'CodeQL / Incomplete URL substring sanitization' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- tests/ncbi_ftp/test_metadata.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/ncbi_ftp/test_metadata.py b/tests/ncbi_ftp/test_metadata.py index d9c307bb..cc13c192 100644 --- a/tests/ncbi_ftp/test_metadata.py +++ b/tests/ncbi_ftp/test_metadata.py @@ -135,8 +135,9 @@ def test_url_references_accession(self) -> None: d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) assert _ACCESSION in d["url"] parsed = urlparse(d["url"]) - assert parsed.hostname is not None - assert parsed.hostname.endswith("ncbi.nlm.nih.gov") + host = parsed.hostname + assert host is not None + assert host == "ncbi.nlm.nih.gov" or host.endswith(".ncbi.nlm.nih.gov") def test_ncbi_contributor(self) -> None: """Contributor is NCBI with the correct ROR ID.""" From 09df49611c06624dcfcf3b8a6700a2e2578c2954 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 23 Apr 2026 12:32:28 -0700 Subject: [PATCH 038/128] merge file upload functions Co-authored-by: Copilot --- src/cdm_data_loaders/ncbi_ftp/promote.py | 4 +- src/cdm_data_loaders/utils/s3.py | 73 ++++++------------------ tests/utils/test_s3.py | 20 +++---- 3 files changed, 28 insertions(+), 69 deletions(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index 7767ccb2..6edd293c 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -26,7 +26,7 @@ copy_object_with_metadata, delete_object, get_s3_client, - upload_file_with_metadata, + upload_file, ) logger = get_cdm_logger() @@ -191,7 +191,7 @@ def _promote_data_files( # noqa: PLR0913, PLR0915 metadata["md5"] = md5_obj["Body"].read().decode().strip() final_key_path = PurePosixPath(final_key) - upload_succeeded = upload_file_with_metadata( + upload_succeeded = upload_file( tmp_path, f"{bucket}/{final_key_path.parent}", metadata=metadata, diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index 56f47a37..f6f4f632 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -167,15 +167,23 @@ def upload_file( local_file_path: Path | str, destination_dir: str, object_name: str | None = None, + metadata: dict[str, str] | None = None, ) -> bool: """Upload an object to an S3 bucket. + When *metadata* is supplied the file is always uploaded (no existence check) + and the dict is attached as S3 user metadata. When *metadata* is ``None`` + (the default) the existing behaviour is preserved: the upload is skipped if + the object is already present. + :param local_file_path: File to upload :type local_file_path: Path | str :param destination_dir: path to the destination directory on s3, INCLUDING the bucket name and EXCLUDING the file name :type destination_dir: str :param object_name: S3 object name. If not specified, the name of the file from local_file_path is used. :type object_name: str | None + :param metadata: user metadata key/value pairs to attach to the object; when provided the upload always runs + :type metadata: dict[str, str] | None :return: True if file was uploaded, else False :rtype: bool """ @@ -190,13 +198,17 @@ def upload_file( object_name = local_file_path.name s3_path = f"{destination_dir.removesuffix('/')}/{object_name}" - if object_exists(s3_path): - print(f"File already present: {s3_path}") # noqa: T201 - return True + + if metadata is None: + if object_exists(s3_path): + print(f"File already present: {s3_path}") # noqa: T201 + return True s3 = get_s3_client() (bucket, key) = split_s3_path(s3_path) + extra_args = {**DEFAULT_EXTRA_ARGS, **(({"Metadata": metadata}) if metadata is not None else {})} + # Upload the file file_size = local_file_path.stat().st_size with tqdm.tqdm(total=file_size, unit="B", unit_scale=True, desc=str(local_file_path)) as pbar: @@ -207,7 +219,7 @@ def upload_file( Bucket=bucket, Key=key, Callback=pbar.update, - ExtraArgs=DEFAULT_EXTRA_ARGS, + ExtraArgs=extra_args, ) except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: print(f"Error uploading to s3: {e!s}") # noqa: T201 @@ -421,59 +433,6 @@ def delete_object(s3_path: str) -> dict[str, Any]: return s3.delete_object(Bucket=bucket, Key=key) -def upload_file_with_metadata( - local_file_path: Path | str, - destination_dir: str, - metadata: dict[str, str], - object_name: str | None = None, -) -> bool: - """Upload a file to S3 with user-defined metadata and CRC64NVME checksum. - - Unlike :func:`upload_file`, this function always uploads (no existence check) - and attaches the supplied *metadata* dict as S3 user metadata. - - :param local_file_path: file to upload - :type local_file_path: Path | str - :param destination_dir: path to the destination directory on s3, INCLUDING the bucket name - :type destination_dir: str - :param metadata: user metadata key/value pairs to attach to the object - :type metadata: dict[str, str] - :param object_name: S3 object name; defaults to the local filename - :type object_name: str | None - :return: True if the upload succeeded, otherwise False - :rtype: bool - """ - if isinstance(local_file_path, str): - local_file_path = Path(local_file_path) - - if not destination_dir: - msg = "No destination directory supplied for the file" - raise ValueError(msg) - - if not object_name: - object_name = local_file_path.name - - s3_path = f"{destination_dir.removesuffix('/')}/{object_name}" - s3 = get_s3_client() - (bucket, key) = split_s3_path(s3_path) - - extra_args = {**DEFAULT_EXTRA_ARGS, "Metadata": metadata} - - file_size = local_file_path.stat().st_size - try: - with tqdm.tqdm(total=file_size, unit="B", unit_scale=True, desc=str(local_file_path)) as pbar: - s3.upload_file( - Filename=str(local_file_path), - Bucket=bucket, - Key=key, - Callback=pbar.update, - ExtraArgs=extra_args, - ) - except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError): - return False - return True - - def head_object(s3_path: str) -> dict[str, Any] | None: """Return metadata for an S3 object, or None if it does not exist. diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index 6bd00208..a2acc80f 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -31,7 +31,7 @@ stream_to_s3, upload_dir, upload_file, - upload_file_with_metadata, + ) AWS_REGION = "us-east-1" @@ -782,13 +782,13 @@ def test_delete_object_removes_object(mock_s3_client: Any, bucket: str, protocol assert resp.get("ResponseMetadata", {}).get("HTTPStatusCode") == HTTP_STATUS_NO_CONTENT -# upload_file_with_metadata +# upload_file with metadata @pytest.mark.parametrize("bucket", BUCKETS) @pytest.mark.s3 def test_upload_file_with_metadata_attaches_metadata(mock_s3_client: Any, sample_file: Path, bucket: str) -> None: - """Verify that upload_file_with_metadata stores user metadata on the uploaded object.""" + """Verify that upload_file with metadata stores user metadata on the uploaded object.""" metadata = {"md5": "abc123", "source": "ncbi"} - result = upload_file_with_metadata(sample_file, f"{bucket}/uploads", metadata=metadata) + result = upload_file(sample_file, f"{bucket}/uploads", metadata=metadata) assert result is True resp = mock_s3_client.head_object(Bucket=bucket, Key=f"uploads/{sample_file.name}") @@ -799,7 +799,7 @@ def test_upload_file_with_metadata_attaches_metadata(mock_s3_client: Any, sample @pytest.mark.s3 def test_upload_file_with_metadata_custom_object_name(mock_s3_client: Any, sample_file: Path) -> None: """Verify that the object_name parameter overrides the filename.""" - result = upload_file_with_metadata( + result = upload_file( sample_file, f"{CDM_LAKE_BUCKET}/uploads", metadata={"k": "v"}, object_name="renamed.txt" ) assert result is True @@ -809,9 +809,9 @@ def test_upload_file_with_metadata_custom_object_name(mock_s3_client: Any, sampl @pytest.mark.s3 def test_upload_file_with_metadata_overwrites_existing(mock_s3_client: Any, sample_file: Path) -> None: - """Verify that upload_file_with_metadata uploads even when the object already exists.""" + """Verify that upload_file with metadata uploads even when the object already exists.""" mock_s3_client.put_object(Bucket=CDM_LAKE_BUCKET, Key=f"uploads/{sample_file.name}", Body=b"old") - result = upload_file_with_metadata(sample_file, f"{CDM_LAKE_BUCKET}/uploads", metadata={"new": "true"}) + result = upload_file(sample_file, f"{CDM_LAKE_BUCKET}/uploads", metadata={"new": "true"}) assert result is True obj = mock_s3_client.get_object(Bucket=CDM_LAKE_BUCKET, Key=f"uploads/{sample_file.name}") assert obj["Body"].read() == b"hello s3" @@ -822,15 +822,15 @@ def test_upload_file_with_metadata_overwrites_existing(mock_s3_client: Any, samp def test_upload_file_with_metadata_raises_on_empty_destination(sample_file: Path) -> None: """Verify ValueError when destination_dir is empty.""" with pytest.raises(ValueError, match="No destination directory"): - upload_file_with_metadata(sample_file, "", metadata={"k": "v"}) + upload_file(sample_file, "", metadata={"k": "v"}) @pytest.mark.usefixtures("mock_s3_client") @pytest.mark.parametrize("path_type", [str, Path]) @pytest.mark.s3 def test_upload_file_with_metadata_accepts_str_and_path(sample_file: Path, path_type: type[str] | type[Path]) -> None: - """Verify that upload_file_with_metadata accepts both str and Path.""" - result = upload_file_with_metadata(path_type(sample_file), f"{CDM_LAKE_BUCKET}/uploads", metadata={}) + """Verify that upload_file with metadata accepts both str and Path.""" + result = upload_file(path_type(sample_file), f"{CDM_LAKE_BUCKET}/uploads", metadata={}) assert result is True From c56c3c9f22e1d12c56edfa81fd7e8c28d0c3d8df Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 23 Apr 2026 12:35:20 -0700 Subject: [PATCH 039/128] formatting --- tests/utils/test_s3.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index a2acc80f..a96a5cab 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -31,7 +31,6 @@ stream_to_s3, upload_dir, upload_file, - ) AWS_REGION = "us-east-1" @@ -799,9 +798,7 @@ def test_upload_file_with_metadata_attaches_metadata(mock_s3_client: Any, sample @pytest.mark.s3 def test_upload_file_with_metadata_custom_object_name(mock_s3_client: Any, sample_file: Path) -> None: """Verify that the object_name parameter overrides the filename.""" - result = upload_file( - sample_file, f"{CDM_LAKE_BUCKET}/uploads", metadata={"k": "v"}, object_name="renamed.txt" - ) + result = upload_file(sample_file, f"{CDM_LAKE_BUCKET}/uploads", metadata={"k": "v"}, object_name="renamed.txt") assert result is True obj = mock_s3_client.get_object(Bucket=CDM_LAKE_BUCKET, Key="uploads/renamed.txt") assert obj["Body"].read() == b"hello s3" From 6d6415beb64dfde2d238cb42e05f61074fa39a8f Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 23 Apr 2026 12:49:21 -0700 Subject: [PATCH 040/128] merge copy object functions Co-authored-by: Copilot --- src/cdm_data_loaders/ncbi_ftp/metadata.py | 4 +- src/cdm_data_loaders/ncbi_ftp/promote.py | 4 +- src/cdm_data_loaders/utils/s3.py | 59 ++++++++--------------- tests/ncbi_ftp/test_metadata.py | 6 +-- tests/utils/test_s3.py | 10 ++-- 5 files changed, 33 insertions(+), 50 deletions(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/metadata.py b/src/cdm_data_loaders/ncbi_ftp/metadata.py index 356ee0d6..4ef60363 100644 --- a/src/cdm_data_loaders/ncbi_ftp/metadata.py +++ b/src/cdm_data_loaders/ncbi_ftp/metadata.py @@ -27,7 +27,7 @@ from frictionless import Package from cdm_data_loaders.utils.cdm_logger import get_cdm_logger -from cdm_data_loaders.utils.s3 import copy_object_with_metadata, get_s3_client +from cdm_data_loaders.utils.s3 import copy_object, get_s3_client logger = get_cdm_logger() @@ -246,7 +246,7 @@ def archive_descriptor( # noqa: PLR0913 raise datestamp = datetime.now(UTC).strftime("%Y-%m-%d") - copy_object_with_metadata( + copy_object( f"{bucket}/{source_key}", f"{bucket}/{archive_key}", metadata={ diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index 6edd293c..ab345ba3 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -23,7 +23,7 @@ ) from cdm_data_loaders.utils.cdm_logger import get_cdm_logger from cdm_data_loaders.utils.s3 import ( - copy_object_with_metadata, + copy_object, delete_object, get_s3_client, upload_file, @@ -309,7 +309,7 @@ def _archive_assemblies( # noqa: PLR0913 continue try: - copy_object_with_metadata( + copy_object( f"{bucket}/{source_key}", f"{bucket}/{archive_key}", metadata={ diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index f6f4f632..13a850bd 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -386,19 +386,29 @@ def upload_dir( return all_successful -def copy_object(current_s3_path: str, new_s3_path: str) -> dict[str, Any]: +def copy_object( + current_s3_path: str, + new_s3_path: str, + metadata: dict[str, str] | None = None, +) -> dict[str, Any]: """Copy an object from one place to another, adding in a CRC64NVME checksum. + When *metadata* is supplied the destination object carries exactly those + key/value pairs (``MetadataDirective='REPLACE'``). When *metadata* is + ``None`` (the default) the source metadata is inherited. + A successful copy operation will return a response where resp["ResponseMetadata"]["HTTPStatusCode"] == 200 Errors (e.g, buckets or keys not existing, wrong credentials, etc.) are passed directly to the user without being caught. - :param current_path: path to the file on s3, INCLUDING the bucket name - :type current_path: str - :param new_path: the desired new file path on s3, INCLUDING the bucket name - :type new_path: str + :param current_s3_path: path to the file on s3, INCLUDING the bucket name + :type current_s3_path: str + :param new_s3_path: the desired new file path on s3, INCLUDING the bucket name + :type new_s3_path: str + :param metadata: user metadata to set on the destination object; when provided the source metadata is replaced + :type metadata: dict[str, str] | None :return: dictionary containing response :rtype: dict[str, Any] """ @@ -406,10 +416,16 @@ def copy_object(current_s3_path: str, new_s3_path: str) -> dict[str, Any]: (current_s3_bucket, current_s3_key) = split_s3_path(current_s3_path) (new_s3_bucket, new_s3_key) = split_s3_path(new_s3_path) + extra: dict[str, Any] = {} + if metadata is not None: + extra["Metadata"] = metadata + extra["MetadataDirective"] = "REPLACE" + return s3.copy_object( CopySource={"Bucket": current_s3_bucket, "Key": current_s3_key}, Bucket=new_s3_bucket, Key=new_s3_key, + **extra, **DEFAULT_EXTRA_ARGS, ) @@ -461,37 +477,4 @@ def head_object(s3_path: str) -> dict[str, Any] | None: } -def copy_object_with_metadata( - current_s3_path: str, - new_s3_path: str, - metadata: dict[str, str], -) -> dict[str, Any]: - """Copy an S3 object to a new location, replacing its user metadata. - - Uses ``MetadataDirective='REPLACE'`` so the destination object carries - exactly the supplied *metadata* rather than inheriting the source's metadata. - - A successful copy returns a response where - ``resp["ResponseMetadata"]["HTTPStatusCode"] == 200``. - - :param current_s3_path: source path on s3, INCLUDING the bucket name - :type current_s3_path: str - :param new_s3_path: destination path on s3, INCLUDING the bucket name - :type new_s3_path: str - :param metadata: user metadata to set on the destination object - :type metadata: dict[str, str] - :return: dictionary containing response - :rtype: dict[str, Any] - """ - s3 = get_s3_client() - (current_bucket, current_key) = split_s3_path(current_s3_path) - (new_bucket, new_key) = split_s3_path(new_s3_path) - return s3.copy_object( - CopySource={"Bucket": current_bucket, "Key": current_key}, - Bucket=new_bucket, - Key=new_key, - Metadata=metadata, - MetadataDirective="REPLACE", - **DEFAULT_EXTRA_ARGS, - ) diff --git a/tests/ncbi_ftp/test_metadata.py b/tests/ncbi_ftp/test_metadata.py index cc13c192..bcdd825d 100644 --- a/tests/ncbi_ftp/test_metadata.py +++ b/tests/ncbi_ftp/test_metadata.py @@ -323,7 +323,7 @@ def mock_s3_with_descriptor(self) -> Generator[tuple[botocore.client.BaseClient, with ( patch.object(s3_utils, "get_s3_client", return_value=client), patch.object(metadata_mod, "get_s3_client", return_value=client), - patch.object(metadata_mod, "copy_object_with_metadata") as mock_copy, + patch.object(metadata_mod, "copy_object") as mock_copy, ): yield client, mock_copy reset_s3_client() @@ -339,7 +339,7 @@ def test_returns_true_when_descriptor_exists( def test_calls_copy_with_correct_keys( self, mock_s3_with_descriptor: tuple[botocore.client.BaseClient, MagicMock] ) -> None: - """copy_object_with_metadata is called with the live and archive keys.""" + """copy_object is called with the live and archive keys.""" _, mock_copy = mock_s3_with_descriptor archive_descriptor(_ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX, _RELEASE_TAG) live_key = build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX) @@ -352,7 +352,7 @@ def test_calls_copy_with_correct_keys( def test_dry_run_returns_true_without_copy( self, mock_s3_with_descriptor: tuple[botocore.client.BaseClient, MagicMock] ) -> None: - """Dry-run returns True but does not call copy_object_with_metadata.""" + """Dry-run returns True but does not call copy_object.""" _, mock_copy = mock_s3_with_descriptor result = archive_descriptor(_ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX, _RELEASE_TAG, dry_run=True) assert result is True diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index a96a5cab..8924f95a 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -19,7 +19,7 @@ CDM_LAKE_BUCKET, DEFAULT_EXTRA_ARGS, copy_object, - copy_object_with_metadata, + delete_object, download_file, get_s3_client, @@ -862,16 +862,16 @@ def test_head_object_with_protocols(mock_s3_client: Any, protocol: str) -> None: assert result["size"] == SIZE_DATA -# copy_object_with_metadata +# copy_object with metadata @pytest.mark.parametrize("destination", BUCKETS) @pytest.mark.s3 def test_copy_object_with_metadata_replaces_metadata(mocked_s3_client_no_checksum: Any, destination: str) -> None: - """Verify that copy_object_with_metadata copies and replaces metadata.""" + """Verify that copy_object with metadata copies and replaces metadata.""" mocked_s3_client_no_checksum.put_object( Bucket=CDM_LAKE_BUCKET, Key="src/file.txt", Body=b"archive me", Metadata={"old_key": "old_val"} ) new_metadata = {"archive_reason": "replaced", "archive_date": "2026-04-16"} - response = copy_object_with_metadata( + response = copy_object( f"{CDM_LAKE_BUCKET}/src/file.txt", f"{destination}/archive/file.txt", metadata=new_metadata, @@ -892,7 +892,7 @@ def test_copy_object_with_metadata_replaces_metadata(mocked_s3_client_no_checksu def test_copy_object_with_metadata_preserves_content(mocked_s3_client_no_checksum: Any) -> None: """Verify that the content of the copied object matches the original.""" mocked_s3_client_no_checksum.put_object(Bucket=CDM_LAKE_BUCKET, Key="src/data.bin", Body=b"binary data") - copy_object_with_metadata( + copy_object( f"{CDM_LAKE_BUCKET}/src/data.bin", f"{CDM_LAKE_BUCKET}/dst/data.bin", metadata={"tag": "value"}, From bf0c01e0930da2b48103720f4bf314303d34215b Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 23 Apr 2026 12:59:32 -0700 Subject: [PATCH 041/128] use tenacity Co-authored-by: Copilot --- src/cdm_data_loaders/utils/ftp_client.py | 63 ++++++++++++++---------- src/cdm_data_loaders/utils/s3.py | 3 -- tests/utils/test_s3.py | 1 - 3 files changed, 36 insertions(+), 31 deletions(-) diff --git a/src/cdm_data_loaders/utils/ftp_client.py b/src/cdm_data_loaders/utils/ftp_client.py index 0f8409e7..902779e5 100644 --- a/src/cdm_data_loaders/utils/ftp_client.py +++ b/src/cdm_data_loaders/utils/ftp_client.py @@ -6,12 +6,21 @@ """ import contextlib +import logging import socket import threading import time from ftplib import FTP, error_temp from pathlib import Path +from tenacity import ( + before_sleep_log, + retry, + retry_if_exception_type, + stop_after_attempt, + wait_fixed, +) + from cdm_data_loaders.utils.cdm_logger import get_cdm_logger logger = get_cdm_logger() @@ -73,19 +82,20 @@ def ftp_list_dir(ftp: FTP, path: str, retries: int = 3) -> list[str]: :return: list of filenames """ ftp.cwd(path) - for attempt in range(1, retries + 1): - try: - files: list[str] = [] - ftp.retrlines("NLST", files.append) - except error_temp as e: - if attempt < retries: - logger.warning("Transient FTP error listing %s (attempt %d/%d): %s", path, attempt, retries, e) - time.sleep(2) - else: - raise - else: - return files - return [] # unreachable, but keeps type checkers happy + + @retry( + retry=retry_if_exception_type(error_temp), + stop=stop_after_attempt(retries), + wait=wait_fixed(2), + reraise=True, + before_sleep=before_sleep_log(logger, logging.WARNING), + ) + def _list() -> list[str]: + files: list[str] = [] + ftp.retrlines("NLST", files.append) + return files + + return _list() def ftp_download_file(ftp: FTP, remote_path: str, local_path: str, retries: int = 3) -> None: @@ -96,20 +106,19 @@ def ftp_download_file(ftp: FTP, remote_path: str, local_path: str, retries: int :param local_path: local destination path :param retries: number of retry attempts """ - for attempt in range(1, retries + 1): - try: - with Path(local_path).open("wb") as f: - ftp.retrbinary(f"RETR {remote_path}", f.write) - except error_temp as e: - if attempt < retries: - logger.warning( - "Transient FTP error downloading %s (attempt %d/%d): %s", remote_path, attempt, retries, e - ) - time.sleep(2) - else: - raise - else: - return + + @retry( + retry=retry_if_exception_type(error_temp), + stop=stop_after_attempt(retries), + wait=wait_fixed(2), + reraise=True, + before_sleep=before_sleep_log(logger, logging.WARNING), + ) + def _download() -> None: + with Path(local_path).open("wb") as f: + ftp.retrbinary(f"RETR {remote_path}", f.write) + + _download() def ftp_retrieve_text(ftp: FTP, remote_path: str) -> str: diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index 13a850bd..9c3de749 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -475,6 +475,3 @@ def head_object(s3_path: str) -> dict[str, Any] | None: "metadata": resp.get("Metadata", {}), "checksum_crc64nvme": resp.get("ChecksumCRC64NVME"), } - - - diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index 8924f95a..f8bd9e53 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -19,7 +19,6 @@ CDM_LAKE_BUCKET, DEFAULT_EXTRA_ARGS, copy_object, - delete_object, download_file, get_s3_client, From de041dc902ffa1bab2d6124c10caaa7af2fd368b Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 23 Apr 2026 13:19:21 -0700 Subject: [PATCH 042/128] address reviewer comments Co-authored-by: Copilot --- src/cdm_data_loaders/utils/s3.py | 31 ++++++++++++++++++------------- tests/utils/test_s3.py | 8 ++++---- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index 9c3de749..2c33bee7 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -10,6 +10,8 @@ import tqdm from botocore.config import Config +from cdm_data_loaders.utils.cdm_logger import get_cdm_logger + CDM_LAKE_BUCKET = "cdm-lake" DEFAULT_EXTRA_ARGS = {"ChecksumAlgorithm": "CRC64NVME"} @@ -22,6 +24,9 @@ AWS_CLIENT_TOTAL_MAX_ATTEMPTS = 10 +logger = get_cdm_logger() + + _s3_client: botocore.client.BaseClient | None = None @@ -62,7 +67,7 @@ def get_s3_client(args: dict[str, str] | None = None) -> botocore.client.BaseCli "aws_secret_access_key": settings.MINIO_SECRET_KEY, } except (ModuleNotFoundError, ImportError, NameError) as e: - print(e) # noqa: T201 + logger.exception("Failed to load berdl settings: %s", e) raise required_args = ["endpoint_url", "aws_access_key_id", "aws_secret_access_key"] @@ -155,10 +160,10 @@ def object_exists(s3_path: str) -> bool: (bucket, key) = split_s3_path(s3_path) try: s3.head_object(Bucket=bucket, Key=key) - except botocore.exceptions.ClientError as e: + except Exception as e: # noqa: BLE001 error_string = str(e) if not error_string.startswith("An error occurred (404) when calling the HeadObject operation: Not Found"): - print(f"Error performing head operation on s3 object: {e!s}") # noqa: T201 + logger.error("Error performing head operation on s3 object: %s", e) return False return True @@ -201,7 +206,7 @@ def upload_file( if metadata is None: if object_exists(s3_path): - print(f"File already present: {s3_path}") # noqa: T201 + logger.info("File already present: %s", s3_path) return True s3 = get_s3_client() @@ -212,7 +217,7 @@ def upload_file( # Upload the file file_size = local_file_path.stat().st_size with tqdm.tqdm(total=file_size, unit="B", unit_scale=True, desc=str(local_file_path)) as pbar: - print(f"uploading {local_file_path!s} to {s3_path}") # noqa: T201 + logger.info("uploading %s to %s", local_file_path, s3_path) try: s3.upload_file( Filename=str(local_file_path), @@ -221,8 +226,8 @@ def upload_file( Callback=pbar.update, ExtraArgs=extra_args, ) - except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: - print(f"Error uploading to s3: {e!s}") # noqa: T201 + except Exception as e: # noqa: BLE001 + logger.error("Error uploading to s3: %s", e) return False return True @@ -277,7 +282,7 @@ def download_file(s3_path: str, local_file_path: str | Path, version_id: str | N try: parent_dir.mkdir(parents=True, exist_ok=False) except OSError as e: - print(f"Could not save s3 file to {local_file_path}: {e!s}") # noqa: T201 + logger.error("Could not save s3 file to %s: %s", local_file_path, e) raise s3 = get_s3_client() @@ -289,12 +294,12 @@ def download_file(s3_path: str, local_file_path: str | Path, version_id: str | N # Get the object size try: object_size = s3.head_object(**kwargs)["ContentLength"] - except botocore.exceptions.ClientError as e: + except Exception as e: # noqa: BLE001 error_string = str(e) if error_string.startswith("An error occurred (404) when calling the HeadObject operation: Not Found"): - print(f"File not found: {s3_path}") # noqa: T201 + logger.error("File not found: %s", s3_path) else: - print(f"Error downloading {s3_path}: {e!s}") # noqa: T201 + logger.error("Error downloading %s: %s", s3_path, e) raise extra_args = {"VersionId": version_id} if version_id is not None else None @@ -466,8 +471,8 @@ def head_object(s3_path: str) -> dict[str, Any] | None: (bucket, key) = split_s3_path(s3_path) try: resp = s3.head_object(Bucket=bucket, Key=key, ChecksumMode="ENABLED") - except botocore.exceptions.ClientError as e: - if e.response["Error"]["Code"] == "404": + except Exception as e: # noqa: BLE001 + if e.response["Error"]["Code"] == "404": # type: ignore[union-attr] return None raise return { diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index f8bd9e53..b459c467 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -392,13 +392,13 @@ def test_upload_file_uses_custom_object_name(mock_s3_client: Any, sample_file: P @pytest.mark.s3 def test_upload_file_skips_when_already_present( - mock_s3_client: Any, sample_file: Path, capsys: pytest.CaptureFixture + mock_s3_client: Any, sample_file: Path, caplog: pytest.LogCaptureFixture ) -> None: """Verify that uploading a file that already exists is skipped and returns True.""" mock_s3_client.put_object(Bucket=CDM_LAKE_BUCKET, Key=f"uploads/{sample_file.name}", Body=b"old") result = upload_file(sample_file, f"{CDM_LAKE_BUCKET}/uploads") assert result is True - assert "File already present" in capsys.readouterr().out + assert "File already present" in caplog.text @pytest.mark.usefixtures("mock_s3_client") @@ -622,7 +622,7 @@ def test_download_file_does_not_clobber_existing_file_to_mkdir(mock_s3_client: A @pytest.mark.s3 @pytest.mark.usefixtures("mock_s3_client") -def test_download_file_does_not_exist(tmp_path: Path, capsys: pytest.CaptureFixture) -> None: +def test_download_file_does_not_exist(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: """Ensure that attempting to download a file that does not exist raises an error.""" bucket = BUCKETS[0] key = "to/the/door.txt" @@ -634,7 +634,7 @@ def test_download_file_does_not_exist(tmp_path: Path, capsys: pytest.CaptureFixt ): download_file(f"{bucket}/{key}", tmp_path / "file.txt") - assert "File not found" in capsys.readouterr().out + assert "File not found" in caplog.text # TODO: Missing tests From 0fefd290a8b140409b9cf235d17e19b782518ca6 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 23 Apr 2026 13:35:41 -0700 Subject: [PATCH 043/128] remove defaults for minio test env vars Co-authored-by: Copilot --- tests/integration/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 95e60008..06fdb61e 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -28,9 +28,9 @@ # ── MinIO connection defaults ─────────────────────────────────────────── -MINIO_ENDPOINT_URL = os.environ.get("MINIO_ENDPOINT_URL", "http://localhost:9000") -MINIO_ACCESS_KEY = os.environ.get("MINIO_ACCESS_KEY", "minioadmin") -MINIO_SECRET_KEY = os.environ.get("MINIO_SECRET_KEY", "minioadmin") +MINIO_ENDPOINT_URL = os.environ["MINIO_ENDPOINT_URL"] +MINIO_ACCESS_KEY = os.environ["MINIO_ACCESS_KEY"] +MINIO_SECRET_KEY = os.environ["MINIO_SECRET_KEY"] # Maximum length of a bucket name per S3/DNS spec _MAX_BUCKET_LEN = 63 From f9bcaee01059dd69645fd93bc61557238b43ce90 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 23 Apr 2026 13:50:49 -0700 Subject: [PATCH 044/128] cleanup --- notebooks/ncbi_ftp_manifest.ipynb | 2 -- notebooks/ncbi_ftp_promote.ipynb | 14 ++++++-------- scripts/entrypoint.sh | 2 +- scripts/s3_local.py | 2 -- src/cdm_data_loaders/ncbi_ftp/metadata.py | 2 -- tests/integration/conftest.py | 2 -- 6 files changed, 7 insertions(+), 17 deletions(-) diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb index 4d51c5b3..b4e9af5e 100644 --- a/notebooks/ncbi_ftp_manifest.ipynb +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -48,8 +48,6 @@ "source": [ "\"\"\"Imports and S3 client initialisation.\"\"\"\n", "\n", - "from __future__ import annotations\n", - "\n", "import json\n", "from pathlib import Path\n", "\n", diff --git a/notebooks/ncbi_ftp_promote.ipynb b/notebooks/ncbi_ftp_promote.ipynb index 81239f39..a1aebcdc 100644 --- a/notebooks/ncbi_ftp_promote.ipynb +++ b/notebooks/ncbi_ftp_promote.ipynb @@ -34,8 +34,8 @@ "| `_S3_KEY` | S3 object key (no scheme/bucket) | `staging/transfer_manifest.txt` |\n", "| `_PATH` | local filesystem path | `output/removed_manifest.txt` |\n", "\n", - "Lakehouse object: `s3://{STORE_BUCKET}/{LAKEHOUSE_KEY_PREFIX}raw_data/\u2026/{filename}`\n", - "Staging object: `s3://{STORE_BUCKET}/{STAGING_KEY_PREFIX}raw_data/\u2026/{filename}`" + "Lakehouse object: `s3://{STORE_BUCKET}/{LAKEHOUSE_KEY_PREFIX}raw_data/…/{filename}`\n", + "Staging object: `s3://{STORE_BUCKET}/{STAGING_KEY_PREFIX}raw_data/…/{filename}`" ] }, { @@ -47,8 +47,6 @@ "source": [ "\"\"\"Imports and S3 client initialisation.\"\"\"\n", "\n", - "from __future__ import annotations\n", - "\n", "import json\n", "\n", "from cdm_data_loaders.ncbi_ftp.promote import (\n", @@ -100,7 +98,7 @@ "# format: S3 key prefix within STORE_BUCKET (no scheme, no bucket)\n", "LAKEHOUSE_KEY_PREFIX = DEFAULT_LAKEHOUSE_KEY_PREFIX\n", "\n", - "# Dry-run mode \u2014 log actions without making changes\n", + "# Dry-run mode — log actions without making changes\n", "DRY_RUN = False\n", "\n", "print(f\"Bucket: {STORE_BUCKET}\")\n", @@ -185,10 +183,10 @@ "print(f\"Timestamp: {report['timestamp']}\")\n", "\n", "if report[\"failed\"] > 0:\n", - " print(\"\\n\u26a0\ufe0f Some operations failed \u2014 check logs above for details.\")\n", + " print(\"\\n⚠️ Some operations failed — check logs above for details.\")\n", "\n", "if report[\"dry_run\"]:\n", - " print(\"\\n\ud83d\udccb This was a dry-run. Set DRY_RUN = False and re-run to apply changes.\")" + " print(\"\\n📋 This was a dry-run. Set DRY_RUN = False and re-run to apply changes.\")" ] }, { @@ -245,4 +243,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index 68d13b49..fd03e3cd 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -supported_commands="xml_split|uniref|uniprot|ncbi_rest_api|ncbi_ftp_sync|test|bash" +supported_commands="all_the_bacteria|xml_split|uniref|uniprot|ncbi_rest_api|ncbi_ftp_sync|test|bash" # Ensure at least one argument is provided if [ "$#" -eq 0 ]; then diff --git a/scripts/s3_local.py b/scripts/s3_local.py index 5802a133..60bac49f 100755 --- a/scripts/s3_local.py +++ b/scripts/s3_local.py @@ -16,8 +16,6 @@ MINIO_SECRET_KEY minioadmin """ -from __future__ import annotations - import json import os import sys diff --git a/src/cdm_data_loaders/ncbi_ftp/metadata.py b/src/cdm_data_loaders/ncbi_ftp/metadata.py index 4ef60363..42cfe865 100644 --- a/src/cdm_data_loaders/ncbi_ftp/metadata.py +++ b/src/cdm_data_loaders/ncbi_ftp/metadata.py @@ -16,8 +16,6 @@ size, file format, and MD5 hash of each promoted data file. """ -from __future__ import annotations - import json import tempfile from datetime import UTC, datetime diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 06fdb61e..aa3cd5e3 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -6,8 +6,6 @@ the final state of the object store via the MinIO console. """ -from __future__ import annotations - import hashlib import os import re From f88df1d04df3fe2c3e231d39e8a6c8b2ded31576 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 23 Apr 2026 15:14:50 -0700 Subject: [PATCH 045/128] add docker compose integration test option and draft new action Co-authored-by: Copilot --- .github/workflows/integration_tests.yaml | 45 ++++++++++++++++++++++++ README.md | 19 ++++++++++ docker-compose.yml | 43 ++++++++++++++++++++++ pyproject.toml | 6 ++-- scripts/entrypoint.sh | 6 +++- 5 files changed, 115 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/integration_tests.yaml create mode 100644 docker-compose.yml diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml new file mode 100644 index 00000000..a8029378 --- /dev/null +++ b/.github/workflows/integration_tests.yaml @@ -0,0 +1,45 @@ +name: Integration tests + +on: + workflow_call: + + push: + branches: + - main + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review + +jobs: + integration_tests: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build integration test image + uses: docker/build-push-action@v6 + with: + context: . + load: true + tags: cdm-data-loaders_integration-tests:latest + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Run integration tests + run: | + docker compose up \ + --no-build \ + --abort-on-container-exit \ + --exit-code-from integration-tests + + - name: Tear down + if: always() + run: docker compose down --volumes diff --git a/README.md b/README.md index 78aa4d74..95c9ac13 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Repo for CDM input data loading and wrangling - [Development](#development) - [Spark and other non-python dependencies](#spark-and-other-non-python-dependencies) - [Tests](#tests) + - [Integration tests (MinIO + NCBI FTP)](#integration-tests-minio--ncbi-ftp) - [Loading genomes, contigs, and features](#loading-genomes-contigs-and-features) - [Running bbmap stats and checkm2 on genome or contigset files](#running-bbmap-stats-and-checkm2-on-genome-or-contigset-files) - [Changelog](#changelog) @@ -140,6 +141,24 @@ End-to-end integration tests for the NCBI assembly pipeline live in `tests/integ - Docker (for MinIO) - Network access to `ftp.ncbi.nlm.nih.gov` +**Running with Docker Compose (easiest)** + +The [docker-compose.yml](docker-compose.yml) at the repo root defines both a MinIO service and the integration test runner. To build the image, start MinIO, and run the integration tests in one command: + +```sh +docker compose up --build --abort-on-container-exit +``` + +Compose will stream test output to the terminal and exit with the pytest exit code. To clean up afterwards: + +```sh +docker compose down --volumes +``` + +**Running manually** + +If you prefer to run the tests directly against a local MinIO instance (e.g. for faster iteration during development), follow the steps below. + **1. Start MinIO locally:** ```sh diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..cc1074eb --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,43 @@ +services: + minio: + image: quay.io/minio/minio:latest + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + ports: + - "9000:9000" + - "9001:9001" + healthcheck: + test: [ "CMD", "mc", "ready", "local" ] + interval: 5s + timeout: 5s + retries: 5 + + integration-tests: + build: + context: . + depends_on: + minio: + condition: service_healthy + environment: + MINIO_ENDPOINT_URL: http://minio:9000 + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + entrypoint: + - /bin/sh + - -c + - | + attempts=0 + until python3 -c " + import urllib.request, os + urllib.request.urlopen(os.environ['MINIO_ENDPOINT_URL'] + '/minio/health/live', timeout=1) + " 2>/dev/null; do + attempts=$$((attempts + 1)) + if [ "$$attempts" -ge 30 ]; then + echo 'Timed out waiting for MinIO.' && exit 1 + fi + echo 'Waiting for MinIO...' && sleep 1 + done + exec /app/scripts/entrypoint.sh integration-test + command: [] diff --git a/pyproject.toml b/pyproject.toml index 1826340c..84d74eb2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -197,9 +197,9 @@ markers = ["requires_spark: must be run in an environment where spark is availab USER = "fake_user" KBASE_AUTH_TOKEN = "test-token-123" CDM_TASK_SERVICE_URL = "http://localhost:8080" -MINIO_ENDPOINT_URL = "http://localhost:9000" -MINIO_ACCESS_KEY = "minioadmin" -MINIO_SECRET_KEY = "minioadmin" +MINIO_ENDPOINT_URL = { value = "http://localhost:9000", skip_if_set = true } +MINIO_ACCESS_KEY = { value = "minioadmin", skip_if_set = true } +MINIO_SECRET_KEY = { value = "minioadmin", skip_if_set = true } MINIO_SECURE_FLAG = "false" BERDL_POD_IP = "192.168.1.100" SPARK_MASTER_URL = "spark://localhost:7077" diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index fd03e3cd..2838f9b1 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -supported_commands="all_the_bacteria|xml_split|uniref|uniprot|ncbi_rest_api|ncbi_ftp_sync|test|bash" +supported_commands="all_the_bacteria|xml_split|uniref|uniprot|ncbi_rest_api|ncbi_ftp_sync|test|integration-test|bash" # Ensure at least one argument is provided if [ "$#" -eq 0 ]; then @@ -41,6 +41,10 @@ case "$cmd" in # run the tests exec /usr/bin/tini -- uv run --no-sync pytest -m "not requires_spark" ;; + integration-test) + # run the integration tests (requires a running MinIO instance) + exec /usr/bin/tini -- uv run --no-sync pytest -m "integration" -v "$@" + ;; bash) exec /usr/bin/tini -- /bin/bash ;; From c9aa9ffa3a141724b830ec3fab71c2bea8b25f87 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 23 Apr 2026 15:31:00 -0700 Subject: [PATCH 046/128] try again for the integration action Co-authored-by: Copilot --- .github/workflows/integration_tests.yaml | 2 +- docker-compose.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml index a8029378..fce2870a 100644 --- a/.github/workflows/integration_tests.yaml +++ b/.github/workflows/integration_tests.yaml @@ -29,7 +29,7 @@ jobs: with: context: . load: true - tags: cdm-data-loaders_integration-tests:latest + tags: cdm-data-loaders-integration-tests:latest cache-from: type=gha cache-to: type=gha,mode=max diff --git a/docker-compose.yml b/docker-compose.yml index cc1074eb..ba488cce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,7 @@ services: retries: 5 integration-tests: + image: cdm-data-loaders-integration-tests:latest build: context: . depends_on: From 36369cac79cf2bd19f44a03f73130e3b63854989 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 23 Apr 2026 15:31:31 -0700 Subject: [PATCH 047/128] Potential fix for pull request finding 'CodeQL / Workflow does not contain permissions' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .github/workflows/integration_tests.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml index fce2870a..77962029 100644 --- a/.github/workflows/integration_tests.yaml +++ b/.github/workflows/integration_tests.yaml @@ -13,6 +13,9 @@ on: - synchronize - ready_for_review +permissions: + contents: read + jobs: integration_tests: runs-on: ubuntu-latest From f2f8a78dce6b0f204c28e3c0c427c8a6a88c3ac8 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 23 Apr 2026 16:22:45 -0700 Subject: [PATCH 048/128] add labels to tests the send requests to the NCBI FTP server Co-authored-by: Copilot --- pyproject.toml | 2 +- tests/integration/test_download_e2e.py | 2 ++ tests/integration/test_full_pipeline.py | 2 ++ tests/integration/test_manifest_e2e.py | 3 +++ 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 84d74eb2..5e817110 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -190,7 +190,7 @@ log_cli = true log_cli_level = "INFO" log_level = "INFO" addopts = ["-v"] -markers = ["requires_spark: must be run in an environment where spark is available", "s3: tests that mock s3 interactions", "slow_test: does what it says on the tin", "integration: end-to-end tests requiring a running MinIO instance and network access"] +markers = ["requires_spark: must be run in an environment where spark is available", "s3: tests that mock s3 interactions", "slow_test: does what it says on the tin", "integration: end-to-end tests requiring a running MinIO instance and network access", "external_request: tests that make real network requests to external services (e.g. NCBI FTP)"] # environment settings for running tests [tool.pytest_env] diff --git a/tests/integration/test_download_e2e.py b/tests/integration/test_download_e2e.py index b527de96..452501d3 100644 --- a/tests/integration/test_download_e2e.py +++ b/tests/integration/test_download_e2e.py @@ -49,6 +49,7 @@ def _manifest_for_one_assembly(tmp_path: Path) -> tuple[Path, str]: @pytest.mark.integration @pytest.mark.slow_test +@pytest.mark.external_request class TestDownloadSmallBatch: """Download a single assembly from NCBI FTP and verify local output.""" @@ -91,6 +92,7 @@ def test_download_small_batch(self, tmp_path: Path) -> None: @pytest.mark.integration @pytest.mark.slow_test +@pytest.mark.external_request class TestDownloadResumeIncomplete: """Verify download handles re-runs when some files are already present.""" diff --git a/tests/integration/test_full_pipeline.py b/tests/integration/test_full_pipeline.py index 2d1faab7..2654215b 100644 --- a/tests/integration/test_full_pipeline.py +++ b/tests/integration/test_full_pipeline.py @@ -38,6 +38,7 @@ @pytest.mark.integration @pytest.mark.slow_test +@pytest.mark.external_request class TestFullPipelineSmallBatch: """Run the complete pipeline for a single assembly: diff → download → promote.""" @@ -103,6 +104,7 @@ def test_full_pipeline_small_batch( @pytest.mark.integration @pytest.mark.slow_test +@pytest.mark.external_request class TestFullPipelineIncrementalSync: """Run the pipeline twice to test incremental sync with archival.""" diff --git a/tests/integration/test_manifest_e2e.py b/tests/integration/test_manifest_e2e.py index a0357a51..fb46ad3a 100644 --- a/tests/integration/test_manifest_e2e.py +++ b/tests/integration/test_manifest_e2e.py @@ -55,6 +55,7 @@ def _download_and_filter() -> tuple[dict[str, AssemblyRecord], dict[str, Assembl @pytest.mark.integration @pytest.mark.slow_test +@pytest.mark.external_request class TestFreshSyncNoPrevious: """Phase 1 with no previous snapshot — everything is 'new'.""" @@ -92,6 +93,7 @@ def test_fresh_sync_no_previous(self, tmp_path: Path) -> None: @pytest.mark.integration @pytest.mark.slow_test +@pytest.mark.external_request class TestIncrementalDiffSyntheticPrevious: """Phase 1 incremental diff with a manufactured 'previous' snapshot.""" @@ -154,6 +156,7 @@ def test_incremental_diff(self, tmp_path: Path) -> None: @pytest.mark.integration @pytest.mark.slow_test +@pytest.mark.external_request class TestVerifyTransferCandidatesPrunes: """verify_transfer_candidates should prune assemblies already in the store.""" From caf43e06f0c75e8a481ac0918e135e43cefecec4 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Mon, 27 Apr 2026 06:45:13 -0700 Subject: [PATCH 049/128] Adding kernel creation instructions --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index 322ad2ed..0a3cbe1d 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ Repo for CDM input data loading and wrangling - [cdm-data-loaders](#cdm-data-loaders) - [Environment and python management](#environment-and-python-management) - [Installation](#installation) + - [Lakehouse and Use with Jupyter notebooks](#lakehouse-and-use-with-jupyter-notebooks) - [Running import pipelines](#running-import-pipelines) - [Development](#development) - [Spark and other non-python dependencies](#spark-and-other-non-python-dependencies) @@ -48,6 +49,28 @@ To activate a virtual environment with these dependencies installed, run If you are using IDEs like VSCode, they should pick up the creation of the new environment and offer it for executing python code. +### Lakehouse and Use with Jupyter notebooks + +`cdm-data-loaders` can be installed on platforms like the KBase Lakehouse using the same installation steps: + +```sh +cd cdm-data-loaders +uv sync +source .venv/bin/activate +``` + +To use the library in a Jupyter notebook, it must be registered as a Jupyter kernel. After performing the three steps above, +run the following commands: + +```sh +uv pip install -e . +uv pip install ipykernel +uv run python -m ipykernel install --user --name cdm-data-loaders --display-name "cdm-data-loaders" +``` + +The `cdm-data-loaders` kernel should now be available from the dropdown list of kernels in the Jupyter notebook interface. + + ## Running import pipelines The repo provides a Docker container that can be used to run several import pipelines or to run unit tests for the repo. The [entrypoint script](scripts/entrypoint.sh) parses the container `run` arguments and launches the appropriate functions. From 7a931d99ff6afca4c2a56d3940ab1b8b1c767e7d Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Mon, 27 Apr 2026 09:02:22 -0700 Subject: [PATCH 050/128] add in to(schema) to ensure schema comments are saved --- src/cdm_data_loaders/pipelines/all_the_bacteria.py | 2 ++ src/cdm_data_loaders/utils/spark_delta.py | 3 ++- tests/pipelines/conftest.py | 9 --------- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/cdm_data_loaders/pipelines/all_the_bacteria.py b/src/cdm_data_loaders/pipelines/all_the_bacteria.py index d874b8b8..8455ad5a 100644 --- a/src/cdm_data_loaders/pipelines/all_the_bacteria.py +++ b/src/cdm_data_loaders/pipelines/all_the_bacteria.py @@ -6,6 +6,8 @@ all_atb_files.tsv: https://osf.io/xv7q9/files/r6gcp (or Rg6cp, casing varies) +# TODO: change the output to delta tables. + """ import csv diff --git a/src/cdm_data_loaders/utils/spark_delta.py b/src/cdm_data_loaders/utils/spark_delta.py index e2422d77..113347d8 100644 --- a/src/cdm_data_loaders/utils/spark_delta.py +++ b/src/cdm_data_loaders/utils/spark_delta.py @@ -193,7 +193,8 @@ def write_delta(spark: SparkSession, sdf: DataFrame, delta_ns: str, table: str, ) """ merge_or_overwrite_schema = "mergeSchema" if mode == APPEND else "overwriteSchema" - writer = sdf.write.format("delta").mode(mode).option(merge_or_overwrite_schema, "true") + # use to(schema) to ensure that the schema is saved with the dataframe + writer = sdf.to(sdf.schema).write.format("delta").mode(mode).option(merge_or_overwrite_schema, "true") logger.info("Writing table %s in mode %s (rows=%d)", db_table, mode, sdf.count()) logger.debug(sdf.printSchema()) diff --git a/tests/pipelines/conftest.py b/tests/pipelines/conftest.py index 203038a1..4e891d1c 100644 --- a/tests/pipelines/conftest.py +++ b/tests/pipelines/conftest.py @@ -122,15 +122,6 @@ def _generate_dlt_config() -> dict[str, Any]: ) -@pytest.fixture(autouse=True) -def logging_setup(caplog: pytest.LogCaptureFixture) -> None: - """Ensure that the dlt logger propagates logs to the root logger, is set to INFO, and that any messages are cleared.""" - logger = logging.getLogger("dlt") - logger.propagate = True - caplog.set_level(logging.INFO) - caplog.clear() - - def make_batcher(files: list[Path], batch_size: int = 5) -> MagicMock: """Return a mock BatchCursor that yields ``files`` in batches then an empty list.""" batches = [list(b) for b in batched(files, batch_size, strict=False)] From 15ba0ef91ff14cc8aa2e496055b98c981aebe72a Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Mon, 27 Apr 2026 13:58:17 -0700 Subject: [PATCH 051/128] Add a copy_directory command to the s3 utils --- src/cdm_data_loaders/utils/s3.py | 89 +++++++++++++++--- tests/utils/test_s3.py | 151 +++++++++++++++++++++++++++++-- 2 files changed, 221 insertions(+), 19 deletions(-) diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index fd7e8849..5d803f00 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -10,6 +10,8 @@ import tqdm from botocore.config import Config +from cdm_data_loaders.utils.cdm_logger import get_cdm_logger + CDM_LAKE_BUCKET = "cdm-lake" DEFAULT_EXTRA_ARGS = {"ChecksumAlgorithm": "CRC64NVME"} @@ -21,9 +23,12 @@ # how many times to retry, including the initial attempt AWS_CLIENT_TOTAL_MAX_ATTEMPTS = 10 +SUCCESS_RESPONSE = 200 _s3_client: botocore.client.BaseClient | None = None +logger = get_cdm_logger() + def get_s3_client(args: dict[str, str] | None = None) -> botocore.client.BaseClient: """Create an S3 client using the provided arguments. @@ -61,8 +66,8 @@ def get_s3_client(args: dict[str, str] | None = None) -> botocore.client.BaseCli "aws_access_key_id": settings.MINIO_ACCESS_KEY, "aws_secret_access_key": settings.MINIO_SECRET_KEY, } - except (ModuleNotFoundError, ImportError, NameError) as e: - print(e) + except (ModuleNotFoundError, ImportError, NameError): + logger.exception("Error initialising boto3 client") raise except Exception: raise @@ -170,7 +175,7 @@ def object_exists(s3_path: str) -> bool: except Exception as e: error_string = str(e) if not error_string.startswith("An error occurred (404) when calling the HeadObject operation: Not Found"): - print(f"Error performing head operation on s3 object: {e!s}") + logger.exception("Error performing head operation on s3 object") return False return True @@ -203,7 +208,7 @@ def upload_file( s3_path = f"{destination_dir.removesuffix('/')}/{object_name}" if object_exists(s3_path): - print(f"File already present: {s3_path}") + logger.info("File already present: %s", s3_path) return True s3 = get_s3_client() @@ -212,7 +217,7 @@ def upload_file( # Upload the file file_size = local_file_path.stat().st_size with tqdm.tqdm(total=file_size, unit="B", unit_scale=True, desc=str(local_file_path)) as pbar: - print(f"uploading {local_file_path!s} to {s3_path}") + logger.info("uploading %s to %s", str(local_file_path), s3_path) try: s3.upload_file( Filename=str(local_file_path), @@ -221,8 +226,8 @@ def upload_file( Callback=pbar.update, ExtraArgs=DEFAULT_EXTRA_ARGS, ) - except Exception as e: - print(f"Error uploading to s3: {e!s}") + except Exception: + logger.exception("Error uploading to s3") return False return True @@ -276,8 +281,8 @@ def download_file(s3_path: str, local_file_path: str | Path, version_id: str | N if not parent_dir.is_dir(): try: parent_dir.mkdir(parents=True, exist_ok=False) - except Exception as e: - print(f"Could not save s3 file to {local_file_path}: {e!s}") + except Exception: + logger.exception("Could not save s3 file to %s", local_file_path) raise s3 = get_s3_client() @@ -292,9 +297,9 @@ def download_file(s3_path: str, local_file_path: str | Path, version_id: str | N except Exception as e: error_string = str(e) if error_string.startswith("An error occurred (404) when calling the HeadObject operation: Not Found"): - print(f"File not found: {s3_path}") + logger.exception("File not found: %s", s3_path) else: - print(f"Error downloading {s3_path}: {e!s}") + logger.exception("Error downloading %s", s3_path) raise extra_args = {"VersionId": version_id} if version_id is not None else None @@ -414,6 +419,68 @@ def copy_object(current_s3_path: str, new_s3_path: str) -> dict[str, Any]: ) +def copy_directory(current_s3_path: str, new_s3_path: str) -> tuple[dict[str, str], dict[str, Any]]: + """Copy all objects under a given S3 prefix to a new prefix. + + Preserves the relative key structure under the source prefix. For example, + copying s3://my-bucket/foo/ to s3://my-bucket/bar/ will copy + s3://my-bucket/foo/a/b.txt -> s3://my-bucket/bar/a/b.txt + + If the source bucket does not exist, a NoSuchBucket error will be thrown. + + :param current_s3_path: path to the directory on s3, INCLUDING the bucket name + :type current_s3_path: str + :param new_s3_path: the desired new directory path on s3, INCLUDING the bucket name + :type new_s3_path: str + :return: a tuple of (successes, errors) where: + - successes maps "bucket/source_key" -> "bucket/dest_key" for each + successfully copied object + - errors maps "bucket/source_key" -> the exception or response object + for each failed copy + :rtype: tuple[dict[str, str], dict[str, Any]] + """ + s3 = get_s3_client() + (current_s3_bucket, current_s3_prefix) = split_s3_path(current_s3_path) + (new_s3_bucket, new_s3_prefix) = split_s3_path(new_s3_path) + + if current_s3_prefix and not current_s3_prefix.endswith("/"): + current_s3_prefix += "/" + if new_s3_prefix and not new_s3_prefix.endswith("/"): + new_s3_prefix += "/" + + paginator = s3.get_paginator("list_objects_v2") + pages = paginator.paginate(Bucket=current_s3_bucket, Prefix=current_s3_prefix) + + successes: dict[str, str] = {} + errors: dict[str, Any] = {} + + for page in pages: + for obj in page.get("Contents", []): + current_key = obj["Key"] + relative_key = current_key[len(current_s3_prefix) :] + new_key = new_s3_prefix + relative_key + + source_path = f"{current_s3_bucket}/{current_key}" + dest_path = f"{new_s3_bucket}/{new_key}" + + try: + resp = s3.copy_object( + CopySource={"Bucket": current_s3_bucket, "Key": current_key}, + Bucket=new_s3_bucket, + Key=new_key, + **DEFAULT_EXTRA_ARGS, + ) + if resp["ResponseMetadata"]["HTTPStatusCode"] == SUCCESS_RESPONSE: + successes[source_path] = dest_path + else: + errors[source_path] = resp + except Exception as e: + logger.exception("Failed to copy %s to %s", source_path, dest_path) + errors[source_path] = e + + return successes, errors + + def delete_object(s3_path: str) -> dict[str, Any]: """Delete an object from s3. diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index 0effb71a..ff0a5062 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -2,6 +2,7 @@ import functools import io +import logging from collections.abc import Callable, Generator from pathlib import Path from typing import Any @@ -18,6 +19,7 @@ from cdm_data_loaders.utils.s3 import ( CDM_LAKE_BUCKET, DEFAULT_EXTRA_ARGS, + copy_directory, copy_object, delete_object, download_file, @@ -387,13 +389,15 @@ def test_upload_file_uses_custom_object_name(mock_s3_client: Any, sample_file: P @pytest.mark.s3 def test_upload_file_skips_when_already_present( - mock_s3_client: Any, sample_file: Path, capsys: pytest.CaptureFixture + mock_s3_client: Any, sample_file: Path, caplog: pytest.LogCaptureFixture ) -> None: """Verify that uploading a file that already exists is skipped and returns True.""" mock_s3_client.put_object(Bucket=CDM_LAKE_BUCKET, Key=f"uploads/{sample_file.name}", Body=b"old") result = upload_file(sample_file, f"{CDM_LAKE_BUCKET}/uploads") assert result is True - assert "File already present" in capsys.readouterr().out + last_log_message = caplog.records[-1] + assert "File already present" in last_log_message.message + assert last_log_message.levelno == logging.INFO @pytest.mark.usefixtures("mock_s3_client") @@ -617,7 +621,7 @@ def test_download_file_does_not_clobber_existing_file_to_mkdir(mock_s3_client: A @pytest.mark.s3 @pytest.mark.usefixtures("mock_s3_client") -def test_download_file_does_not_exist(tmp_path: Path, capsys: pytest.CaptureFixture) -> None: +def test_download_file_does_not_exist(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: """Ensure that attempting to download a file that does not exist raises an error.""" bucket = BUCKETS[0] key = "to/the/door.txt" @@ -629,7 +633,9 @@ def test_download_file_does_not_exist(tmp_path: Path, capsys: pytest.CaptureFixt ): download_file(f"{bucket}/{key}", tmp_path / "file.txt") - assert "File not found" in capsys.readouterr().out + last_log_message = caplog.records[-1] + assert "File not found" in last_log_message.message + assert last_log_message.levelno == logging.ERROR # TODO: Missing tests @@ -710,8 +716,8 @@ def mocked_s3_client_no_checksum(mock_s3_client: Any) -> Generator[Any, Any]: # copy_object @pytest.mark.parametrize("destination", BUCKETS) @pytest.mark.s3 -def test_copy_file(mocked_s3_client_no_checksum: Any, destination: str) -> None: - """Verify that copy_file copies an object to a new key within the same bucket.""" +def test_copy_object(mocked_s3_client_no_checksum: Any, destination: str) -> None: + """Verify that copy_object copies an object to a new key within the same bucket.""" mocked_s3_client_no_checksum.put_object(Bucket=CDM_LAKE_BUCKET, Key="src/file.txt", Body=b"copy me") assert object_exists(f"{CDM_LAKE_BUCKET}/src/file.txt") response = copy_object(f"{CDM_LAKE_BUCKET}/src/file.txt", f"{destination}/dst/path/to/file.txt") @@ -727,7 +733,7 @@ def test_copy_file(mocked_s3_client_no_checksum: Any, destination: str) -> None: @pytest.mark.s3 @pytest.mark.usefixtures("mock_s3_client") -def test_copy_file_source_object_nonexistent() -> None: +def test_copy_object_source_object_nonexistent() -> None: """Ensure that the code throws an error if the source object does not exist.""" s3_path = f"{CDM_LAKE_BUCKET}/some/path/to/file" assert object_exists(s3_path) is False @@ -737,7 +743,7 @@ def test_copy_file_source_object_nonexistent() -> None: @pytest.mark.s3 @pytest.mark.usefixtures("mock_s3_client") -def test_copy_file_source_bucket_nonexistent() -> None: +def test_copy_object_source_bucket_nonexistent() -> None: """Ensure that the code throws an error if the bucket does not exist.""" s3_path = "some-bucket/some/path/to/file" assert object_exists(s3_path) is False @@ -745,6 +751,135 @@ def test_copy_file_source_bucket_nonexistent() -> None: copy_object(s3_path, f"{CDM_LAKE_BUCKET}/a/different/path/to/file") +# copy_directory tests + +ALT_BUCKET = ALT_BUCKET + + +def put_objects(mock_s3_client: Any, bucket: str, keys: list[str], body: bytes = b"data") -> None: + """Helper to seed objects into a bucket.""" + for key in keys: + mock_s3_client.put_object(Bucket=bucket, Key=key, Body=body) + + +def list_keys(mock_s3_client: Any, bucket: str, prefix: str = "") -> set[str]: + """Helper to list all keys in a bucket under a prefix.""" + paginator = mock_s3_client.get_paginator("list_objects_v2") + keys = [] + for page in paginator.paginate(Bucket=bucket, Prefix=prefix): + keys.extend(obj["Key"] for obj in page.get("Contents", [])) + return set(keys) + + +@pytest.mark.s3 +@pytest.mark.parametrize("source_suffix", ["", "/"]) +@pytest.mark.parametrize("dest_suffix", ["", "/"]) +def test_copy_directory_copies_all_objects_to_dest( + mocked_s3_client_no_checksum: Any, source_suffix: str, dest_suffix: str +) -> None: + """Verify that all objects under the source prefix are present in the successes dict. + + Ensure that copy works correctly with or without a slash at the end of the directory name. + """ + mock_s3_client = mocked_s3_client_no_checksum + populate_mock_s3(mock_s3_client, {CDM_LAKE_BUCKET: FILES_IN_BUCKETS[CDM_LAKE_BUCKET]}) + source_bucket_files = list_keys(mock_s3_client, CDM_LAKE_BUCKET) + assert set(source_bucket_files) == set(SAMPLE_FILES) + dest_bucket_files = list_keys(mock_s3_client, ALT_BUCKET) + assert dest_bucket_files == set() + + successes, errors = copy_directory( + f"s3://{CDM_LAKE_BUCKET}/dir_one{source_suffix}", f"s3://{ALT_BUCKET}/some/destination/dir{dest_suffix}" + ) + + assert errors == {} + expected_files = { + f"{CDM_LAKE_BUCKET}/{f}": f"{ALT_BUCKET}/some/destination/dir{f.replace('dir_one', '')}" for f in SAMPLE_FILES + } + assert successes == expected_files + # ensure that the original files are still in place + assert set(list_keys(mock_s3_client, CDM_LAKE_BUCKET)) == set(SAMPLE_FILES) + # destination should have new files from the source + assert set(list_keys(mock_s3_client, ALT_BUCKET)) == { + f.removeprefix(f"{ALT_BUCKET}/") for f in expected_files.values() + } + + # check the content + for src, dest in expected_files.items(): + src_resp = mock_s3_client.get_object(Bucket=CDM_LAKE_BUCKET, Key=src.removeprefix(f"{CDM_LAKE_BUCKET}/")) + assert src_resp["Body"].read() == src.encode() + dest_resp = mock_s3_client.get_object(Bucket=ALT_BUCKET, Key=dest.removeprefix(f"{ALT_BUCKET}/")) + assert dest_resp["Body"].read() == src.encode() + + +@pytest.mark.s3 +def test_copy_directory_copy_within_same_bucket(mock_s3_client: Any) -> None: + """Verify that copying between two prefixes within the same bucket works correctly.""" + populate_mock_s3(mock_s3_client, {CDM_LAKE_BUCKET: ["foo/a.txt", "foo/b.txt"]}) + + successes, errors = copy_directory(f"s3://{CDM_LAKE_BUCKET}/foo", f"s3://{CDM_LAKE_BUCKET}/bar") + + assert successes == {"cdm-lake/foo/a.txt": "cdm-lake/bar/a.txt", "cdm-lake/foo/b.txt": "cdm-lake/bar/b.txt"} + assert errors == {} + assert list_keys(mock_s3_client, CDM_LAKE_BUCKET, prefix="bar") == {"bar/a.txt", "bar/b.txt"} + assert list_keys(mock_s3_client, CDM_LAKE_BUCKET) == {"foo/a.txt", "foo/b.txt", "bar/a.txt", "bar/b.txt"} + + +@pytest.mark.s3 +@pytest.mark.usefixtures("mock_s3_client") +def test_copy_directory_empty_directory_returns_empty_dicts() -> None: + """Verify that when the source prefix matches no objects, both the successes and errors dictionaries are returned empty.""" + successes, errors = copy_directory(f"s3://{CDM_LAKE_BUCKET}/nonexistent/", f"s3://{ALT_BUCKET}/bar/") + + assert successes == {} + assert errors == {} + + +@pytest.mark.s3 +@pytest.mark.usefixtures("mock_s3_client") +def test_copy_directory_does_not_copy_objects_outside_prefix(mock_s3_client: Any) -> None: + """Verify that objects whose keys share a prefix string but are not under the source directory. + + Example: 'foobar/' when copying 'foo/'. + """ + populate_mock_s3( + mock_s3_client, + { + CDM_LAKE_BUCKET: [ + "foo/a.txt", + "foobar/should-not-be-copied.txt", + ] + }, + ) + copy_directory(f"s3://{CDM_LAKE_BUCKET}/foo", f"s3://{ALT_BUCKET}/bar") + assert list_keys(mock_s3_client, ALT_BUCKET) == {"bar/a.txt"} + + +@pytest.mark.s3 +@pytest.mark.usefixtures("mock_s3_client") +def test_copy_directory_missing_source_bucket_returns_error() -> None: + """Verify that when the source bucket does not exist, botocore throws an error.""" + # FIXME: throws a s3.Client.exceptions.NoSuchBucket + with pytest.raises(Exception, match="The specified bucket does not exist"): + copy_directory("s3://nonexistent-bucket/bar", f"s3://{CDM_LAKE_BUCKET}/foo") + + +@pytest.mark.s3 +@pytest.mark.usefixtures("mock_s3_client") +def test_copy_directory_missing_dest_bucket_records_errors(mock_s3_client: Any) -> None: + """Verify that when the destination bucket does not exist, the errors dict contains all objects under the original dir.""" + # FIXME: throw a bucket not exists error? + populate_mock_s3(mock_s3_client, {CDM_LAKE_BUCKET: ["foo/a.txt", "foo/b.txt"]}) + + # with pytest.raises(FileNotFoundError, match="The specified bucket does not exist"): + successes, errors = copy_directory(f"s3://{CDM_LAKE_BUCKET}/foo", "s3://nonexistent-bucket/bar") + + assert successes == {} + assert f"{CDM_LAKE_BUCKET}/foo/a.txt" in errors + assert f"{CDM_LAKE_BUCKET}/foo/b.txt" in errors + assert isinstance(errors[f"{CDM_LAKE_BUCKET}/foo/a.txt"], Exception) + + # delete_object @pytest.mark.parametrize("bucket", BUCKETS) @pytest.mark.parametrize("protocol", ["", "s3://", "s3a://"]) From b1426a27d908642e1c06d591300a93ce2ea2cc8a Mon Sep 17 00:00:00 2001 From: i alarmed alien Date: Mon, 27 Apr 2026 14:15:03 -0700 Subject: [PATCH 052/128] Potential fix for pull request finding 'Redundant assignment' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- tests/utils/test_s3.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index ff0a5062..0ba7d567 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -753,8 +753,6 @@ def test_copy_object_source_bucket_nonexistent() -> None: # copy_directory tests -ALT_BUCKET = ALT_BUCKET - def put_objects(mock_s3_client: Any, bucket: str, keys: list[str], body: bytes = b"data") -> None: """Helper to seed objects into a bucket.""" From 6d923cea4ec9bd8889b017011cbb95e1d3ccc2b7 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 27 Apr 2026 15:45:23 -0700 Subject: [PATCH 053/128] address reviewer comments Co-authored-by: Copilot --- README.md | 29 + docs/ncbi_ftp_e2e_walkthrough.md | 3 + pyproject.toml | 4 +- scripts/entrypoint.sh | 10 +- .../pipelines/ncbi_ftp_download.py | 56 +- src/cdm_data_loaders/utils/s3.py | 68 +- tests/integration/conftest.py | 2 +- tests/integration/test_download_e2e.py | 6 +- tests/integration/test_promote_e2e.py | 6 +- tests/ncbi_ftp/conftest.py | 15 +- tests/ncbi_ftp/test_assembly.py | 165 +-- tests/ncbi_ftp/test_manifest.py | 1272 ++++++++--------- tests/ncbi_ftp/test_metadata.py | 456 +++--- tests/ncbi_ftp/test_notebooks.py | 72 +- tests/ncbi_ftp/test_promote.py | 402 +++--- tests/s3_helpers.py | 25 + tests/utils/test_s3.py | 22 +- 17 files changed, 1154 insertions(+), 1459 deletions(-) create mode 100644 tests/s3_helpers.py diff --git a/README.md b/README.md index 839ba7f3..edb8198e 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,35 @@ uv run python -m ipykernel install --user --name cdm-data-loaders --display-name The `cdm-data-loaders` kernel should now be available from the dropdown list of kernels in the Jupyter notebook interface. +#### Jupyter Kernel Environment Variables + +Often you will need access to environment variables that are included in the default Lakehouse +Jupyter environment, but will not be automatically included in your custom Jupyter kernel. To address +this, first identify the needed variables and values, and add them to your new kernel configuration +with the following steps: + +Open a new Jupyter Notebook __with the default kernel__ and run this in a new cell: +```python +import os +for k, v in sorted(os.environ.items()): + if "AWS" in k or "S3" in k or "MINIO" in k: # replace with whatever keys you're interested in + print(f"{k}={v}") +``` +Take the output and add the environment vars to the `kernel.json` for your new kernel (e.g., in `cdm-data-loaders/.venv/share/jupyter/kernels/python3/kernel.json`): +```json +{ + "argv": ["..."], + "display_name": "cdm-data-loaders", + "language": "python", + "env": { + "AWS_ACCESS_KEY_ID": "...", + "AWS_SECRET_ACCESS_KEY": "...", + "AWS_DEFAULT_REGION": "...", + ... + } +} +``` + ## Running import pipelines diff --git a/docs/ncbi_ftp_e2e_walkthrough.md b/docs/ncbi_ftp_e2e_walkthrough.md index d814b73b..4a710046 100644 --- a/docs/ncbi_ftp_e2e_walkthrough.md +++ b/docs/ncbi_ftp_e2e_walkthrough.md @@ -92,6 +92,9 @@ docker run -d \ minio/minio:RELEASE.2025-02-28T09-55-16Z server /data --console-address ":9001" ``` +(Note that a similar service is included in the `docker-compose` configuration file at the root of +this repository that is used in CI test workflows.) + Create a test bucket via the [MinIO console](http://localhost:9001) (login: `minioadmin` / `minioadmin`), or from the command line using the included `scripts/s3_local.py` helper (requires no extra installs — only diff --git a/pyproject.toml b/pyproject.toml index 5630b776..60785ba0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,10 +25,10 @@ dependencies = [ [project.scripts] all_the_bacteria = "cdm_data_loaders.pipelines.all_the_bacteria:cli" +ncbi_ftp_sync = "cdm_data_loaders.pipelines.ncbi_ftp_download:cli" +ncbi_rest_api = "cdm_data_loaders.pipelines.ncbi_rest_api:cli" uniprot = "cdm_data_loaders.pipelines.uniprot_kb:cli" uniref = "cdm_data_loaders.pipelines.uniref:cli" -ncbi_rest_api = "cdm_data_loaders.pipelines.ncbi_rest_api:cli" -ncbi_ftp_sync = "cdm_data_loaders.pipelines.ncbi_ftp_download:cli" [dependency-groups] dev = [ diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index 9a1a7af5..c1c0b6d3 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -VALID_COMMANDS=(all_the_bacteria ncbi_rest_api uniprot uniref xml_split ncbi_ftp_sync test integration-test bash) +VALID_COMMANDS=(all_the_bacteria ncbi_ftp_sync ncbi_rest_api uniprot uniref xml_split test integration-test bash) usage() { local joined @@ -21,6 +21,10 @@ case "$cmd" in all_the_bacteria) exec /usr/bin/tini -- uv run --no-sync all_the_bacteria "$@" ;; + ncbi_ftp_sync) + # Run the NCBI FTP assembly download pipeline (Phase 2) + exec /usr/bin/tini -- uv run --no-sync ncbi_ftp_sync "$@" + ;; ncbi_rest_api) exec /usr/bin/tini -- uv run --no-sync ncbi_rest_api "$@" ;; @@ -33,10 +37,6 @@ case "$cmd" in xml_split) exec /usr/bin/tini -- xml_file_splitter "$@" ;; - ncbi_ftp_sync) - # Run the NCBI FTP assembly download pipeline (Phase 2) - exec /usr/bin/tini -- uv run --no-sync ncbi_ftp_sync "$@" - ;; test) exec /usr/bin/tini -- uv run --no-sync pytest -m "not requires_spark" ;; diff --git a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py index 0f740ce9..20cd77e8 100644 --- a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py +++ b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py @@ -6,15 +6,16 @@ """ import json +import logging import threading -import time from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import UTC, datetime from ftplib import error_temp from pathlib import Path from typing import Any -from pydantic import AliasChoices, Field, field_validator +from pydantic import AliasChoices, Field +from tenacity import before_sleep_log, retry, retry_if_exception_type, stop_after_attempt, wait_fixed from pydantic_settings import BaseSettings, SettingsConfigDict from cdm_data_loaders.ncbi_ftp.assembly import FTP_HOST, download_assembly_to_local @@ -63,20 +64,6 @@ class DownloadSettings(BaseSettings): validation_alias=AliasChoices("l", "limit"), ) - @field_validator("threads") - @classmethod - def validate_threads(cls, v: int) -> int: - """Validate threads is within range. - - :param v: number of threads - :raises ValueError: if out of range - :return: validated thread count - """ - if v < 1 or v > 32: # noqa: PLR2004 - msg = f"threads must be between 1 and 32, got {v}" - raise ValueError(msg) - return v - # ── Batch download ─────────────────────────────────────────────────────── @@ -113,23 +100,26 @@ def download_batch( def _download_one(path: str) -> tuple[str, Exception | None]: nonlocal success_count - last_error: Exception | None = None - for attempt in range(1, 4): - try: - stats = download_assembly_to_local(path, output_dir, ftp_host=ftp_host, ftp=pool.get()) - except error_temp as e: - last_error = e - if attempt < 3: # noqa: PLR2004 - logger.warning("Transient FTP error for %s, retry %d/3: %s", path, attempt, e) - time.sleep(5) - except Exception as e: # noqa: BLE001 - return path, e - else: - with lock: - success_count += 1 - all_stats.append(stats) - return path, None - return path, last_error + + @retry( + retry=retry_if_exception_type(error_temp), + stop=stop_after_attempt(3), + wait=wait_fixed(5), + reraise=True, + before_sleep=before_sleep_log(logger, logging.WARNING), + ) + def _attempt() -> dict[str, Any]: + return download_assembly_to_local(path, output_dir, ftp_host=ftp_host, ftp=pool.get()) + + try: + stats = _attempt() + except Exception as e: # noqa: BLE001 + return path, e + else: + with lock: + success_count += 1 + all_stats.append(stats) + return path, None try: with ThreadPoolExecutor(max_workers=threads) as executor: diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index 2c33bee7..15fc5ecd 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -67,7 +67,7 @@ def get_s3_client(args: dict[str, str] | None = None) -> botocore.client.BaseCli "aws_secret_access_key": settings.MINIO_SECRET_KEY, } except (ModuleNotFoundError, ImportError, NameError) as e: - logger.exception("Failed to load berdl settings: %s", e) + logger.exception("Failed to load berdl settings") raise required_args = ["endpoint_url", "aws_access_key_id", "aws_secret_access_key"] @@ -147,6 +147,34 @@ def list_matching_objects(s3_path: str) -> list[dict[str, Any]]: return contents +def head_object(s3_path: str) -> dict[str, Any] | None: + """Return metadata for an S3 object, or None if it does not exist. + + The returned dict contains: + - ``size``: content length in bytes + - ``metadata``: user metadata dict + - ``checksum_crc64nvme``: CRC64NVME checksum string (if available) + + :param s3_path: path to the object on s3, INCLUDING the bucket name + :type s3_path: str + :return: dict with object info, or None if the object does not exist + :rtype: dict[str, Any] | None + """ + s3 = get_s3_client() + (bucket, key) = split_s3_path(s3_path) + try: + resp = s3.head_object(Bucket=bucket, Key=key, ChecksumMode="ENABLED") + except Exception as e: # noqa: BLE001 + if e.response["Error"]["Code"] == "404": # type: ignore[union-attr] + return None + raise + return { + "size": resp["ContentLength"], + "metadata": resp.get("Metadata", {}), + "checksum_crc64nvme": resp.get("ChecksumCRC64NVME"), + } + + def object_exists(s3_path: str) -> bool: """Check whether an object exists on s3. @@ -163,7 +191,7 @@ def object_exists(s3_path: str) -> bool: except Exception as e: # noqa: BLE001 error_string = str(e) if not error_string.startswith("An error occurred (404) when calling the HeadObject operation: Not Found"): - logger.error("Error performing head operation on s3 object: %s", e) + logger.exception("Error performing head operation on s3 object") return False return True @@ -227,7 +255,7 @@ def upload_file( ExtraArgs=extra_args, ) except Exception as e: # noqa: BLE001 - logger.error("Error uploading to s3: %s", e) + logger.exception("Error uploading to s3") return False return True @@ -282,7 +310,7 @@ def download_file(s3_path: str, local_file_path: str | Path, version_id: str | N try: parent_dir.mkdir(parents=True, exist_ok=False) except OSError as e: - logger.error("Could not save s3 file to %s: %s", local_file_path, e) + logger.exception("Could not save s3 file to %s", local_file_path) raise s3 = get_s3_client() @@ -297,9 +325,9 @@ def download_file(s3_path: str, local_file_path: str | Path, version_id: str | N except Exception as e: # noqa: BLE001 error_string = str(e) if error_string.startswith("An error occurred (404) when calling the HeadObject operation: Not Found"): - logger.error("File not found: %s", s3_path) + logger.exception("File not found: %s", s3_path) else: - logger.error("Error downloading %s: %s", s3_path, e) + logger.exception("Error downloading %s", s3_path) raise extra_args = {"VersionId": version_id} if version_id is not None else None @@ -452,31 +480,3 @@ def delete_object(s3_path: str) -> dict[str, Any]: s3 = get_s3_client() (bucket, key) = split_s3_path(s3_path) return s3.delete_object(Bucket=bucket, Key=key) - - -def head_object(s3_path: str) -> dict[str, Any] | None: - """Return metadata for an S3 object, or None if it does not exist. - - The returned dict contains: - - ``size``: content length in bytes - - ``metadata``: user metadata dict - - ``checksum_crc64nvme``: CRC64NVME checksum string (if available) - - :param s3_path: path to the object on s3, INCLUDING the bucket name - :type s3_path: str - :return: dict with object info, or None if the object does not exist - :rtype: dict[str, Any] | None - """ - s3 = get_s3_client() - (bucket, key) = split_s3_path(s3_path) - try: - resp = s3.head_object(Bucket=bucket, Key=key, ChecksumMode="ENABLED") - except Exception as e: # noqa: BLE001 - if e.response["Error"]["Code"] == "404": # type: ignore[union-attr] - return None - raise - return { - "size": resp["ContentLength"], - "metadata": resp.get("Metadata", {}), - "checksum_crc64nvme": resp.get("ChecksumCRC64NVME"), - } diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index aa3cd5e3..0fe7384a 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -75,7 +75,7 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item # ── Fixtures ──────────────────────────────────────────────────────────── -@pytest.fixture(scope="session") +@pytest.fixture def minio_s3_client() -> botocore.client.BaseClient: """Session-scoped real boto3 S3 client pointed at the local MinIO instance. diff --git a/tests/integration/test_download_e2e.py b/tests/integration/test_download_e2e.py index 452501d3..2126bd81 100644 --- a/tests/integration/test_download_e2e.py +++ b/tests/integration/test_download_e2e.py @@ -5,15 +5,11 @@ unreachable. """ -from __future__ import annotations - import json -from typing import TYPE_CHECKING import pytest -if TYPE_CHECKING: - from pathlib import Path +from pathlib import Path from cdm_data_loaders.ncbi_ftp.manifest import ( compute_diff, diff --git a/tests/integration/test_promote_e2e.py b/tests/integration/test_promote_e2e.py index b97aa542..46324603 100644 --- a/tests/integration/test_promote_e2e.py +++ b/tests/integration/test_promote_e2e.py @@ -8,11 +8,8 @@ unreachable. Each test method gets its own bucket. """ -from __future__ import annotations - import hashlib import json -from typing import TYPE_CHECKING import pytest @@ -26,8 +23,7 @@ from .conftest import get_object_metadata, list_all_keys, seed_lakehouse -if TYPE_CHECKING: - from pathlib import Path +from pathlib import Path # Fake assembly details used across tests ACCESSION_A = "GCF_900000001.1" diff --git a/tests/ncbi_ftp/conftest.py b/tests/ncbi_ftp/conftest.py index 2c0923dd..07fff0c6 100644 --- a/tests/ncbi_ftp/conftest.py +++ b/tests/ncbi_ftp/conftest.py @@ -1,7 +1,6 @@ """Shared fixtures for ncbi_ftp tests.""" -import functools -from collections.abc import Callable, Generator +from collections.abc import Generator from unittest.mock import patch import boto3 @@ -11,6 +10,7 @@ import cdm_data_loaders.ncbi_ftp.promote as promote_mod import cdm_data_loaders.utils.s3 as s3_utils +from tests.s3_helpers import strip_checksum_algorithm from cdm_data_loaders.utils.s3 import CDM_LAKE_BUCKET, reset_s3_client AWS_REGION = "us-east-1" @@ -45,17 +45,6 @@ ) -def strip_checksum_algorithm(method: Callable) -> Callable: - """Wrap a boto3 S3 method to remove ChecksumAlgorithm (moto CRC64NVME workaround).""" - - @functools.wraps(method) - def wrapper(*args: object, **kwargs: object) -> object: - kwargs.pop("ChecksumAlgorithm", None) # type: ignore[arg-type] - return method(*args, **kwargs) - - return wrapper - - @pytest.fixture def mock_s3_client() -> Generator[botocore.client.BaseClient]: """Yield a mocked S3 client with the CDM Lake bucket created.""" diff --git a/tests/ncbi_ftp/test_assembly.py b/tests/ncbi_ftp/test_assembly.py index 8aa24b1c..559788bf 100644 --- a/tests/ncbi_ftp/test_assembly.py +++ b/tests/ncbi_ftp/test_assembly.py @@ -9,106 +9,87 @@ parse_md5_checksums_file, ) -_EXPECTED_TWO_ENTRIES = 2 - # ── Path helpers ───────────────────────────────────────────────────────── -class TestBuildAccessionPath: - """Test output directory path construction from assembly names.""" - - def test_basic(self) -> None: - """Verify standard GCF accession path construction.""" - result = build_accession_path("GCF_000001215.4_Release_6_plus_ISO1_MT") - assert result == "raw_data/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/" - - def test_gca_prefix(self) -> None: - """Verify GCA prefix path construction.""" - result = build_accession_path("GCA_012345678.1_ASM1234v1") - assert result == "raw_data/GCA/012/345/678/GCA_012345678.1_ASM1234v1/" - - def test_invalid_raises(self) -> None: - """Verify ValueError on invalid assembly name.""" - with pytest.raises(ValueError, match="Cannot parse"): - build_accession_path("invalid_name") - - -class TestParseAssemblyPath: - """Test FTP path parsing.""" - - def test_basic(self) -> None: - """Verify db, assembly_dir, and accession are parsed correctly.""" - path = "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/" - _db, _assembly_dir, accession = parse_assembly_path(path) - assert _db == "GCF" - assert _assembly_dir == "GCF_000001215.4_Release_6_plus_ISO1_MT" - assert accession == "GCF_000001215.4" - - def test_without_trailing_slash(self) -> None: - """Verify parsing works without trailing slash.""" - path = "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT" - _db, _assembly_dir, accession = parse_assembly_path(path) - assert accession == "GCF_000001215.4" - - def test_invalid_raises(self) -> None: - """Verify ValueError on invalid path.""" - with pytest.raises(ValueError, match="Cannot parse"): - parse_assembly_path("/random/path/") - - -# ── FILE_FILTERS sanity ───────────────────────────────────────────────── - - -class TestFileFilters: - """Sanity checks for the file suffix filter list.""" - - def test_not_empty(self) -> None: - """Verify FILE_FILTERS is not empty.""" - assert len(FILE_FILTERS) > 0 - - def test_all_start_with_underscore(self) -> None: - """Verify all filter patterns start with an underscore.""" - for f in FILE_FILTERS: - assert f.startswith("_"), f"Filter should start with underscore: {f}" +@pytest.mark.parametrize( + ("assembly_dir", "expected"), + [ + pytest.param( + "GCF_000001215.4_Release_6_plus_ISO1_MT", + "raw_data/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/", + id="gcf", + ), + pytest.param( + "GCA_012345678.1_ASM1234v1", + "raw_data/GCA/012/345/678/GCA_012345678.1_ASM1234v1/", + id="gca", + ), + ], +) +def test_build_accession_path(assembly_dir: str, expected: str) -> None: + """Verify accession path construction for various inputs.""" + assert build_accession_path(assembly_dir) == expected + + +def test_build_accession_path_invalid() -> None: + """Verify ValueError on invalid assembly name.""" + with pytest.raises(ValueError, match="Cannot parse"): + build_accession_path("invalid_name") + + +@pytest.mark.parametrize( + ("path", "expected"), + [ + pytest.param( + "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/", + ("GCF", "GCF_000001215.4_Release_6_plus_ISO1_MT", "GCF_000001215.4"), + id="with_trailing_slash", + ), + pytest.param( + "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT", + ("GCF", "GCF_000001215.4_Release_6_plus_ISO1_MT", "GCF_000001215.4"), + id="without_trailing_slash", + ), + ], +) +def test_parse_assembly_path(path: str, expected: tuple[str, str, str]) -> None: + """Verify db, assembly_dir, and accession are parsed correctly.""" + assert parse_assembly_path(path) == expected - def test_genomic_fna_included(self) -> None: - """Verify _genomic.fna.gz is in the filter list.""" - assert "_genomic.fna.gz" in FILE_FILTERS - def test_assembly_report_included(self) -> None: - """Verify _assembly_report.txt is in the filter list.""" - assert "_assembly_report.txt" in FILE_FILTERS +def test_parse_assembly_path_invalid() -> None: + """Verify ValueError on invalid path.""" + with pytest.raises(ValueError, match="Cannot parse"): + parse_assembly_path("/random/path/") # ── parse_md5_checksums_file ───────────────────────────────────────────── -class TestParseMd5ChecksumsFile: - """Test NCBI md5checksums.txt parsing.""" - - def test_basic(self) -> None: - """Verify parsing of standard md5checksums.txt format.""" - text = "abc123 ./GCF_000001215.4_genomic.fna.gz\ndef456 ./GCF_000001215.4_genomic.gff.gz\n" - result = parse_md5_checksums_file(text) - assert result == { - "GCF_000001215.4_genomic.fna.gz": "abc123", - "GCF_000001215.4_genomic.gff.gz": "def456", - } - - def test_no_leading_dot_slash(self) -> None: - """Verify parsing works without leading ./ prefix.""" - text = "abc123 GCF_000001215.4_genomic.fna.gz\n" - result = parse_md5_checksums_file(text) - assert result == {"GCF_000001215.4_genomic.fna.gz": "abc123"} - - def test_empty(self) -> None: - """Verify empty or whitespace-only input returns empty dict.""" - assert parse_md5_checksums_file("") == {} - assert parse_md5_checksums_file(" \n \n") == {} - - def test_blank_lines_ignored(self) -> None: - """Verify blank lines between entries are skipped.""" - text = "abc123 file1.txt\n\n\ndef456 file2.txt\n" - result = parse_md5_checksums_file(text) - assert len(result) == _EXPECTED_TWO_ENTRIES +@pytest.mark.parametrize( + ("text", "expected"), + [ + pytest.param( + "abc123 ./GCF_000001215.4_genomic.fna.gz\ndef456 ./GCF_000001215.4_genomic.gff.gz\n", + {"GCF_000001215.4_genomic.fna.gz": "abc123", "GCF_000001215.4_genomic.gff.gz": "def456"}, + id="dot_slash_prefix", + ), + pytest.param( + "abc123 GCF_000001215.4_genomic.fna.gz\n", + {"GCF_000001215.4_genomic.fna.gz": "abc123"}, + id="no_dot_slash_prefix", + ), + pytest.param("", {}, id="empty_string"), + pytest.param(" \n \n", {}, id="whitespace_only"), + pytest.param( + "abc123 file1.txt\n\n\ndef456 file2.txt\n", + {"file1.txt": "abc123", "file2.txt": "def456"}, + id="blank_lines_ignored", + ), + ], +) +def test_parse_md5_checksums_file(text: str, expected: dict[str, str]) -> None: + """Verify parse_md5_checksums_file handles various input formats.""" + assert parse_md5_checksums_file(text) == expected diff --git a/tests/ncbi_ftp/test_manifest.py b/tests/ncbi_ftp/test_manifest.py index 9a3add97..5611e925 100644 --- a/tests/ncbi_ftp/test_manifest.py +++ b/tests/ncbi_ftp/test_manifest.py @@ -7,6 +7,7 @@ import pytest from cdm_data_loaders.ncbi_ftp.manifest import ( + AssemblyRecord, DiffResult, _extract_accession_from_s3_key, _extract_assembly_dir_from_s3_key, @@ -26,294 +27,245 @@ from .conftest import SAMPLE_SUMMARY -_EXPECTED_ENTRIES = 4 _EXPECTED_TWO = 2 -_EXPECTED_TOTAL_TRANSFER = 2 # ── parse_assembly_summary ─────────────────────────────────────────────── -class TestParseAssemblySummary: - """Test assembly summary parsing.""" - - def test_parse_basic(self) -> None: - """Verify basic parsing returns expected number of assemblies.""" - assemblies = parse_assembly_summary(SAMPLE_SUMMARY) - assert len(assemblies) == _EXPECTED_ENTRIES - assert "GCF_000001215.4" in assemblies - assert "GCF_000005845.2" in assemblies - assert "GCF_000099999.1" not in assemblies # ftp_path == "na" - - def test_parse_status(self) -> None: - """Verify status field is parsed correctly.""" - assemblies = parse_assembly_summary(SAMPLE_SUMMARY) - assert assemblies["GCF_000001215.4"].status == "latest" - assert assemblies["GCF_000005845.2"].status == "replaced" - assert assemblies["GCF_000009999.1"].status == "suppressed" - - def test_parse_seq_rel_date(self) -> None: - """Verify seq_rel_date field is parsed correctly.""" - assemblies = parse_assembly_summary(SAMPLE_SUMMARY) - assert assemblies["GCF_000001215.4"].seq_rel_date == "2014/10/21" - - def test_parse_assembly_dir(self) -> None: - """Verify assembly_dir is extracted from the FTP path.""" - assemblies = parse_assembly_summary(SAMPLE_SUMMARY) - assert assemblies["GCF_000001215.4"].assembly_dir == "GCF_000001215.4_Release_6_plus_ISO1_MT" - - def test_parse_ftp_path(self) -> None: - """Verify full FTP path is stored.""" - assemblies = parse_assembly_summary(SAMPLE_SUMMARY) - assert assemblies["GCF_000001215.4"].ftp_path == ( - "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT" - ) - - def test_parse_empty(self) -> None: - """Verify empty or comment-only input returns empty dict.""" - assemblies = parse_assembly_summary("# comment only\n") - assert len(assemblies) == 0 - - def test_parse_skips_comments(self) -> None: - """Verify comment lines are not included in results.""" - assemblies = parse_assembly_summary(SAMPLE_SUMMARY) - for acc in assemblies: - assert acc.startswith("GCF_") - - def test_parse_from_file(self, tmp_path: Path) -> None: - """Verify parsing from a file path object.""" +_EXPECTED_ASSEMBLIES = { + "GCF_000001215.4": AssemblyRecord( + accession="GCF_000001215.4", + status="latest", + seq_rel_date="2014/10/21", + ftp_path="https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT", + assembly_dir="GCF_000001215.4_Release_6_plus_ISO1_MT", + ), + "GCF_000001405.40": AssemblyRecord( + accession="GCF_000001405.40", + status="latest", + seq_rel_date="2022/02/03", + ftp_path="https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/405/GCF_000001405.40_GRCh38.p14", + assembly_dir="GCF_000001405.40_GRCh38.p14", + ), + "GCF_000005845.2": AssemblyRecord( + accession="GCF_000005845.2", + status="replaced", + seq_rel_date="2013/09/26", + ftp_path="https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/005/845/GCF_000005845.2_ASM584v2", + assembly_dir="GCF_000005845.2_ASM584v2", + ), + "GCF_000009999.1": AssemblyRecord( + accession="GCF_000009999.1", + status="suppressed", + seq_rel_date="2010/01/01", + ftp_path="https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/009/999/GCF_000009999.1_ASM999v1", + assembly_dir="GCF_000009999.1_ASM999v1", + ), + # GCF_000099999.1 is excluded because ftp_path == "na" +} + + +def test_parse_assembly_summary() -> None: + """SAMPLE_SUMMARY is parsed to the expected assemblies.""" + assert parse_assembly_summary(SAMPLE_SUMMARY) == _EXPECTED_ASSEMBLIES + + +def test_parse_assembly_summary_empty() -> None: + """Comment-only input returns empty dict.""" + assert parse_assembly_summary("# comment only\n") == {} + + +@pytest.mark.parametrize("source", ["file", "file_str", "list_of_lines"]) +def test_parse_assembly_summary_input_types(source: str, tmp_path: Path) -> None: + """Parsing works from a file path, string path, and list of lines.""" + if source == "list_of_lines": + arg = SAMPLE_SUMMARY.splitlines(keepends=True) + else: f = tmp_path / "summary.tsv" f.write_text(SAMPLE_SUMMARY) - assemblies = parse_assembly_summary(f) - assert len(assemblies) == _EXPECTED_ENTRIES - - def test_parse_from_file_str(self, tmp_path: Path) -> None: - """Verify parsing from a string file path.""" - f = tmp_path / "summary.tsv" - f.write_text(SAMPLE_SUMMARY) - assemblies = parse_assembly_summary(str(f)) - assert len(assemblies) == _EXPECTED_ENTRIES - - def test_parse_from_list_of_lines(self) -> None: - """Verify parsing from a list of lines.""" - lines = SAMPLE_SUMMARY.splitlines(keepends=True) - assemblies = parse_assembly_summary(lines) - assert len(assemblies) == _EXPECTED_ENTRIES + arg = f if source == "file" else str(f) + assert parse_assembly_summary(arg) == _EXPECTED_ASSEMBLIES # ── get_latest_assembly_paths ──────────────────────────────────────────── -class TestGetLatestAssemblyPaths: - """Test extraction of FTP paths for latest assemblies.""" +def test_get_latest_assembly_paths() -> None: + """Only 'latest' assemblies appear; paths are FTP directories with trailing slash.""" + assert dict(get_latest_assembly_paths(parse_assembly_summary(SAMPLE_SUMMARY))) == { + "GCF_000001215.4": "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/", + "GCF_000001405.40": "/genomes/all/GCF/000/001/405/GCF_000001405.40_GRCh38.p14/", + } - def test_only_latest(self) -> None: - """Verify only assemblies with status 'latest' are returned.""" - assemblies = parse_assembly_summary(SAMPLE_SUMMARY) - paths = get_latest_assembly_paths(assemblies) - accessions = [acc for acc, _ in paths] - assert "GCF_000001215.4" in accessions - assert "GCF_000001405.40" in accessions - assert "GCF_000005845.2" not in accessions # replaced - assert "GCF_000009999.1" not in accessions # suppressed - def test_path_conversion(self) -> None: - """Verify HTTPS paths are converted to FTP-relative paths.""" - assemblies = parse_assembly_summary(SAMPLE_SUMMARY) - paths = dict(get_latest_assembly_paths(assemblies)) - assert paths["GCF_000001215.4"] == "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/" - - def test_paths_end_with_slash(self) -> None: - """Verify all returned paths end with a trailing slash.""" - assemblies = parse_assembly_summary(SAMPLE_SUMMARY) - for _, path in get_latest_assembly_paths(assemblies): - assert path.endswith("/") - - def test_empty(self) -> None: - """Verify empty input returns empty list.""" - assemblies = parse_assembly_summary("# empty\n") - assert get_latest_assembly_paths(assemblies) == [] +def test_get_latest_assembly_paths_empty() -> None: + """Empty input returns empty list.""" + assert get_latest_assembly_paths(parse_assembly_summary("# empty\n")) == [] # ── compute_diff ───────────────────────────────────────────────────────── -class TestComputeDiff: - """Test diff computation between current and previous assembly state.""" - - def test_all_new_no_previous(self) -> None: - """Verify all latest assemblies are marked new when no previous state.""" - current = parse_assembly_summary(SAMPLE_SUMMARY) - diff = compute_diff(current, previous_accessions=set()) - assert "GCF_000001215.4" in diff.new - assert "GCF_000001405.40" in diff.new - assert "GCF_000005845.2" not in diff.new # replaced - - def test_nothing_new_when_all_known(self) -> None: - """Verify no new assemblies when all are already known.""" - current = parse_assembly_summary(SAMPLE_SUMMARY) - known = {"GCF_000001215.4", "GCF_000001405.40"} - diff = compute_diff(current, previous_accessions=known) - assert len(diff.new) == 0 - - def test_detects_updated_seq_rel_date_newer(self) -> None: - """Assemblies whose seq_rel_date moved forward are marked updated.""" - current = parse_assembly_summary(SAMPLE_SUMMARY) - previous = parse_assembly_summary(SAMPLE_SUMMARY) - previous["GCF_000001215.4"].seq_rel_date = "2010/01/01" - diff = compute_diff(current, previous_assemblies=previous) - assert "GCF_000001215.4" in diff.updated - - def test_does_not_flag_updated_when_seq_rel_date_older(self) -> None: - """Assemblies whose seq_rel_date in current is older (e.g. synthetic baseline) are not flagged.""" - current = parse_assembly_summary(SAMPLE_SUMMARY) - previous = parse_assembly_summary(SAMPLE_SUMMARY) - previous["GCF_000001215.4"].seq_rel_date = "2099/12/31" - diff = compute_diff(current, previous_assemblies=previous) - assert "GCF_000001215.4" not in diff.updated - - def test_detects_replaced(self) -> None: - """Verify assemblies with status 'replaced' are detected.""" - current = parse_assembly_summary(SAMPLE_SUMMARY) - diff = compute_diff(current, previous_accessions={"GCF_000005845.2"}) - assert "GCF_000005845.2" in diff.replaced - - def test_detects_suppressed(self) -> None: - """Verify assemblies with status 'suppressed' are detected.""" - current = parse_assembly_summary(SAMPLE_SUMMARY) - diff = compute_diff(current, previous_accessions={"GCF_000009999.1"}) - assert "GCF_000009999.1" in diff.suppressed - - def test_detects_withdrawn(self) -> None: - """Accessions in previous but entirely absent from current.""" - current = parse_assembly_summary("# empty\n") - diff = compute_diff(current, previous_accessions={"GCF_000001215.4"}) - assert "GCF_000001215.4" in diff.suppressed - - def test_scan_store_fallback(self) -> None: - """Verify known accessions are not marked as new.""" - current = parse_assembly_summary(SAMPLE_SUMMARY) - diff = compute_diff(current, previous_accessions={"GCF_000001215.4"}) - assert "GCF_000001215.4" not in diff.new - assert "GCF_000001405.40" in diff.new - - def test_results_are_sorted(self) -> None: - """Verify diff results are sorted alphabetically.""" - current = parse_assembly_summary(SAMPLE_SUMMARY) - diff = compute_diff(current, previous_accessions=set()) - assert diff.new == sorted(diff.new) +def test_compute_diff_new() -> None: + """All latest assemblies are new with no previous state; result is sorted.""" + current = parse_assembly_summary(SAMPLE_SUMMARY) + diff = compute_diff(current, previous_accessions=set()) + assert "GCF_000001215.4" in diff.new + assert "GCF_000001405.40" in diff.new + assert "GCF_000005845.2" not in diff.new # replaced + assert diff.new == sorted(diff.new) -# ── accession_prefix & filter_by_prefix_range ──────────────────────────── +def test_compute_diff_updated() -> None: + """seq_rel_date moving forward marks updated; moving backward does not.""" + current = parse_assembly_summary(SAMPLE_SUMMARY) + previous = parse_assembly_summary(SAMPLE_SUMMARY) + previous["GCF_000001215.4"].seq_rel_date = "2010/01/01" + assert "GCF_000001215.4" in compute_diff(current, previous_assemblies=previous).updated + previous2 = parse_assembly_summary(SAMPLE_SUMMARY) + previous2["GCF_000001215.4"].seq_rel_date = "2099/12/31" + assert "GCF_000001215.4" not in compute_diff(current, previous_assemblies=previous2).updated -class TestPrefixFiltering: - """Test prefix extraction and range filtering.""" - def test_accession_prefix(self) -> None: - """Verify 3-digit prefix extraction from accessions.""" - assert accession_prefix("GCF_000001215.4") == "000" - assert accession_prefix("GCF_123456789.1") == "123" - assert accession_prefix("invalid") is None +def test_compute_diff_removed() -> None: + """Replaced, suppressed, and entirely-absent accessions are classified correctly.""" + current = parse_assembly_summary(SAMPLE_SUMMARY) + assert "GCF_000005845.2" in compute_diff(current, previous_accessions={"GCF_000005845.2"}).replaced + assert "GCF_000009999.1" in compute_diff(current, previous_accessions={"GCF_000009999.1"}).suppressed + # Accession absent from current entirely → suppressed + assert ( + "GCF_000001215.4" + in compute_diff(parse_assembly_summary("# empty\n"), previous_accessions={"GCF_000001215.4"}).suppressed + ) - def test_filter_range_inclusive(self) -> None: - """Verify prefix range filter is inclusive.""" - assemblies = parse_assembly_summary(SAMPLE_SUMMARY) - filtered = filter_by_prefix_range(assemblies, "000", "000") - assert len(filtered) == len(assemblies) - def test_filter_excludes_out_of_range(self) -> None: - """Verify assemblies outside the prefix range are excluded.""" - assemblies = parse_assembly_summary(SAMPLE_SUMMARY) - filtered = filter_by_prefix_range(assemblies, "001", "999") - assert len(filtered) == 0 +def test_compute_diff_scan_store_fallback() -> None: + """Known accessions are not marked new; unknown ones are.""" + current = parse_assembly_summary(SAMPLE_SUMMARY) + diff = compute_diff(current, previous_accessions={"GCF_000001215.4"}) + assert "GCF_000001215.4" not in diff.new + assert "GCF_000001405.40" in diff.new - def test_no_filter_returns_all(self) -> None: - """Verify no prefix range returns all assemblies.""" - assemblies = parse_assembly_summary(SAMPLE_SUMMARY) - filtered = filter_by_prefix_range(assemblies) - assert len(filtered) == len(assemblies) +# ── accession_prefix & filter_by_prefix_range ──────────────────────────── -# ── Manifest writing ──────────────────────────────────────────────────── +@pytest.mark.parametrize( + ("accession", "expected"), + [ + pytest.param("GCF_000001215.4", "000", id="three_zeros"), + pytest.param("GCF_123456789.1", "123", id="non_zero"), + pytest.param("invalid", None, id="invalid"), + ], +) +def test_accession_prefix(accession: str, expected: str | None) -> None: + """3-digit prefix is extracted from the accession; invalid input returns None.""" + assert accession_prefix(accession) == expected -class TestManifestWriting: - """Test manifest file writing.""" - - def test_write_transfer_manifest(self, tmp_path: Path) -> None: - """Verify transfer manifest file is written correctly.""" - current = parse_assembly_summary(SAMPLE_SUMMARY) - diff = compute_diff(current, previous_accessions=set()) - manifest_file = tmp_path / "transfer.txt" - paths = write_transfer_manifest(diff, current, manifest_file) - assert len(paths) > 0 - lines = [line.strip() for line in manifest_file.read_text().splitlines() if line.strip()] - assert len(lines) == len(paths) - for line in lines: - assert line.startswith("/genomes/") - assert line.endswith("/") - - def test_write_removed_manifest(self, tmp_path: Path) -> None: - """Verify removed manifest lists replaced and suppressed accessions.""" - current = parse_assembly_summary(SAMPLE_SUMMARY) - diff = compute_diff(current, previous_accessions={"GCF_000005845.2", "GCF_000009999.1"}) - removed_file = tmp_path / "removed.txt" - removed = write_removed_manifest(diff, removed_file) - assert len(removed) == _EXPECTED_TWO - lines = [line.strip() for line in removed_file.read_text().splitlines() if line.strip()] - assert len(lines) == _EXPECTED_TWO - - def test_write_updated_manifest(self, tmp_path: Path) -> None: - """Verify updated manifest lists only updated accessions.""" - diff = DiffResult(new=["GCF_000001215.4"], updated=["GCF_000005845.2", "GCF_000001405.40"]) - updated_file = tmp_path / "updated.txt" - updated = write_updated_manifest(diff, updated_file) - assert len(updated) == _EXPECTED_TWO - lines = [line.strip() for line in updated_file.read_text().splitlines() if line.strip()] - assert len(lines) == _EXPECTED_TWO - # Should be sorted - assert lines[0] == "GCF_000001405.40" - assert lines[1] == "GCF_000005845.2" - - def test_write_diff_summary(self, tmp_path: Path) -> None: - """Verify diff summary JSON is written with correct counts.""" - diff = DiffResult(new=["a"], updated=["b"], replaced=["c"], suppressed=[]) - summary_file = tmp_path / "summary.json" - summary = write_diff_summary(diff, summary_file, "refseq", "000", "003") - assert summary["counts"]["new"] == 1 - assert summary["counts"]["total_to_transfer"] == _EXPECTED_TOTAL_TRANSFER - assert summary["prefix_range"]["from"] == "000" - - loaded = json.loads(summary_file.read_text()) - assert loaded["database"] == "refseq" +def test_filter_by_prefix_range() -> None: + """Range filter is inclusive; out-of-range excluded; no range returns all.""" + assemblies = parse_assembly_summary(SAMPLE_SUMMARY) + assert len(filter_by_prefix_range(assemblies, "000", "000")) == len(assemblies) + assert len(filter_by_prefix_range(assemblies, "001", "999")) == 0 + assert len(filter_by_prefix_range(assemblies)) == len(assemblies) -# ── _ftp_dir_from_url ─────────────────────────────────────────────────── +# ── Manifest writing ──────────────────────────────────────────────────── -class TestFtpDirFromUrl: - """Test FTP URL to directory path conversion.""" - def test_https_url(self) -> None: - """Verify https:// URLs are converted to FTP paths.""" - url = "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6" - assert _ftp_dir_from_url(url) == "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6" +def test_write_transfer_manifest(tmp_path: Path) -> None: + """Transfer manifest is written with FTP paths that start with /genomes/ and end with /.""" + current = parse_assembly_summary(SAMPLE_SUMMARY) + diff = compute_diff(current, previous_accessions=set()) + manifest_file = tmp_path / "transfer.txt" + paths = write_transfer_manifest(diff, current, manifest_file) + assert len(paths) > 0 + lines = [line.strip() for line in manifest_file.read_text().splitlines() if line.strip()] + assert len(lines) == len(paths) + for line in lines: + assert line.startswith("/genomes/") + assert line.endswith("/") + + +def test_write_removed_manifest(tmp_path: Path) -> None: + """Removed manifest lists replaced and suppressed accessions.""" + current = parse_assembly_summary(SAMPLE_SUMMARY) + diff = compute_diff(current, previous_accessions={"GCF_000005845.2", "GCF_000009999.1"}) + removed_file = tmp_path / "removed.txt" + removed = write_removed_manifest(diff, removed_file) + assert len(removed) == _EXPECTED_TWO + lines = [line.strip() for line in removed_file.read_text().splitlines() if line.strip()] + assert len(lines) == _EXPECTED_TWO + + +def test_write_updated_manifest(tmp_path: Path) -> None: + """Updated manifest lists only updated accessions, sorted.""" + diff = DiffResult(new=["GCF_000001215.4"], updated=["GCF_000005845.2", "GCF_000001405.40"]) + updated_file = tmp_path / "updated.txt" + updated = write_updated_manifest(diff, updated_file) + assert len(updated) == _EXPECTED_TWO + lines = [line.strip() for line in updated_file.read_text().splitlines() if line.strip()] + assert lines == ["GCF_000001405.40", "GCF_000005845.2"] + + +def test_write_diff_summary(tmp_path: Path) -> None: + """Diff summary JSON is written with correct counts, prefix range, and database.""" + diff = DiffResult(new=["a"], updated=["b"], replaced=["c"], suppressed=[]) + summary_file = tmp_path / "summary.json" + summary = write_diff_summary(diff, summary_file, "refseq", "000", "003") + assert json.loads(summary_file.read_text()) == summary + assert {k: summary[k] for k in ("database", "counts", "prefix_range", "accessions")} == { + "database": "refseq", + "counts": { + "new": 1, + "updated": 1, + "replaced": 1, + "suppressed": 0, + "total_to_transfer": 2, + "total_to_remove": 1, + }, + "prefix_range": {"from": "000", "to": "003"}, + "accessions": {"new": ["a"], "updated": ["b"], "replaced": ["c"], "suppressed": []}, + } - def test_ftp_url(self) -> None: - """Verify ftp:// URLs are converted to FTP paths.""" - url = "ftp://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6" - assert _ftp_dir_from_url(url) == "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6" - def test_bare_path(self) -> None: - """Verify bare paths are returned unchanged.""" - path = "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6" - assert _ftp_dir_from_url(path) == path +# ── _ftp_dir_from_url ─────────────────────────────────────────────────── + - def test_custom_ftp_host(self) -> None: - """Verify custom FTP host is stripped from ftp:// URLs.""" - url = "ftp://custom.host.example.com/genomes/all/GCF/000/001/215" - assert _ftp_dir_from_url(url, ftp_host="custom.host.example.com") == "/genomes/all/GCF/000/001/215" +@pytest.mark.parametrize( + ("url", "expected", "kwargs"), + [ + pytest.param( + "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6", + "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6", + {}, + id="https_url", + ), + pytest.param( + "ftp://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6", + "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6", + {}, + id="ftp_url", + ), + pytest.param( + "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6", + "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6", + {}, + id="bare_path", + ), + pytest.param( + "ftp://custom.host.example.com/genomes/all/GCF/000/001/215", + "/genomes/all/GCF/000/001/215", + {"ftp_host": "custom.host.example.com"}, + id="custom_ftp_host", + ), + ], +) +def test_ftp_dir_from_url(url: str, expected: str, kwargs: dict) -> None: + assert _ftp_dir_from_url(url, **kwargs) == expected # ── verify_transfer_candidates ─────────────────────────────────────────── @@ -341,468 +293,384 @@ def _mock_s3_empty() -> MagicMock: return client -class TestVerifyTransferCandidates: - """Test S3 checksum verification to prune transfer candidates.""" - - def _assemblies(self) -> dict: - return parse_assembly_summary(SAMPLE_SUMMARY) - - @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) - @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") - @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) - @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") - def test_prunes_when_all_match( - self, - mock_connect: MagicMock, - mock_retrieve: MagicMock, - mock_head: MagicMock, - mock_s3: MagicMock, - ) -> None: - """Assemblies where every file matches S3 are pruned from the list.""" - mock_connect.return_value = MagicMock() - - def head_side_effect(s3_path: str) -> dict | None: +_BUCKET = "cdm-lake" +_KEY_PREFIX = "tenant-general-warehouse/kbase/datasets/ncbi/" + + +def _assemblies() -> dict: + return parse_assembly_summary(SAMPLE_SUMMARY) + + +@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) +@patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") +@patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) +@patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") +def test_verify_transfer_candidates_prunes_when_all_match( + mock_connect: MagicMock, + mock_retrieve: MagicMock, + mock_head: MagicMock, + mock_s3: MagicMock, +) -> None: + """Assemblies where every file matches S3 are pruned from the list.""" + mock_connect.return_value = MagicMock() + + def head_side_effect(s3_path: str) -> dict | None: + if "_genomic.fna.gz" in s3_path: + return {"size": 100, "metadata": {"md5": "d41d8cd98f00b204e9800998ecf8427e"}, "checksum_crc64nvme": None} + if "_protein.faa.gz" in s3_path: + return {"size": 100, "metadata": {"md5": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4"}, "checksum_crc64nvme": None} + if "_assembly_report.txt" in s3_path: + return {"size": 100, "metadata": {"md5": "ffffffffffffffffffffffffffffffff"}, "checksum_crc64nvme": None} + return None + + mock_head.side_effect = head_side_effect + assert verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) == [] + + +@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) +@patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") +@patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) +@patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") +def test_verify_transfer_candidates_keeps_when_md5_differs( + mock_connect: MagicMock, + mock_retrieve: MagicMock, + mock_head: MagicMock, + mock_s3: MagicMock, +) -> None: + """Assembly is kept when at least one file has a different MD5.""" + mock_connect.return_value = MagicMock() + mock_head.return_value = {"size": 100, "metadata": {"md5": "WRONG"}, "checksum_crc64nvme": None} + assert verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) == ["GCF_000001215.4"] + + +@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) +@patch("cdm_data_loaders.ncbi_ftp.manifest.head_object", return_value=None) +@patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) +@patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") +def test_verify_transfer_candidates_keeps_when_s3_object_missing( + mock_connect: MagicMock, + mock_retrieve: MagicMock, + mock_head: MagicMock, + mock_s3: MagicMock, +) -> None: + """Assembly is kept when at least one file doesn't exist in S3.""" + mock_connect.return_value = MagicMock() + assert verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) == ["GCF_000001215.4"] + + +@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) +@patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") +@patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) +@patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") +def test_verify_transfer_candidates_keeps_when_no_md5_metadata( + mock_connect: MagicMock, + mock_retrieve: MagicMock, + mock_head: MagicMock, + mock_s3: MagicMock, +) -> None: + """Assembly is kept when S3 object exists but has no md5 metadata.""" + mock_connect.return_value = MagicMock() + mock_head.return_value = {"size": 100, "metadata": {}, "checksum_crc64nvme": None} + assert verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) == ["GCF_000001215.4"] + + +@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) +@patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", side_effect=Exception("FTP error")) +@patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") +def test_verify_transfer_candidates_keeps_when_ftp_fails( + mock_connect: MagicMock, mock_retrieve: MagicMock, mock_s3: MagicMock +) -> None: + """Assembly is kept (conservative) when md5checksums.txt cannot be fetched.""" + mock_connect.return_value = MagicMock() + assert verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) == ["GCF_000001215.4"] + + +@patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") +def test_verify_transfer_candidates_empty_input(mock_connect: MagicMock) -> None: + """Empty accession list returns empty result without connecting.""" + assert verify_transfer_candidates([], {}, _BUCKET, "prefix/") == [] + + +@patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") +def test_verify_transfer_candidates_unknown_accession_kept(mock_connect: MagicMock) -> None: + """Accessions not in assemblies dict are kept (conservative).""" + mock_connect.return_value = MagicMock() + assert verify_transfer_candidates(["GCF_999999999.1"], {}, _BUCKET, "prefix/") == ["GCF_999999999.1"] + + +@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) +@patch("cdm_data_loaders.ncbi_ftp.manifest.head_object", return_value=None) +@patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) +@patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") +def test_verify_transfer_candidates_short_circuits_on_first_mismatch( + mock_connect: MagicMock, + mock_retrieve: MagicMock, + mock_head: MagicMock, + mock_s3: MagicMock, +) -> None: + """Verification stops checking after the first missing/mismatched file.""" + mock_connect.return_value = MagicMock() + verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) + assert mock_head.call_count == 1 + + +@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) +@patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") +@patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) +@patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") +def test_verify_transfer_candidates_mixed( + mock_connect: MagicMock, + mock_retrieve: MagicMock, + mock_head: MagicMock, + mock_s3: MagicMock, +) -> None: + """A mix of matching and non-matching assemblies: matched pruned, unmatched kept.""" + mock_connect.return_value = MagicMock() + + def head_side_effect(s3_path: str) -> dict | None: + if "GCF_000001215.4_Release_6_plus_ISO1_MT/" in s3_path: if "_genomic.fna.gz" in s3_path: - return { - "size": 100, - "metadata": {"md5": "d41d8cd98f00b204e9800998ecf8427e"}, - "checksum_crc64nvme": None, - } + return {"size": 1, "metadata": {"md5": "d41d8cd98f00b204e9800998ecf8427e"}, "checksum_crc64nvme": None} if "_protein.faa.gz" in s3_path: - return { - "size": 100, - "metadata": {"md5": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4"}, - "checksum_crc64nvme": None, - } + return {"size": 1, "metadata": {"md5": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4"}, "checksum_crc64nvme": None} if "_assembly_report.txt" in s3_path: - return { - "size": 100, - "metadata": {"md5": "ffffffffffffffffffffffffffffffff"}, - "checksum_crc64nvme": None, - } - return None - - mock_head.side_effect = head_side_effect - result = verify_transfer_candidates( - ["GCF_000001215.4"], - self._assemblies(), - "cdm-lake", - "tenant-general-warehouse/kbase/datasets/ncbi/", - ) - assert result == [] - - @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) - @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") - @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) - @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") - def test_keeps_when_md5_differs( - self, - mock_connect: MagicMock, - mock_retrieve: MagicMock, - mock_head: MagicMock, - mock_s3: MagicMock, - ) -> None: - """Assembly is kept when at least one file has a different MD5.""" - mock_connect.return_value = MagicMock() - mock_head.return_value = {"size": 100, "metadata": {"md5": "WRONG"}, "checksum_crc64nvme": None} - - result = verify_transfer_candidates( - ["GCF_000001215.4"], - self._assemblies(), - "cdm-lake", - "tenant-general-warehouse/kbase/datasets/ncbi/", - ) - assert result == ["GCF_000001215.4"] - - @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) - @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object", return_value=None) - @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) - @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") - def test_keeps_when_s3_object_missing( - self, - mock_connect: MagicMock, - mock_retrieve: MagicMock, - mock_head: MagicMock, - mock_s3: MagicMock, - ) -> None: - """Assembly is kept when at least one file doesn't exist in S3.""" - mock_connect.return_value = MagicMock() - - result = verify_transfer_candidates( - ["GCF_000001215.4"], - self._assemblies(), - "cdm-lake", - "tenant-general-warehouse/kbase/datasets/ncbi/", - ) - assert result == ["GCF_000001215.4"] - - @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) - @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") - @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) - @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") - def test_keeps_when_s3_has_no_md5_metadata( - self, - mock_connect: MagicMock, - mock_retrieve: MagicMock, - mock_head: MagicMock, - mock_s3: MagicMock, - ) -> None: - """Assembly is kept when S3 object exists but has no md5 metadata.""" - mock_connect.return_value = MagicMock() - mock_head.return_value = {"size": 100, "metadata": {}, "checksum_crc64nvme": None} - - result = verify_transfer_candidates( - ["GCF_000001215.4"], - self._assemblies(), - "cdm-lake", - "tenant-general-warehouse/kbase/datasets/ncbi/", - ) - assert result == ["GCF_000001215.4"] - - @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) - @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", side_effect=Exception("FTP error")) - @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") - def test_keeps_when_ftp_fails(self, mock_connect: MagicMock, mock_retrieve: MagicMock, mock_s3: MagicMock) -> None: - """Assembly is kept (conservative) when md5checksums.txt cannot be fetched.""" - mock_connect.return_value = MagicMock() - - result = verify_transfer_candidates( - ["GCF_000001215.4"], - self._assemblies(), - "cdm-lake", - "tenant-general-warehouse/kbase/datasets/ncbi/", - ) - assert result == ["GCF_000001215.4"] - - @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") - def test_empty_input(self, mock_connect: MagicMock) -> None: - """Empty accession list returns empty result without connecting.""" - result = verify_transfer_candidates([], {}, "cdm-lake", "prefix/") - assert result == [] - - @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") - def test_unknown_accession_kept(self, mock_connect: MagicMock) -> None: - """Accessions not in assemblies dict are kept (conservative).""" - mock_connect.return_value = MagicMock() - result = verify_transfer_candidates(["GCF_999999999.1"], {}, "cdm-lake", "prefix/") - assert result == ["GCF_999999999.1"] - - @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) - @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object", return_value=None) - @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) - @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") - def test_short_circuits_on_first_mismatch( - self, - mock_connect: MagicMock, - mock_retrieve: MagicMock, - mock_head: MagicMock, - mock_s3: MagicMock, - ) -> None: - """Verification stops checking after the first missing/mismatched file.""" - mock_connect.return_value = MagicMock() - - verify_transfer_candidates( - ["GCF_000001215.4"], - self._assemblies(), - "cdm-lake", - "tenant-general-warehouse/kbase/datasets/ncbi/", - ) - assert mock_head.call_count == 1 - - @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) - @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") - @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) - @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") - def test_mixed_candidates( - self, - mock_connect: MagicMock, - mock_retrieve: MagicMock, - mock_head: MagicMock, - mock_s3: MagicMock, - ) -> None: - """Verify a mix of matching and non-matching assemblies.""" - mock_connect.return_value = MagicMock() - - def head_side_effect(s3_path: str) -> dict | None: - # GCF_000001215.4 assembly dir → all match; GCF_000001405.40 → missing - if "GCF_000001215.4_Release_6_plus_ISO1_MT/" in s3_path: - if "_genomic.fna.gz" in s3_path: - return { - "size": 1, - "metadata": {"md5": "d41d8cd98f00b204e9800998ecf8427e"}, - "checksum_crc64nvme": None, - } - if "_protein.faa.gz" in s3_path: - return { - "size": 1, - "metadata": {"md5": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4"}, - "checksum_crc64nvme": None, - } - if "_assembly_report.txt" in s3_path: - return { - "size": 1, - "metadata": {"md5": "ffffffffffffffffffffffffffffffff"}, - "checksum_crc64nvme": None, - } - return None - - mock_head.side_effect = head_side_effect - result = verify_transfer_candidates( - ["GCF_000001215.4", "GCF_000001405.40"], - self._assemblies(), - "cdm-lake", - "tenant-general-warehouse/kbase/datasets/ncbi/", - ) - assert result == ["GCF_000001405.40"] - - @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_empty()) - @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") - def test_skips_ftp_when_folder_missing_from_store( - self, - mock_connect: MagicMock, - mock_s3: MagicMock, - ) -> None: - """Accessions with no objects in S3 are confirmed without FTP round-trip.""" - result = verify_transfer_candidates( - ["GCF_000001215.4"], - self._assemblies(), - "cdm-lake", - "tenant-general-warehouse/kbase/datasets/ncbi/", - ) - assert result == ["GCF_000001215.4"] - # FTP should never have been connected (lazy init) - mock_connect.assert_not_called() + return {"size": 1, "metadata": {"md5": "ffffffffffffffffffffffffffffffff"}, "checksum_crc64nvme": None} + return None + mock_head.side_effect = head_side_effect + result = verify_transfer_candidates(["GCF_000001215.4", "GCF_000001405.40"], _assemblies(), _BUCKET, _KEY_PREFIX) + assert result == ["GCF_000001405.40"] -# ── Synthetic summary from S3 store scan ──────────────────────────────── +@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_empty()) +@patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") +def test_verify_transfer_candidates_skips_ftp_when_folder_missing( + mock_connect: MagicMock, + mock_s3: MagicMock, +) -> None: + """Accessions with no objects in S3 are confirmed without FTP round-trip.""" + result = verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) + assert result == ["GCF_000001215.4"] + mock_connect.assert_not_called() -class TestExtractAccessionFromS3Key: - """Test accession extraction from S3 paths.""" - - @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") - def test_extracts_accession_from_path(self, _mock_s3: MagicMock) -> None: - """Verify accession is extracted correctly from S3 keys.""" - assert ( - _extract_accession_from_s3_key( - "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6_plus_ISO1_MT/file.gz" - ) - == "GCF_000001215.4" - ) - assert _extract_accession_from_s3_key("some/path/GCA_999999999.1_whatever/data.txt") == "GCA_999999999.1" - - def test_returns_none_for_invalid_path(self) -> None: - """Verify None is returned when no accession is found.""" - assert _extract_accession_from_s3_key("some/random/path") is None - assert _extract_accession_from_s3_key("") is None +# ── Synthetic summary from S3 store scan ──────────────────────────────── -class TestExtractAssemblyDirFromS3Key: - """Test assembly directory extraction from S3 paths.""" - def test_extracts_assembly_dir(self) -> None: - """Verify assembly directory is extracted correctly from S3 keys.""" - assert ( - _extract_assembly_dir_from_s3_key( - "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6_plus_ISO1_MT/file.gz" +@pytest.mark.parametrize( + ("key", "expected"), + [ + pytest.param( + "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6_plus_ISO1_MT/file.gz", + "GCF_000001215.4", + id="long_path", + ), + pytest.param("some/path/GCA_999999999.1_whatever/data.txt", "GCA_999999999.1", id="short_path"), + pytest.param("some/random/path", None, id="no_accession"), + pytest.param("", None, id="empty"), + ], +) +def test_extract_accession_from_s3_key(key: str, expected: str | None) -> None: + """Accession is extracted from S3 key paths; invalid/empty paths return None.""" + assert _extract_accession_from_s3_key(key) == expected + + +@pytest.mark.parametrize( + ("key", "expected"), + [ + pytest.param( + "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6_plus_ISO1_MT/file.gz", + "GCF_000001215.4_Release_6_plus_ISO1_MT", + id="long_path", + ), + pytest.param( + "prefix/GCA_999999999.1_assembly_name/subdir/data.txt", + "GCA_999999999.1_assembly_name", + id="subdir", + ), + pytest.param("some/random/path", None, id="no_assembly_dir"), + pytest.param("", None, id="empty"), + ], +) +def test_extract_assembly_dir_from_s3_key(key: str, expected: str | None) -> None: + """Assembly directory is extracted from S3 key paths; invalid/empty paths return None.""" + assert _extract_assembly_dir_from_s3_key(key) == expected + + +def _make_mock_s3_paginator() -> MagicMock: + """Return a mock S3 client with two assemblies (GCF_000001215.4, GCF_000005845.2).""" + from datetime import datetime, timezone + + mock = MagicMock() + mock_paginator = MagicMock() + mock.get_paginator.return_value = mock_paginator + mock_paginator.paginate.return_value = [ + { + "Contents": [ + { + "Key": "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6/file1.gz", + "LastModified": datetime(2024, 1, 15, tzinfo=timezone.utc), + }, + { + "Key": "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6/file2.gz", + "LastModified": datetime(2024, 1, 16, tzinfo=timezone.utc), + }, + { + "Key": "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000005845.2_Assembly/file.gz", + "LastModified": datetime(2024, 2, 20, tzinfo=timezone.utc), + }, + ] + } + ] + return mock + + +@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") +def test_scan_store_builds_summary(mock_get_s3: MagicMock) -> None: + """Synthetic summary is built correctly with provided release_date for all assemblies.""" + mock_get_s3.return_value = _make_mock_s3_paginator() + assert scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/01/31") == { + "GCF_000001215.4": AssemblyRecord( + accession="GCF_000001215.4", + status="latest", + seq_rel_date="2024/01/31", + ftp_path="https://ftp.ncbi.nlm.nih.gov/synthetic/GCF_000001215.4_Release_6", + assembly_dir="GCF_000001215.4_Release_6", + ), + "GCF_000005845.2": AssemblyRecord( + accession="GCF_000005845.2", + status="latest", + seq_rel_date="2024/01/31", + ftp_path="https://ftp.ncbi.nlm.nih.gov/synthetic/GCF_000005845.2_Assembly", + assembly_dir="GCF_000005845.2_Assembly", + ), + } + + +@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") +def test_scan_store_applies_release_date_to_all(mock_get_s3: MagicMock) -> None: + """Provided release_date is used even when files have different LastModified dates.""" + from datetime import datetime, timezone + + mock = MagicMock() + mock_paginator = MagicMock() + mock.get_paginator.return_value = mock_paginator + mock_paginator.paginate.return_value = [ + { + "Contents": [ + { + "Key": "prefix/GCF_000001215.4_v1/file_newer.gz", + "LastModified": datetime(2024, 3, 20, tzinfo=timezone.utc), + }, + { + "Key": "prefix/GCF_000001215.4_v1/file_older.gz", + "LastModified": datetime(2024, 1, 10, tzinfo=timezone.utc), + }, + ] + } + ] + mock_get_s3.return_value = mock + assert ( + scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/03/31")["GCF_000001215.4"].seq_rel_date + == "2024/03/31" + ) + + +@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") +def test_scan_store_raises_for_invalid_release_date(mock_get_s3: MagicMock) -> None: + """Invalid release_date format is rejected.""" + mock_get_s3.return_value = _make_mock_s3_paginator() + with pytest.raises(ValueError, match="Invalid release_date"): + scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024-03-31") + + +@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") +def test_scan_store_invokes_progress_callback(mock_get_s3: MagicMock) -> None: + """Progress callback is called once per unique assembly discovered.""" + mock_get_s3.return_value = _make_mock_s3_paginator() + calls: list[tuple[int, str]] = [] + scan_store_to_synthetic_summary( + "test-bucket", "prefix/", "2024/01/31", progress_callback=lambda n, a: calls.append((n, a)) + ) + assert len(calls) == 2 + assert calls[0][0] == 1 + assert calls[1][0] == 2 + + +@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") +def test_scan_store_handles_empty_store(mock_get_s3: MagicMock) -> None: + """Empty store returns empty dict.""" + mock = MagicMock() + mock_paginator = MagicMock() + mock.get_paginator.return_value = mock_paginator + mock_paginator.paginate.return_value = [{"Contents": []}] + mock_get_s3.return_value = mock + assert scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/01/31") == {} + + +@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") +def test_scan_store_skips_objects_without_accession(mock_get_s3: MagicMock) -> None: + """Objects without valid accessions in the key are skipped.""" + from datetime import datetime, timezone + + mock = MagicMock() + mock_paginator = MagicMock() + mock.get_paginator.return_value = mock_paginator + mock_paginator.paginate.return_value = [ + { + "Contents": [ + {"Key": "prefix/some/random/file.txt", "LastModified": datetime(2024, 1, 1, tzinfo=timezone.utc)}, + { + "Key": "prefix/GCF_000001215.4_Assembly/valid_file.gz", + "LastModified": datetime(2024, 2, 1, tzinfo=timezone.utc), + }, + ] + } + ] + mock_get_s3.return_value = mock + result = scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/01/31") + assert len(result) == 1 + assert "GCF_000001215.4" in result + + +@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") +def test_scan_store_assembly_dir_survives_round_trip(mock_get_s3: MagicMock, tmp_path: Path) -> None: + """assembly_dir is preserved after save-to-file / parse-back round-trip. + + Regression: previously ftp_path was written as "" causing assembly_dir="" + and compute_diff flagging every assembly as updated. + """ + from datetime import datetime, timezone + + mock = MagicMock() + mock_paginator = MagicMock() + mock_paginator.paginate.return_value = [ + { + "Contents": [ + { + "Key": "prefix/GCF_000001215.4_Release_6_plus_ISO1_MT/file.gz", + "LastModified": datetime(2024, 3, 10, tzinfo=timezone.utc), + } + ] + } + ] + mock.get_paginator.return_value = mock_paginator + mock_get_s3.return_value = mock + + synthetic = scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/03/10") + + out_file = tmp_path / "synthetic_summary.txt" + with out_file.open("w") as f: + for acc in sorted(synthetic.keys()): + rec = synthetic[acc] + f.write( + f"{rec.accession}\t.\t.\t.\t.\t.\t.\t.\t.\t.\t{rec.status}\t.\t.\t.\t{rec.seq_rel_date}\t.\t.\t.\t.\t{rec.ftp_path}\t.\n" ) - == "GCF_000001215.4_Release_6_plus_ISO1_MT" - ) - assert ( - _extract_assembly_dir_from_s3_key("prefix/GCA_999999999.1_assembly_name/subdir/data.txt") - == "GCA_999999999.1_assembly_name" - ) - - def test_returns_none_for_invalid_path(self) -> None: - """Verify None is returned when no assembly directory is found.""" - assert _extract_assembly_dir_from_s3_key("some/random/path") is None - assert _extract_assembly_dir_from_s3_key("") is None - - -class TestScanStoreToSyntheticSummary: - """Test synthetic assembly summary generation from S3 store scan.""" - - def _mock_s3_with_objects(self) -> MagicMock: - """Return a mock S3 client with assembly objects.""" - from datetime import datetime, timezone - - mock = MagicMock() - mock_paginator = MagicMock() - mock.get_paginator.return_value = mock_paginator - - # Mock objects from two assemblies - page_contents = [ - { - "Key": "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6/file1.gz", - "LastModified": datetime(2024, 1, 15, tzinfo=timezone.utc), - }, - { - "Key": "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6/file2.gz", - "LastModified": datetime(2024, 1, 16, tzinfo=timezone.utc), - }, - { - "Key": "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000005845.2_Assembly/file.gz", - "LastModified": datetime(2024, 2, 20, tzinfo=timezone.utc), - }, - ] - mock_paginator.paginate.return_value = [{"Contents": page_contents}] - return mock - - @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") - def test_builds_summary_from_store(self, mock_get_s3: MagicMock) -> None: - """Verify synthetic summary is built correctly from S3 objects.""" - mock_get_s3.return_value = self._mock_s3_with_objects() - - result = scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/01/31") - - assert len(result) == 2 - assert "GCF_000001215.4" in result - assert "GCF_000005845.2" in result - - # Should use provided release date for all records - rec1 = result["GCF_000001215.4"] - assert rec1.accession == "GCF_000001215.4" - assert rec1.status == "latest" - assert rec1.seq_rel_date == "2024/01/31" - assert rec1.assembly_dir == "GCF_000001215.4_Release_6" - - # Other assembly uses the same provided date - rec2 = result["GCF_000005845.2"] - assert rec2.seq_rel_date == "2024/01/31" - - @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") - def test_applies_release_date_to_all_assemblies(self, mock_get_s3: MagicMock) -> None: - """Verify provided release_date is used for all assemblies.""" - mock = MagicMock() - mock_paginator = MagicMock() - mock.get_paginator.return_value = mock_paginator - - from datetime import datetime, timezone - - # Files from same assembly with different dates - page_contents = [ - { - "Key": "prefix/GCF_000001215.4_v1/file_newer.gz", - "LastModified": datetime(2024, 3, 20, tzinfo=timezone.utc), - }, - { - "Key": "prefix/GCF_000001215.4_v1/file_older.gz", - "LastModified": datetime(2024, 1, 10, tzinfo=timezone.utc), - }, - ] - mock_paginator.paginate.return_value = [{"Contents": page_contents}] - mock_get_s3.return_value = mock - - result = scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/03/31") - - assert result["GCF_000001215.4"].seq_rel_date == "2024/03/31" - - @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") - def test_raises_for_invalid_release_date(self, mock_get_s3: MagicMock) -> None: - """Verify invalid release_date format is rejected.""" - mock_get_s3.return_value = self._mock_s3_with_objects() - - with pytest.raises(ValueError, match="Invalid release_date"): - scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024-03-31") - - @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") - def test_invokes_progress_callback(self, mock_get_s3: MagicMock) -> None: - """Verify progress callback is called for each unique assembly.""" - mock_get_s3.return_value = self._mock_s3_with_objects() - callback_calls = [] - - def track_progress(count: int, acc: str) -> None: - callback_calls.append((count, acc)) - - scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/01/31", progress_callback=track_progress) - - # Should have 2 calls (one per assembly discovered) - assert len(callback_calls) == 2 - assert callback_calls[0][0] == 1 # first assembly - assert callback_calls[1][0] == 2 # second assembly - - @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") - def test_handles_empty_store(self, mock_get_s3: MagicMock) -> None: - """Verify function handles empty store gracefully.""" - mock = MagicMock() - mock_paginator = MagicMock() - mock.get_paginator.return_value = mock_paginator - mock_paginator.paginate.return_value = [{"Contents": []}] - mock_get_s3.return_value = mock - - result = scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/01/31") - - assert result == {} - - @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") - def test_skips_objects_without_accession(self, mock_get_s3: MagicMock) -> None: - """Verify objects without valid accessions are skipped.""" - from datetime import datetime, timezone - - mock = MagicMock() - mock_paginator = MagicMock() - mock.get_paginator.return_value = mock_paginator - - page_contents = [ - { - "Key": "prefix/some/random/file.txt", # No accession - "LastModified": datetime(2024, 1, 1, tzinfo=timezone.utc), - }, - { - "Key": "prefix/GCF_000001215.4_Assembly/valid_file.gz", - "LastModified": datetime(2024, 2, 1, tzinfo=timezone.utc), - }, - ] - mock_paginator.paginate.return_value = [{"Contents": page_contents}] - mock_get_s3.return_value = mock - - result = scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/01/31") - - # Only one valid assembly should be found - assert len(result) == 1 - assert "GCF_000001215.4" in result - - @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") - def test_assembly_dir_survives_file_round_trip(self, mock_get_s3: MagicMock, tmp_path: Path) -> None: - """Verify assembly_dir is preserved when saving to file and parsing back. - - Regression test: previously ftp_path was written as "" which caused - parse_assembly_summary to recover assembly_dir="" for all records, - making compute_diff flag every assembly as updated. - """ - from datetime import datetime, timezone - - mock = MagicMock() - mock_paginator = MagicMock() - page_contents = [ - { - "Key": "prefix/GCF_000001215.4_Release_6_plus_ISO1_MT/file.gz", - "LastModified": datetime(2024, 3, 10, tzinfo=timezone.utc), - }, - ] - mock_paginator.paginate.return_value = [{"Contents": page_contents}] - mock.get_paginator.return_value = mock_paginator - mock_get_s3.return_value = mock - - synthetic = scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/03/10") - - # Simulate the notebook's save logic - out_file = tmp_path / "synthetic_summary.txt" - with out_file.open("w") as f: - for acc in sorted(synthetic.keys()): - rec = synthetic[acc] - f.write( - f"{rec.accession}\t.\t.\t.\t.\t.\t.\t.\t.\t.\t{rec.status}\t.\t.\t.\t{rec.seq_rel_date}\t.\t.\t.\t.\t{rec.ftp_path}\t.\n" - ) - - # Parse the file back - reparsed = parse_assembly_summary(out_file) - - assert "GCF_000001215.4" in reparsed - reparsed_rec = reparsed["GCF_000001215.4"] - original_rec = synthetic["GCF_000001215.4"] - - # assembly_dir must survive the round-trip so diffs are accurate - assert reparsed_rec.assembly_dir == original_rec.assembly_dir - assert reparsed_rec.seq_rel_date == original_rec.seq_rel_date - assert reparsed_rec.status == original_rec.status + + reparsed = parse_assembly_summary(out_file) + assert "GCF_000001215.4" in reparsed + reparsed_rec = reparsed["GCF_000001215.4"] + original_rec = synthetic["GCF_000001215.4"] + assert reparsed_rec.assembly_dir == original_rec.assembly_dir + assert reparsed_rec.seq_rel_date == original_rec.seq_rel_date + assert reparsed_rec.status == original_rec.status diff --git a/tests/ncbi_ftp/test_metadata.py b/tests/ncbi_ftp/test_metadata.py index bcdd825d..0b471b48 100644 --- a/tests/ncbi_ftp/test_metadata.py +++ b/tests/ncbi_ftp/test_metadata.py @@ -57,317 +57,207 @@ ] -# ── build_descriptor_key ───────────────────────────────────────────────── +# ── build_descriptor_key / build_archive_descriptor_key ───────────────── -class TestBuildDescriptorKey: - """Tests for build_descriptor_key path helper.""" +@pytest.mark.parametrize("prefix", [_KEY_PREFIX, _KEY_PREFIX.rstrip("/")]) +def test_build_descriptor_key(prefix: str) -> None: + """Key is under metadata/, ends with _datapackage.json, trailing slash on prefix is normalized.""" + key = build_descriptor_key(_ASSEMBLY_DIR, prefix) + assert key == f"{_KEY_PREFIX}metadata/{_ASSEMBLY_DIR}_datapackage.json" + assert "//" not in key - def test_produces_metadata_path(self) -> None: - """Key is located under metadata/ with _datapackage.json suffix.""" - key = build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX) - assert key == f"{_KEY_PREFIX}metadata/{_ASSEMBLY_DIR}_datapackage.json" - def test_trailing_slash_normalised(self) -> None: - """Key is the same whether key_prefix ends with a slash or not.""" - key_no_slash = build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX.rstrip("/")) - key_with_slash = build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX) - assert key_no_slash == key_with_slash +@pytest.mark.parametrize( + ("prefix", "tag"), + [ + pytest.param(_KEY_PREFIX, _RELEASE_TAG, id="trailing_slash"), + pytest.param(_KEY_PREFIX.rstrip("/"), _RELEASE_TAG, id="no_trailing_slash"), + pytest.param(_KEY_PREFIX, "2025-06", id="different_tag"), + ], +) +def test_build_archive_descriptor_key(prefix: str, tag: str) -> None: + """Archive key includes tag and has no double slash; trailing slash on prefix is normalized.""" + key = build_archive_descriptor_key(_ASSEMBLY_DIR, tag, prefix) + assert key == f"{_KEY_PREFIX}archive/{tag}/metadata/{_ASSEMBLY_DIR}_datapackage.json" + assert "//" not in key - def test_no_double_slash(self) -> None: - """Key never contains a double slash.""" - key = build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX) - assert "//" not in key +# ── create_descriptor ──────────────────────────────────────────────────── -# ── build_archive_descriptor_key ───────────────────────────────────────── +def test_create_descriptor() -> None: + """create_descriptor produces a fully populated descriptor matching the expected structure.""" + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + + # URL hostname is computed; can't express as equality + host = urlparse(d["url"]).hostname + assert host is not None and (host == "ncbi.nlm.nih.gov" or host.endswith(".ncbi.nlm.nih.gov")) + + # resource[1]: hash=None → key absent; bytes=512 → key present + r1 = d["resources"][1] + assert "hash" not in r1 + assert "bytes" in r1 + + assert {k: d[k] for k in ("identifier", "resource_type", "version", "license", "contributors")} == { + "identifier": f"NCBI:{_ACCESSION}", + "resource_type": "dataset", + "version": "4", + "license": {}, + "contributors": [ + { + "name": "National Center for Biotechnology Information", + "contributor_id": "ROR:02meqm098", + "contributor_type": "Organization", + "contributor_roles": "DataCurator", + } + ], + } + assert { + k: d["meta"][k] for k in ("saved_by", "credit_metadata_schema_version", "timestamp", "credit_metadata_source") + } == { + "saved_by": "cdm-data-loaders-ncbi-ftp", + "credit_metadata_schema_version": "1.0", + "timestamp": _TIMESTAMP, + "credit_metadata_source": [ + { + "access_timestamp": _TIMESTAMP, + "source_name": "NCBI Genomes FTP", + "source_url": "ftp.ncbi.nlm.nih.gov/genomes/all/", + } + ], + } + assert _ASSEMBLY_DIR in d["titles"][0]["title"] + assert _ACCESSION in d["descriptions"][0]["description_text"] + r0 = d["resources"][0] + assert {k: r0[k] for k in ("hash", "bytes", "path")} == { + "hash": "abc123", + "bytes": 1024, + "path": _SAMPLE_RESOURCES[0]["path"], + } -class TestBuildArchiveDescriptorKey: - """Tests for build_archive_descriptor_key path helper.""" - def test_produces_archive_path(self) -> None: - """Key is located under archive/{release_tag}/metadata/.""" - key = build_archive_descriptor_key(_ASSEMBLY_DIR, _RELEASE_TAG, _KEY_PREFIX) - expected = f"{_KEY_PREFIX}archive/{_RELEASE_TAG}/metadata/{_ASSEMBLY_DIR}_datapackage.json" - assert key == expected +def test_create_descriptor_default_timestamp_is_recent() -> None: + """Default timestamp is close to current time when not specified.""" + before = int(time.time()) + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES) + after = int(time.time()) + assert before <= d["meta"]["timestamp"] <= after + 1 - def test_trailing_slash_normalised(self) -> None: - """Key is the same whether key_prefix ends with a slash or not.""" - a = build_archive_descriptor_key(_ASSEMBLY_DIR, _RELEASE_TAG, _KEY_PREFIX.rstrip("/")) - b = build_archive_descriptor_key(_ASSEMBLY_DIR, _RELEASE_TAG, _KEY_PREFIX) - assert a == b - def test_release_tag_in_path(self) -> None: - """Release tag appears in the archive key path.""" - key = build_archive_descriptor_key(_ASSEMBLY_DIR, "2025-06", _KEY_PREFIX) - assert "2025-06" in key +def test_create_descriptor_resource_name_lowercased() -> None: + """Resource names are converted to lowercase.""" + resources: list[DescriptorResource] = [ + {"name": "FILE_UPPER.FNA.GZ", "path": "s3://bucket/a", "format": "gz", "bytes": 100, "hash": "x"}, + ] + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, resources, timestamp=_TIMESTAMP) + assert d["resources"][0]["name"] == "file_upper.fna.gz" -# ── create_descriptor ──────────────────────────────────────────────────── +def test_create_descriptor_null_bytes_omitted() -> None: + """Resources with bytes=None have the 'bytes' key removed from the output.""" + resources: list[DescriptorResource] = [ + {"name": "f.txt", "path": "s3://b/f.txt", "format": "txt", "bytes": None, "hash": "x"}, + ] + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, resources, timestamp=_TIMESTAMP) + assert "bytes" not in d["resources"][0] -class TestCreateDescriptor: - """Tests for create_descriptor().""" - - def test_identifier(self) -> None: - """Identifier field is prefixed with NCBI:.""" - d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - assert d["identifier"] == f"NCBI:{_ACCESSION}" - - def test_version_extracted_from_accession(self) -> None: - """Version is the suffix after the last dot in the accession.""" - d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - assert d["version"] == "4" # last segment of GCF_000001215.4 - - def test_title_includes_assembly_dir(self) -> None: - """Title includes the full assembly directory name.""" - d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - assert _ASSEMBLY_DIR in d["titles"][0]["title"] - - def test_description_includes_accession(self) -> None: - """Description text includes the accession.""" - d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - assert _ACCESSION in d["descriptions"][0]["description_text"] - - def test_url_references_accession(self) -> None: - """URL points to the NCBI genome page for the accession.""" - d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - assert _ACCESSION in d["url"] - parsed = urlparse(d["url"]) - host = parsed.hostname - assert host is not None - assert host == "ncbi.nlm.nih.gov" or host.endswith(".ncbi.nlm.nih.gov") - - def test_ncbi_contributor(self) -> None: - """Contributor is NCBI with the correct ROR ID.""" - d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - assert d["contributors"][0]["name"] == "National Center for Biotechnology Information" - assert d["contributors"][0]["contributor_id"] == "ROR:02meqm098" - - def test_saved_by(self) -> None: - """meta.saved_by is the cdm-data-loaders identifier.""" - d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - assert d["meta"]["saved_by"] == "cdm-data-loaders-ncbi-ftp" - - def test_timestamp_propagated(self) -> None: - """Explicit timestamp is used for both meta.timestamp and access_timestamp.""" - d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - assert d["meta"]["timestamp"] == _TIMESTAMP - assert d["meta"]["credit_metadata_source"][0]["access_timestamp"] == _TIMESTAMP - - def test_default_timestamp_is_recent(self) -> None: - """Default timestamp is close to current time when not specified.""" - before = int(time.time()) - d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES) - after = int(time.time()) - ts = d["meta"]["timestamp"] - assert before <= ts <= after + 1 - - def test_resource_names_lowercased(self) -> None: - """Resource names are converted to lowercase.""" - resources: list[DescriptorResource] = [ - {"name": "FILE_UPPER.FNA.GZ", "path": "s3://bucket/a", "format": "gz", "bytes": 100, "hash": "x"}, - ] - d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, resources, timestamp=_TIMESTAMP) - assert d["resources"][0]["name"] == "file_upper.fna.gz" - - def test_null_hash_omitted(self) -> None: - """Resources with hash=None must not include the 'hash' key.""" - d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - resources = d["resources"] - # Second resource has hash=None → key absent - assert "hash" not in resources[1] - - def test_non_null_hash_present(self) -> None: - """Non-null hash is retained in the resource entry.""" - d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - assert d["resources"][0]["hash"] == _SAMPLE_RESOURCES[0]["hash"] - - def test_resource_count(self) -> None: - """Resource list length matches the number of input resources.""" - d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - assert len(d["resources"]) == len(_SAMPLE_RESOURCES) - - def test_resource_bytes(self) -> None: - """Resource bytes matches the input bytes value.""" - d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - assert d["resources"][0]["bytes"] == _SAMPLE_RESOURCES[0]["bytes"] - - def test_resource_path(self) -> None: - """Resource path matches the input path value.""" - d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - assert d["resources"][0]["path"] == _SAMPLE_RESOURCES[0]["path"] - - def test_license_is_empty_dict(self) -> None: - """License field is an empty dict.""" - d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - assert d["license"] == {} - - def test_resource_type_is_dataset(self) -> None: - """resource_type is 'dataset'.""" - d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - assert d["resource_type"] == "dataset" - - def test_schema_version(self) -> None: - """credit_metadata_schema_version is '1.0'.""" - d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - assert d["meta"]["credit_metadata_schema_version"] == "1.0" - - def test_empty_resources_allowed(self) -> None: - """Empty resources list produces a valid descriptor.""" - d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, [], timestamp=_TIMESTAMP) - assert d["resources"] == [] - - def test_null_bytes_omitted(self) -> None: - """Resources with bytes=None have the 'bytes' key removed from the output.""" - resources: list[DescriptorResource] = [ - {"name": "f.txt", "path": "s3://b/f.txt", "format": "txt", "bytes": None, "hash": "x"}, - ] - d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, resources, timestamp=_TIMESTAMP) - assert "bytes" not in d["resources"][0] +def test_create_descriptor_empty_resources() -> None: + """Empty resources list produces a valid descriptor.""" + d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, [], timestamp=_TIMESTAMP) + assert d["resources"] == [] # ── validate_descriptor ────────────────────────────────────────────────── -class TestValidateDescriptor: - """Tests for validate_descriptor().""" +def test_validate_descriptor_valid() -> None: + """Valid descriptor does not raise.""" + validate_descriptor( + create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP), + _ACCESSION, + ) + + +def test_validate_descriptor_empty_raises() -> None: + """Empty dict fails frictionless validation and raises.""" + with pytest.raises((ValueError, Exception)): + validate_descriptor({}, _ACCESSION) + + +# ── upload_descriptor / archive_descriptor ─────────────────────────────── - def test_valid_descriptor_passes(self) -> None: - """Valid descriptor does not raise.""" - d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - # Should not raise - validate_descriptor(d, _ACCESSION) - def test_empty_descriptor_raises(self) -> None: - """Empty dict fails frictionless validation and raises.""" - with pytest.raises((ValueError, Exception)): - validate_descriptor({}, _ACCESSION) +@pytest.fixture +def mock_s3() -> Generator[botocore.client.BaseClient]: + """Mocked S3 client with the CDM Lake bucket pre-created.""" + with mock_aws(): + client = boto3.client("s3", region_name=AWS_REGION) + client.create_bucket(Bucket=TEST_BUCKET) + reset_s3_client() + with ( + patch.object(s3_utils, "get_s3_client", return_value=client), + patch.object(metadata_mod, "get_s3_client", return_value=client), + ): + yield client + reset_s3_client() -# ── upload_descriptor ──────────────────────────────────────────────────── +@pytest.fixture +def mock_s3_with_descriptor(mock_s3: botocore.client.BaseClient) -> tuple[botocore.client.BaseClient, MagicMock]: + """mock_s3 with a live descriptor pre-uploaded and copy_object patched.""" + descriptor = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + live_key = build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX) + mock_s3.put_object(Bucket=TEST_BUCKET, Key=live_key, Body=json.dumps(descriptor).encode()) + with patch.object(metadata_mod, "copy_object") as mock_copy: + yield mock_s3, mock_copy + + +@pytest.mark.s3 +def test_upload_descriptor(mock_s3: botocore.client.BaseClient) -> None: + """Uploaded object is valid JSON at the expected key with the expected identifier.""" + descriptor = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + key = upload_descriptor(descriptor, _ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX) + expected_key = build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX) + assert key == expected_key + assert key.startswith(_KEY_PREFIX) + assert key.endswith("_datapackage.json") + body = json.loads(mock_s3.get_object(Bucket=TEST_BUCKET, Key=key)["Body"].read()) + assert body["identifier"] == f"NCBI:{_ACCESSION}" + + +@pytest.mark.s3 +def test_upload_descriptor_dry_run(mock_s3: botocore.client.BaseClient) -> None: + """Dry-run returns the correct key but creates no S3 object.""" + descriptor = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) + key = upload_descriptor(descriptor, _ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX, dry_run=True) + assert key == build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX) + objs = mock_s3.list_objects_v2(Bucket=TEST_BUCKET).get("Contents", []) + assert not any(o["Key"] == key for o in objs) + + +@pytest.mark.s3 +def test_archive_descriptor(mock_s3_with_descriptor: tuple[botocore.client.BaseClient, MagicMock]) -> None: + """archive_descriptor returns True and calls copy_object with the correct keys.""" + _, mock_copy = mock_s3_with_descriptor + result = archive_descriptor(_ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX, _RELEASE_TAG) + assert result is True + mock_copy.assert_called_once() + args = mock_copy.call_args[0] + assert f"{TEST_BUCKET}/{build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX)}" in args + assert f"{TEST_BUCKET}/{build_archive_descriptor_key(_ASSEMBLY_DIR, _RELEASE_TAG, _KEY_PREFIX)}" in args @pytest.mark.s3 -class TestUploadDescriptor: - """Tests for upload_descriptor() using moto-mocked S3.""" - - @pytest.fixture - def mock_s3(self) -> Generator[botocore.client.BaseClient]: - """Yield a mocked S3 client with the CDM Lake bucket pre-created.""" - with mock_aws(): - client = boto3.client("s3", region_name=AWS_REGION) - client.create_bucket(Bucket=TEST_BUCKET) - reset_s3_client() - with ( - patch.object(s3_utils, "get_s3_client", return_value=client), - patch.object(metadata_mod, "get_s3_client", return_value=client), - ): - yield client - reset_s3_client() - - def test_uploads_json(self, mock_s3: botocore.client.BaseClient) -> None: - """Uploaded object is valid JSON with the expected identifier.""" - descriptor = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - key = upload_descriptor(descriptor, _ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX) - assert key == build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX) - obj = mock_s3.get_object(Bucket=TEST_BUCKET, Key=key) - body = json.loads(obj["Body"].read()) - assert body["identifier"] == f"NCBI:{_ACCESSION}" - - def test_returns_expected_key(self, mock_s3: botocore.client.BaseClient) -> None: - """Return value is the metadata/ S3 key for the assembly.""" - descriptor = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - key = upload_descriptor(descriptor, _ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX) - assert key.startswith(_KEY_PREFIX) - assert key.endswith("_datapackage.json") - - def test_dry_run_skips_upload(self, mock_s3: botocore.client.BaseClient) -> None: - """Dry-run returns the key but does not create any S3 object.""" - descriptor = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - key = upload_descriptor(descriptor, _ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX, dry_run=True) - # No object in S3 - objs = mock_s3.list_objects_v2(Bucket=TEST_BUCKET).get("Contents", []) - assert not any(o["Key"] == key for o in objs) - - def test_dry_run_returns_key(self, mock_s3: botocore.client.BaseClient) -> None: - """Dry-run returns the same key as a real upload would.""" - descriptor = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - key = upload_descriptor(descriptor, _ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX, dry_run=True) - assert key == build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX) - - -# ── archive_descriptor ─────────────────────────────────────────────────── +def test_archive_descriptor_dry_run(mock_s3_with_descriptor: tuple[botocore.client.BaseClient, MagicMock]) -> None: + """Dry-run returns True but does not call copy_object.""" + _, mock_copy = mock_s3_with_descriptor + assert archive_descriptor(_ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX, _RELEASE_TAG, dry_run=True) is True + mock_copy.assert_not_called() @pytest.mark.s3 -class TestArchiveDescriptor: - """Tests for archive_descriptor() using moto-mocked S3.""" - - @pytest.fixture - def mock_s3_with_descriptor(self) -> Generator[tuple[botocore.client.BaseClient, MagicMock]]: - """S3 with a live descriptor already uploaded.""" - with mock_aws(): - client = boto3.client("s3", region_name=AWS_REGION) - client.create_bucket(Bucket=TEST_BUCKET) - # Pre-upload a descriptor - descriptor = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) - live_key = build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX) - client.put_object( - Bucket=TEST_BUCKET, - Key=live_key, - Body=json.dumps(descriptor).encode(), - ) - reset_s3_client() - with ( - patch.object(s3_utils, "get_s3_client", return_value=client), - patch.object(metadata_mod, "get_s3_client", return_value=client), - patch.object(metadata_mod, "copy_object") as mock_copy, - ): - yield client, mock_copy - reset_s3_client() - - def test_returns_true_when_descriptor_exists( - self, mock_s3_with_descriptor: tuple[botocore.client.BaseClient, MagicMock] - ) -> None: - """Returns True when the live descriptor object exists in S3.""" - _, _ = mock_s3_with_descriptor - result = archive_descriptor(_ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX, _RELEASE_TAG) - assert result is True - - def test_calls_copy_with_correct_keys( - self, mock_s3_with_descriptor: tuple[botocore.client.BaseClient, MagicMock] - ) -> None: - """copy_object is called with the live and archive keys.""" - _, mock_copy = mock_s3_with_descriptor - archive_descriptor(_ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX, _RELEASE_TAG) - live_key = build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX) - archive_key = build_archive_descriptor_key(_ASSEMBLY_DIR, _RELEASE_TAG, _KEY_PREFIX) - mock_copy.assert_called_once() - args = mock_copy.call_args - assert f"{TEST_BUCKET}/{live_key}" in args[0] - assert f"{TEST_BUCKET}/{archive_key}" in args[0] - - def test_dry_run_returns_true_without_copy( - self, mock_s3_with_descriptor: tuple[botocore.client.BaseClient, MagicMock] - ) -> None: - """Dry-run returns True but does not call copy_object.""" - _, mock_copy = mock_s3_with_descriptor - result = archive_descriptor(_ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX, _RELEASE_TAG, dry_run=True) - assert result is True - mock_copy.assert_not_called() - - def test_missing_descriptor_returns_false(self) -> None: - """Returns False when no descriptor exists at the live key.""" - with mock_aws(): - client = boto3.client("s3", region_name=AWS_REGION) - client.create_bucket(Bucket=TEST_BUCKET) - reset_s3_client() - with ( - patch.object(s3_utils, "get_s3_client", return_value=client), - patch.object(metadata_mod, "get_s3_client", return_value=client), - ): - result = archive_descriptor(_ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX, _RELEASE_TAG) - reset_s3_client() - assert result is False +def test_archive_descriptor_missing_returns_false(mock_s3: botocore.client.BaseClient) -> None: + """Returns False when no descriptor exists at the live key.""" + assert archive_descriptor(_ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX, _RELEASE_TAG) is False diff --git a/tests/ncbi_ftp/test_notebooks.py b/tests/ncbi_ftp/test_notebooks.py index 6d2e3758..fc3fa793 100644 --- a/tests/ncbi_ftp/test_notebooks.py +++ b/tests/ncbi_ftp/test_notebooks.py @@ -44,47 +44,31 @@ def _extract_code_cells(notebook_path: Path) -> list[str]: @pytest.mark.parametrize("notebook", NCBI_NOTEBOOKS) -class TestNotebookSyntax: - """Validate that every code cell in each notebook is syntactically valid Python.""" - - def test_all_cells_parse(self, notebook: str) -> None: - """Verify every code cell compiles without SyntaxError.""" - path = NOTEBOOKS_DIR / notebook - assert path.exists(), f"Notebook not found: {path}" - cells = _extract_code_cells(path) - assert len(cells) > 0, f"No code cells found in {notebook}" - for i, source in enumerate(cells, 1): - try: - ast.parse(source, filename=f"{notebook}:cell{i}") - except SyntaxError as exc: - pytest.fail(f"{notebook} cell {i} has a syntax error: {exc}") - - def test_no_empty_code_cells(self, notebook: str) -> None: - """Verify no code cell is completely empty.""" - path = NOTEBOOKS_DIR / notebook - cells = _extract_code_cells(path) - for i, source in enumerate(cells, 1): - assert source.strip(), f"{notebook} cell {i} is empty" - - -class TestManifestNotebookImports: - """Verify that all imports in the manifest notebook resolve.""" - - def test_imports_resolve(self) -> None: - """All manifest notebook imports are verified at module load time above.""" - assert isinstance(FTP_HOST, str) - assert FTP_HOST - assert AssemblyRecord is not None - assert callable(download_assembly_summary) - assert callable(compute_diff) - assert callable(write_updated_manifest) - - -class TestPromoteNotebookImports: - """Verify that all imports in the promote notebook resolve.""" - - def test_imports_resolve(self) -> None: - """All promote notebook imports are verified at module load time above.""" - assert callable(promote_from_s3) - assert isinstance(DEFAULT_LAKEHOUSE_KEY_PREFIX, str) - assert callable(split_s3_path) +def test_notebook_syntax(notebook: str) -> None: + """Every code cell is syntactically valid Python and non-empty.""" + path = NOTEBOOKS_DIR / notebook + assert path.exists(), f"Notebook not found: {path}" + cells = _extract_code_cells(path) + assert len(cells) > 0, f"No code cells found in {notebook}" + for i, source in enumerate(cells, 1): + assert source.strip(), f"{notebook} cell {i} is empty" + try: + ast.parse(source, filename=f"{notebook}:cell{i}") + except SyntaxError as exc: + pytest.fail(f"{notebook} cell {i} has a syntax error: {exc}") + + +def test_manifest_notebook_imports() -> None: + """All manifest notebook imports are verified at module load time above.""" + assert isinstance(FTP_HOST, str) and FTP_HOST + assert AssemblyRecord is not None + assert callable(download_assembly_summary) + assert callable(compute_diff) + assert callable(write_updated_manifest) + + +def test_promote_notebook_imports() -> None: + """All promote notebook imports are verified at module load time above.""" + assert callable(promote_from_s3) + assert isinstance(DEFAULT_LAKEHOUSE_KEY_PREFIX, str) + assert callable(split_s3_path) diff --git a/tests/ncbi_ftp/test_promote.py b/tests/ncbi_ftp/test_promote.py index fa9898d0..fdb180db 100644 --- a/tests/ncbi_ftp/test_promote.py +++ b/tests/ncbi_ftp/test_promote.py @@ -14,215 +14,182 @@ from tests.ncbi_ftp.conftest import TEST_BUCKET -@pytest.mark.s3 -class TestPromoteFromS3: - """Test promote_from_s3 with moto-mocked S3.""" - - def _stage_files(self, s3_client: botocore.client.BaseClient, prefix: str) -> None: - """Upload sample staged files to mock S3.""" - for key in [ - f"{prefix}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz", - f"{prefix}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz.md5", - f"{prefix}download_report.json", - ]: - body = b"md5hash123" if key.endswith(".md5") else b"data" - s3_client.put_object(Bucket=TEST_BUCKET, Key=key, Body=body) - - def test_dry_run_no_writes(self, mock_s3_client_no_checksum: botocore.client.BaseClient) -> None: - """Verify dry_run does not write any objects.""" - prefix = "staging/run1/" - self._stage_files(mock_s3_client_no_checksum, prefix) - - report = promote_from_s3( - staging_key_prefix=prefix, - bucket=TEST_BUCKET, - dry_run=True, - ) - assert report["promoted"] == 1 - assert report["dry_run"] is True - - # Final path should NOT exist - final_key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz" - resp = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=final_key) - assert resp.get("KeyCount", 0) == 0 +def _stage_files(s3_client: botocore.client.BaseClient, prefix: str) -> None: + """Upload sample staged files to mock S3.""" + for key in [ + f"{prefix}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz", + f"{prefix}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz.md5", + f"{prefix}download_report.json", + ]: + body = b"md5hash123" if key.endswith(".md5") else b"data" + s3_client.put_object(Bucket=TEST_BUCKET, Key=key, Body=body) - def test_promotes_with_metadata(self, mock_s3_client_no_checksum: botocore.client.BaseClient) -> None: - """Verify objects are promoted with MD5 metadata attached.""" - prefix = "staging/run1/" - self._stage_files(mock_s3_client_no_checksum, prefix) - report = promote_from_s3( - staging_key_prefix=prefix, - bucket=TEST_BUCKET, - ) - assert report["promoted"] == 1 - assert report["failed"] == 0 - - # Check final object exists with metadata - final_key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz" - resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=final_key) - assert resp["Metadata"].get("md5") == "md5hash123" +@pytest.mark.s3 +def test_promote_dry_run_no_writes(mock_s3_client_no_checksum: botocore.client.BaseClient) -> None: + """Verify dry_run does not write any objects.""" + prefix = "staging/run1/" + _stage_files(mock_s3_client_no_checksum, prefix) - def test_skips_download_report(self, mock_s3_client_no_checksum: botocore.client.BaseClient) -> None: - """Verify download_report.json is not promoted.""" - prefix = "staging/run1/" - self._stage_files(mock_s3_client_no_checksum, prefix) + report = promote_from_s3(staging_key_prefix=prefix, bucket=TEST_BUCKET, dry_run=True) + assert report["promoted"] == 1 + assert report["dry_run"] is True - report = promote_from_s3(staging_key_prefix=prefix, bucket=TEST_BUCKET) - # Only the .fna.gz data file, not download_report.json - assert report["promoted"] == 1 + final_key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz" + assert mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=final_key).get("KeyCount", 0) == 0 @pytest.mark.s3 -class TestTrimManifest: - """Test _trim_manifest removes promoted accessions from S3 manifest.""" +def test_promote_with_metadata(mock_s3_client_no_checksum: botocore.client.BaseClient) -> None: + """Objects are promoted with MD5 metadata; download_report.json is skipped.""" + prefix = "staging/run1/" + _stage_files(mock_s3_client_no_checksum, prefix) - def test_trims_promoted(self, mock_s3_client_no_checksum: botocore.client.BaseClient) -> None: - """Verify promoted accessions are removed from manifest.""" - manifest_key = "manifests/transfer_manifest.txt" - manifest_body = ( - "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6/\n" - "/genomes/all/GCF/000/001/405/GCF_000001405.40_GRCh38.p14/\n" - ) - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=manifest_key, Body=manifest_body.encode()) - - _trim_manifest(manifest_key, TEST_BUCKET, {"GCF_000001215.4"}) + report = promote_from_s3(staging_key_prefix=prefix, bucket=TEST_BUCKET) + assert report["promoted"] == 1 # only .fna.gz, not download_report.json + assert report["failed"] == 0 - resp = mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=manifest_key) - remaining = resp["Body"].read().decode() - assert "GCF_000001215.4" not in remaining - assert "GCF_000001405.40" in remaining + final_key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz" + resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=final_key) + assert resp["Metadata"].get("md5") == "md5hash123" - def test_trims_all(self, mock_s3_client_no_checksum: botocore.client.BaseClient) -> None: - """Verify all entries can be trimmed leaving an empty manifest.""" - manifest_key = "manifests/transfer_manifest.txt" - manifest_body = "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6/\n" - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=manifest_key, Body=manifest_body.encode()) - _trim_manifest(manifest_key, TEST_BUCKET, {"GCF_000001215.4"}) - - resp = mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=manifest_key) - remaining = resp["Body"].read().decode().strip() - assert remaining == "" +@pytest.mark.s3 +@pytest.mark.parametrize( + ("manifest_body", "promoted_set", "expected_present", "expected_absent"), + [ + pytest.param( + "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6/\n" + "/genomes/all/GCF/000/001/405/GCF_000001405.40_GRCh38.p14/\n", + {"GCF_000001215.4"}, + ["GCF_000001405.40"], + ["GCF_000001215.4"], + id="partial", + ), + pytest.param( + "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6/\n", + {"GCF_000001215.4"}, + [], + ["GCF_000001215.4"], + id="all", + ), + ], +) +def test_trim_manifest( + mock_s3_client_no_checksum: botocore.client.BaseClient, + manifest_body: str, + promoted_set: set[str], + expected_present: list[str], + expected_absent: list[str], +) -> None: + """Promoted accessions are removed; others remain (partial) or the manifest empties (all).""" + manifest_key = "manifests/transfer_manifest.txt" + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=manifest_key, Body=manifest_body.encode()) + _trim_manifest(manifest_key, TEST_BUCKET, promoted_set) + remaining = mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=manifest_key)["Body"].read().decode() + for acc in expected_present: + assert acc in remaining + for acc in expected_absent: + assert acc not in remaining @pytest.mark.s3 -class TestArchiveAssemblies: - """Test _archive_assemblies with moto-mocked S3.""" +def test_archive_assemblies_removed(mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path) -> None: + """Removed accessions are archived and originals deleted.""" + accession = "GCF_000005845.2" + key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/005/845/{accession}_ASM584v2/{accession}_genomic.fna.gz" + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"data") - def test_archives_and_deletes_removed( - self, mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path - ) -> None: - """Verify removed accessions are archived and originals deleted.""" - accession = "GCF_000005845.2" - key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/005/845/{accession}_ASM584v2/{accession}_genomic.fna.gz" - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"data") + manifest = tmp_path / "removed.txt" + manifest.write_text(f"{accession}\n") - manifest = tmp_path / "removed.txt" - manifest.write_text(f"{accession}\n") - - count = _archive_assemblies( + assert ( + _archive_assemblies( str(manifest), bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="replaced_or_suppressed", delete_source=True, ) - assert count == 1 + == 1 + ) + assert mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=key).get("KeyCount", 0) == 0 - # Original should be deleted - resp = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=key) - assert resp.get("KeyCount", 0) == 0 + archive_key = ( + f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/" + f"raw_data/GCF/000/005/845/{accession}_ASM584v2/{accession}_genomic.fna.gz" + ) + assert mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_key).get("KeyCount", 0) == 1 - # Archived copy should exist - archive_key = ( - f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/" - f"raw_data/GCF/000/005/845/{accession}_ASM584v2/{accession}_genomic.fna.gz" - ) - resp = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_key) - assert resp.get("KeyCount", 0) == 1 - - def test_archives_updated_without_deleting( - self, mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path - ) -> None: - """Verify updated accessions are archived but originals remain.""" - accession = "GCF_000001215.4" - asm_dir = f"{accession}_Release_6" - key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"original-data") - - manifest = tmp_path / "updated.txt" - manifest.write_text(f"{accession}\n") - - count = _archive_assemblies( - str(manifest), - bucket=TEST_BUCKET, - ncbi_release="2024-06", - archive_reason="updated", - delete_source=False, + +@pytest.mark.s3 +def test_archive_assemblies_updated_no_delete( + mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path +) -> None: + """Updated accessions are archived but originals remain.""" + accession = "GCF_000001215.4" + asm_dir = f"{accession}_Release_6" + key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"original-data") + + manifest = tmp_path / "updated.txt" + manifest.write_text(f"{accession}\n") + + assert ( + _archive_assemblies( + str(manifest), bucket=TEST_BUCKET, ncbi_release="2024-06", archive_reason="updated", delete_source=False ) - assert count == 1 + == 1 + ) + assert mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=key).get("KeyCount", 0) == 1 - # Original still exists (promote will overwrite it) - resp = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=key) - assert resp.get("KeyCount", 0) == 1 + archive_key = ( + f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-06/raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" + ) + resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=archive_key) + assert resp["Metadata"]["archive_reason"] == "updated" + assert resp["Metadata"]["ncbi_last_release"] == "2024-06" - # Archived copy exists with correct metadata - archive_key = ( - f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-06/" - f"raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" - ) - resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=archive_key) - assert resp["Metadata"]["archive_reason"] == "updated" - assert resp["Metadata"]["ncbi_last_release"] == "2024-06" - - def test_multiple_releases_no_collision( - self, mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path - ) -> None: - """Verify archiving the same accession in different releases creates distinct folders.""" - accession = "GCF_000001215.4" - asm_dir = f"{accession}_Release_6" - key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"v1-data") - - manifest = tmp_path / "updated.txt" - manifest.write_text(f"{accession}\n") - - # First archive: release 2024-01 - _archive_assemblies(str(manifest), bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated") - - # Simulate promote overwriting source - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"v2-data") - - # Second archive: release 2024-06 - _archive_assemblies(str(manifest), bucket=TEST_BUCKET, ncbi_release="2024-06", archive_reason="updated") - - archive_key_1 = ( - f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/" - f"raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" - ) - archive_key_2 = ( - f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-06/" - f"raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" - ) - resp1 = mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=archive_key_1) - assert resp1["Body"].read() == b"v1-data" - resp2 = mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=archive_key_2) - assert resp2["Body"].read() == b"v2-data" +@pytest.mark.s3 +def test_archive_assemblies_multiple_releases_no_collision( + mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path +) -> None: + """Archiving the same accession in different releases creates distinct folders.""" + accession = "GCF_000001215.4" + asm_dir = f"{accession}_Release_6" + key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"v1-data") + + manifest = tmp_path / "updated.txt" + manifest.write_text(f"{accession}\n") + + _archive_assemblies(str(manifest), bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated") + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"v2-data") + _archive_assemblies(str(manifest), bucket=TEST_BUCKET, ncbi_release="2024-06", archive_reason="updated") + + archive_key_1 = ( + f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" + ) + archive_key_2 = ( + f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-06/raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" + ) + assert mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=archive_key_1)["Body"].read() == b"v1-data" + assert mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=archive_key_2)["Body"].read() == b"v2-data" + - def test_dry_run_no_side_effects( - self, mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path - ) -> None: - """Verify dry_run does not copy or delete anything.""" - accession = "GCF_000005845.2" - key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/005/845/{accession}_ASM584v2/{accession}_genomic.fna.gz" - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"data") +@pytest.mark.s3 +def test_archive_assemblies_dry_run(mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path) -> None: + """dry_run does not copy or delete anything.""" + accession = "GCF_000005845.2" + key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/005/845/{accession}_ASM584v2/{accession}_genomic.fna.gz" + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"data") - manifest = tmp_path / "removed.txt" - manifest.write_text(f"{accession}\n") + manifest = tmp_path / "removed.txt" + manifest.write_text(f"{accession}\n") - count = _archive_assemblies( + assert ( + _archive_assemblies( str(manifest), bucket=TEST_BUCKET, ncbi_release="2024-01", @@ -230,45 +197,40 @@ def test_dry_run_no_side_effects( delete_source=True, dry_run=True, ) - assert count == 1 - - # Original still exists - resp = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=key) - assert resp.get("KeyCount", 0) == 1 - - # No archive created - archive_prefix = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/" - resp = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_prefix) - assert resp.get("KeyCount", 0) == 0 - - def test_no_existing_objects_skips( - self, mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path - ) -> None: - """Verify accessions with no existing S3 objects are silently skipped.""" - manifest = tmp_path / "updated.txt" - manifest.write_text("GCF_000001215.4\n") - - count = _archive_assemblies(str(manifest), bucket=TEST_BUCKET, ncbi_release="2024-01") - assert count == 0 - - def test_unknown_release_fallback( - self, mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path - ) -> None: - """Verify ncbi_release=None falls back to 'unknown'.""" - accession = "GCF_000001215.4" - asm_dir = f"{accession}_Release_6" - key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"data") - - manifest = tmp_path / "updated.txt" - manifest.write_text(f"{accession}\n") - - count = _archive_assemblies(str(manifest), bucket=TEST_BUCKET, ncbi_release=None) - assert count == 1 - - archive_key = ( - f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/unknown/" - f"raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" - ) - resp = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_key) - assert resp.get("KeyCount", 0) == 1 + == 1 + ) + assert mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=key).get("KeyCount", 0) == 1 + + archive_prefix = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/" + assert mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_prefix).get("KeyCount", 0) == 0 + + +@pytest.mark.s3 +def test_archive_assemblies_no_objects_skips( + mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path +) -> None: + """Accessions with no existing S3 objects are silently skipped.""" + manifest = tmp_path / "updated.txt" + manifest.write_text("GCF_000001215.4\n") + assert _archive_assemblies(str(manifest), bucket=TEST_BUCKET, ncbi_release="2024-01") == 0 + + +@pytest.mark.s3 +def test_archive_assemblies_unknown_release_fallback( + mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path +) -> None: + """ncbi_release=None falls back to 'unknown' in the archive path.""" + accession = "GCF_000001215.4" + asm_dir = f"{accession}_Release_6" + key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"data") + + manifest = tmp_path / "updated.txt" + manifest.write_text(f"{accession}\n") + + assert _archive_assemblies(str(manifest), bucket=TEST_BUCKET, ncbi_release=None) == 1 + + archive_key = ( + f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/unknown/raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" + ) + assert mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_key).get("KeyCount", 0) == 1 diff --git a/tests/s3_helpers.py b/tests/s3_helpers.py new file mode 100644 index 00000000..cf98c3d5 --- /dev/null +++ b/tests/s3_helpers.py @@ -0,0 +1,25 @@ +"""Shared S3 test helpers. + +# NOTE: Moto currently does not support CRC64NVME; remove this helper when it does. +""" + +import functools +from collections.abc import Callable +from typing import Any + + +def strip_checksum_algorithm(method: Callable[..., Any]) -> Callable[..., Any]: + """Wrap a boto3 S3 method to remove the ChecksumAlgorithm argument before calling moto. + + Moto does not implement CRC64NVME checksums, so any call that includes + ChecksumAlgorithm='CRC64NVME' would fail. This wrapper silently drops the + argument so the rest of the call proceeds normally against the moto backend. + """ + + @functools.wraps(method) + def wrapper(*args: Any, **kwargs: Any) -> Any: + """Remove the ChecksumAlgorithm argument from the call.""" + kwargs.pop("ChecksumAlgorithm", None) + return method(*args, **kwargs) + + return wrapper diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index b459c467..bebba52c 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -1,8 +1,7 @@ """Tests for s3_utils.py using moto to mock AWS S3.""" -import functools import io -from collections.abc import Callable, Generator +from collections.abc import Generator from pathlib import Path from typing import Any from unittest.mock import MagicMock, patch @@ -15,6 +14,7 @@ from requests.exceptions import HTTPError import cdm_data_loaders.utils.s3 as s3_utils +from tests.s3_helpers import strip_checksum_algorithm from cdm_data_loaders.utils.s3 import ( CDM_LAKE_BUCKET, DEFAULT_EXTRA_ARGS, @@ -683,24 +683,6 @@ def test_upload_dir_raises_on_empty_destination(sample_dir: Path) -> None: upload_dir(sample_dir, "") -# NOTE: Moto currently does not support CRC64NVME; remove this helper when it does. -def strip_checksum_algorithm(method: Callable[..., Any]) -> Callable[..., Any]: - """Wrap a boto3 S3 method to remove the ChecksumAlgorithm argument before calling moto. - - Moto does not implement CRC64NVME checksums, so any call that includes - ChecksumAlgorithm='CRC64NVME' would fail. This wrapper silently drops the - argument so the rest of the call proceeds normally against the moto backend. - """ - - @functools.wraps(method) - def wrapper(*args: Any, **kwargs: Any) -> Any: - """Remove the ChecksumAlgorithm argument from the call.""" - kwargs.pop("ChecksumAlgorithm", None) - return method(*args, **kwargs) - - return wrapper - - @pytest.fixture def mocked_s3_client_no_checksum(mock_s3_client: Any) -> Any: """Return the mocked S3 client with copy_object patched to strip ChecksumAlgorithm. From cc1042bd841ec03fd30226b25a5a9c0d7ff47cab Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 27 Apr 2026 16:01:13 -0700 Subject: [PATCH 054/128] Potential fix for pull request finding 'Unused import' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- tests/ncbi_ftp/test_assembly.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/ncbi_ftp/test_assembly.py b/tests/ncbi_ftp/test_assembly.py index 559788bf..261f4676 100644 --- a/tests/ncbi_ftp/test_assembly.py +++ b/tests/ncbi_ftp/test_assembly.py @@ -3,7 +3,6 @@ import pytest from cdm_data_loaders.ncbi_ftp.assembly import ( - FILE_FILTERS, build_accession_path, parse_assembly_path, parse_md5_checksums_file, From 32164035061bca850a6a9d6eb5fff6ff9d9678ca Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Tue, 28 Apr 2026 12:25:03 -0700 Subject: [PATCH 055/128] add notebook option for download phase Co-authored-by: Copilot --- notebooks/ncbi_ftp_download.ipynb | 199 +++++++++++ .../pipelines/ncbi_ftp_download.py | 103 +++++- tests/integration/test_download_e2e.py | 62 +++- tests/ncbi_ftp/test_notebooks.py | 12 + tests/pipelines/test_ncbi_ftp_download.py | 329 +++++++++++++++++- 5 files changed, 701 insertions(+), 4 deletions(-) create mode 100644 notebooks/ncbi_ftp_download.ipynb diff --git a/notebooks/ncbi_ftp_download.ipynb b/notebooks/ncbi_ftp_download.ipynb new file mode 100644 index 00000000..1a2af714 --- /dev/null +++ b/notebooks/ncbi_ftp_download.ipynb @@ -0,0 +1,199 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "79505884", + "metadata": {}, + "source": [ + "# NCBI Assembly Download & Stage (Phase 2)\n", + "\n", + "Downloads NCBI assemblies listed in a transfer manifest from the NCBI FTP server\n", + "and uploads them to an S3 staging prefix for Phase 3 promotion.\n", + "\n", + "**When to use this notebook vs the CTS container:**\n", + "- Use the CTS container (`ncbi_ftp_sync`) for production runs — it has restart/retry\n", + " support and runs in the data-transfer environment.\n", + "- Use this notebook when CTS is unavailable (e.g. local development, debugging, or\n", + " one-off re-downloads of failed assemblies).\n", + "\n", + "Steps:\n", + "1. Configure bucket, manifest source, staging prefix, and thread count\n", + "2. Preview the first 10 manifest lines to verify before committing\n", + "3. Download assemblies from NCBI FTP and upload to staging\n", + "4. Review the download/stage report" + ] + }, + { + "cell_type": "markdown", + "id": "76d92c54", + "metadata": {}, + "source": [ + "## Path formats quick reference\n", + "\n", + "| Suffix in variable name | Format | Example |\n", + "|-------------------------|--------|---------|\n", + "| `_BUCKET` | bucket name only | `cdm-lake` |\n", + "| `_KEY_PREFIX` | S3 key prefix (no scheme/bucket) | `staging/run1/` |\n", + "| `_S3_KEY` | S3 object key (no scheme/bucket) | `staging/run1/input/transfer_manifest.txt` |\n", + "| `_PATH` | local filesystem path | `output/transfer_manifest.txt` |\n", + "\n", + "Staging object: `s3://{STORE_BUCKET}/{STAGING_KEY_PREFIX}raw_data/…/{filename}`\n", + "Report: `s3://{STORE_BUCKET}/{STAGING_KEY_PREFIX}download_report.json`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "38e7aa6d", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Imports and S3 client initialisation.\"\"\"\n", + "\n", + "import json\n", + "\n", + "from cdm_data_loaders.pipelines.ncbi_ftp_download import (\n", + " DEFAULT_STAGING_KEY_PREFIX,\n", + " download_and_stage,\n", + ")\n", + "from cdm_data_loaders.utils.s3 import get_s3_client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34a18261", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Configure parameters.\n", + "\n", + "Provide exactly one of MANIFEST_S3_KEY (read from S3) or MANIFEST_LOCAL_PATH (read from disk).\n", + "Set the other to None.\n", + "\n", + "Disk space note: ensure sufficient free space in the system temp directory before running.\n", + "A rough estimate is ~500 MB per 1000 assemblies; large genomes can exceed 1 GB each.\n", + "Set LIMIT to a small number (e.g. 5) to test the workflow before a full run.\n", + "\"\"\"\n", + "\n", + "# S3 bucket where the manifest lives and where staged files will be written\n", + "# format: bucket name (no s3:// scheme)\n", + "STORE_BUCKET = \"cdm-lake\"\n", + "\n", + "# S3 object key of the transfer manifest written by Phase 1\n", + "# format: S3 object key within STORE_BUCKET (no scheme, no bucket)\n", + "# Set to None to use MANIFEST_LOCAL_PATH instead\n", + "MANIFEST_S3_KEY: str | None = \"staging/run1/input/transfer_manifest.txt\"\n", + "\n", + "# Local path to the transfer manifest (alternative to MANIFEST_S3_KEY)\n", + "# format: local filesystem path\n", + "# Set to None to use MANIFEST_S3_KEY instead\n", + "MANIFEST_LOCAL_PATH: str | None = None\n", + "\n", + "# S3 key prefix for staged output files (must match what Phase 3 expects)\n", + "# format: S3 key prefix within STORE_BUCKET (no scheme, no bucket)\n", + "STAGING_KEY_PREFIX = DEFAULT_STAGING_KEY_PREFIX + \"run1/\"\n", + "\n", + "# Number of parallel download and upload threads\n", + "THREADS = 4\n", + "\n", + "# Limit to first N assemblies (None = process all)\n", + "LIMIT: int | None = None\n", + "\n", + "# Dry-run mode — download locally but skip S3 uploads\n", + "DRY_RUN = True\n", + "\n", + "print(f\"Bucket: {STORE_BUCKET}\")\n", + "print(f\"Manifest S3 key: {MANIFEST_S3_KEY}\")\n", + "print(f\"Manifest local: {MANIFEST_LOCAL_PATH}\")\n", + "print(f\"Staging prefix: {STAGING_KEY_PREFIX}\")\n", + "print(f\"Threads: {THREADS}\")\n", + "print(f\"Limit: {LIMIT}\")\n", + "print(f\"Dry-run: {DRY_RUN}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "69beeaa9", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Preview the first 10 manifest lines before committing to the full run.\"\"\"\n", + "\n", + "if MANIFEST_S3_KEY is not None:\n", + " s3 = get_s3_client()\n", + " response = s3.get_object(Bucket=STORE_BUCKET, Key=MANIFEST_S3_KEY)\n", + " manifest_lines = response[\"Body\"].read().decode().splitlines()\n", + "else:\n", + " with open(MANIFEST_LOCAL_PATH) as f:\n", + " manifest_lines = f.read().splitlines()\n", + "\n", + "data_lines = [l for l in manifest_lines if l.strip() and not l.startswith(\"#\")]\n", + "\n", + "print(f\"Total entries: {len(data_lines)}\")\n", + "print(\"First 10:\")\n", + "for line in data_lines[:10]:\n", + " print(f\" {line}\")\n", + "if len(data_lines) > 10:\n", + " print(f\" ... and {len(data_lines) - 10} more\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9b76d273", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Download assemblies from NCBI FTP and upload to S3 staging.\"\"\"\n", + "\n", + "report = download_and_stage(\n", + " bucket=STORE_BUCKET,\n", + " staging_key_prefix=STAGING_KEY_PREFIX,\n", + " manifest_s3_key=MANIFEST_S3_KEY,\n", + " manifest_local_path=MANIFEST_LOCAL_PATH,\n", + " threads=THREADS,\n", + " limit=LIMIT,\n", + " dry_run=DRY_RUN,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "192b9d34", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Display download and staging report.\"\"\"\n", + "\n", + "print(\"=\" * 50)\n", + "print(\"DOWNLOAD & STAGE REPORT\")\n", + "print(\"=\" * 50)\n", + "print(f\"Attempted: {report['total_attempted']}\")\n", + "print(f\"Succeeded: {report['succeeded']}\")\n", + "print(f\"Failed: {report['failed']}\")\n", + "print(f\"Staged objects: {report['staged_objects']}\")\n", + "print(f\"Staging prefix: {report['staging_key_prefix']}\")\n", + "print(f\"Dry-run: {report['dry_run']}\")\n", + "print(f\"Timestamp: {report['timestamp']}\")\n", + "\n", + "if report[\"failed\"] > 0:\n", + " print(\"\\nFailed assemblies:\")\n", + " for failure in report[\"failures\"]:\n", + " print(f\" {failure['path']}: {failure['error']}\")\n", + "\n", + "if report[\"dry_run\"]:\n", + " print(\"\\nThis was a dry-run. Set DRY_RUN = False and re-run to upload to S3.\")" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py index 20cd77e8..85581a57 100644 --- a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py +++ b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py @@ -7,6 +7,7 @@ import json import logging +import tempfile import threading from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import UTC, datetime @@ -23,11 +24,17 @@ from cdm_data_loaders.pipelines.cts_defaults import DEFAULT_SETTINGS_CONFIG_DICT, INPUT_MOUNT, OUTPUT_MOUNT from cdm_data_loaders.utils.cdm_logger import get_cdm_logger from cdm_data_loaders.utils.ftp_client import ThreadLocalFTP +from cdm_data_loaders.utils.s3 import get_s3_client, upload_file logger = get_cdm_logger() -# ── Settings ───────────────────────────────────────────────────────────── +# ── Constants ──────────────────────────────────────────────────────────── + +DEFAULT_STAGING_KEY_PREFIX = "staging/" + + + class DownloadSettings(BaseSettings): @@ -181,3 +188,97 @@ def run_download(config: DownloadSettings) -> None: def cli() -> None: """CLI entry point for ``ncbi_ftp_sync``.""" run_cli(DownloadSettings, run_download) + + +# ── Notebook / interactive entry point ────────────────────────────────── + + +def download_and_stage( + *, + bucket: str, + staging_key_prefix: str, + manifest_s3_key: str | None = None, + manifest_local_path: str | Path | None = None, + threads: int = 4, + ftp_host: str = FTP_HOST, + limit: int | None = None, + dry_run: bool = False, +) -> dict[str, Any]: + """Download assemblies from NCBI FTP and stage them to S3 (Phase 2). + + Exactly one of *manifest_s3_key* or *manifest_local_path* must be given. + + :param bucket: destination S3 bucket name + :param staging_key_prefix: key prefix inside the bucket (e.g. ``"staging/run1/"``) + :param manifest_s3_key: S3 object key of the transfer manifest within *bucket* + :param manifest_local_path: local path to the transfer manifest file + :param threads: number of parallel download **and** upload threads + :param ftp_host: NCBI FTP hostname + :param limit: optional limit for testing (pass to :func:`download_batch`) + :param dry_run: when ``True``, download but skip all S3 uploads + :return: download report extended with ``staged_objects``, ``staging_key_prefix``, ``dry_run`` + """ + if manifest_s3_key is not None and manifest_local_path is not None: + msg = "Provide exactly one of manifest_s3_key or manifest_local_path, not both" + raise ValueError(msg) + if manifest_s3_key is None and manifest_local_path is None: + msg = "One of manifest_s3_key or manifest_local_path must be provided" + raise ValueError(msg) + + with tempfile.TemporaryDirectory() as _tmpdir: + tmp = Path(_tmpdir) + manifest_dest = tmp / "transfer_manifest.txt" + + if manifest_s3_key is not None: + s3 = get_s3_client() + response = s3.get_object(Bucket=bucket, Key=manifest_s3_key) + manifest_dest.write_bytes(response["Body"].read()) + logger.info("Manifest read from S3: s3://%s/%s", bucket, manifest_s3_key) + else: + manifest_dest.write_bytes(Path(manifest_local_path).read_bytes()) + logger.info("Manifest read from local path: %s", manifest_local_path) + + report = download_batch( + manifest_path=manifest_dest, + output_dir=tmp, + threads=threads, + ftp_host=ftp_host, + limit=limit, + ) + + staged_objects = 0 + + if not dry_run: + raw_data_dir = tmp / "raw_data" + report_json = tmp / "download_report.json" + + upload_tasks: list[tuple[Path, str]] = [] + + if raw_data_dir.exists(): + for local_file in sorted(raw_data_dir.rglob("*")): + if local_file.is_file(): + relative = local_file.relative_to(tmp) + dest_prefix = f"{bucket}/{staging_key_prefix.rstrip('/')}/{relative.parent}" + upload_tasks.append((local_file, dest_prefix)) + + if report_json.exists(): + upload_tasks.append((report_json, f"{bucket}/{staging_key_prefix.rstrip('/')}")) + + def _upload(task: tuple[Path, str]) -> None: + local_file, dest = task + upload_file(local_file, dest) + + with ThreadPoolExecutor(max_workers=threads) as executor: + futures = [executor.submit(_upload, t) for t in upload_tasks] + for future in as_completed(futures): + future.result() + staged_objects += 1 + + logger.info("Staged %d objects to s3://%s/%s", staged_objects, bucket, staging_key_prefix) + + return { + **report, + "staged_objects": staged_objects, + "staging_key_prefix": staging_key_prefix, + "dry_run": dry_run, + } diff --git a/tests/integration/test_download_e2e.py b/tests/integration/test_download_e2e.py index 2126bd81..37456e3e 100644 --- a/tests/integration/test_download_e2e.py +++ b/tests/integration/test_download_e2e.py @@ -10,7 +10,9 @@ import pytest from pathlib import Path +from unittest.mock import patch +import cdm_data_loaders.utils.s3 as s3_utils from cdm_data_loaders.ncbi_ftp.manifest import ( compute_diff, download_assembly_summary, @@ -18,7 +20,7 @@ parse_assembly_summary, write_transfer_manifest, ) -from cdm_data_loaders.pipelines.ncbi_ftp_download import download_batch +from cdm_data_loaders.pipelines.ncbi_ftp_download import download_batch, download_and_stage # Use same stable prefix as manifest tests STABLE_PREFIX = "900" @@ -125,3 +127,61 @@ def test_download_resume(self, tmp_path: Path) -> None: # All original files should still exist files_after_second = set(output_dir.rglob("*")) assert files_after_first.issubset(files_after_second) + + +@pytest.mark.integration +@pytest.mark.slow_test +@pytest.mark.external_request +def test_download_and_stage_e2e( + tmp_path: Path, + minio_s3_client, + test_bucket: str, +) -> None: + """Download one assembly and verify it is staged under the expected S3 prefix.""" + manifest_path, _acc = _manifest_for_one_assembly(tmp_path) + + staging_prefix = "staging/e2e-test/" + + # Seed the manifest in MinIO so download_and_stage can read it from S3 + manifest_s3_key = f"{staging_prefix}input/transfer_manifest.txt" + minio_s3_client.put_object( + Bucket=test_bucket, + Key=manifest_s3_key, + Body=manifest_path.read_bytes(), + ) + + with patch.object(s3_utils, "get_s3_client", return_value=minio_s3_client): + report = download_and_stage( + bucket=test_bucket, + staging_key_prefix=staging_prefix, + manifest_s3_key=manifest_s3_key, + threads=1, + limit=1, + dry_run=False, + ) + + assert report["succeeded"] >= 1 + assert report["failed"] == 0 + assert report["staged_objects"] > 0 + assert report["staging_key_prefix"] == staging_prefix + assert report["dry_run"] is False + + # Verify raw_data/ files and .md5 sidecars are staged + paginator = minio_s3_client.get_paginator("list_objects_v2") + staged_keys = [ + obj["Key"] + for page in paginator.paginate(Bucket=test_bucket, Prefix=f"{staging_prefix}raw_data/") + for obj in page.get("Contents", []) + ] + assert len(staged_keys) > 0, "Expected staged files under raw_data/" + + data_files = [k for k in staged_keys if not k.endswith(".md5")] + md5_files = [k for k in staged_keys if k.endswith(".md5")] + assert len(data_files) > 0, "Expected data files" + assert len(md5_files) > 0, "Expected .md5 sidecar files" + + # Verify download_report.json was also uploaded + report_key = f"{staging_prefix}download_report.json" + resp = minio_s3_client.get_object(Bucket=test_bucket, Key=report_key) + saved_report = json.loads(resp["Body"].read()) + assert saved_report["succeeded"] >= 1 diff --git a/tests/ncbi_ftp/test_notebooks.py b/tests/ncbi_ftp/test_notebooks.py index fc3fa793..345572db 100644 --- a/tests/ncbi_ftp/test_notebooks.py +++ b/tests/ncbi_ftp/test_notebooks.py @@ -29,6 +29,7 @@ NCBI_NOTEBOOKS = [ "ncbi_ftp_manifest.ipynb", "ncbi_ftp_promote.ipynb", + "ncbi_ftp_download.ipynb", ] @@ -72,3 +73,14 @@ def test_promote_notebook_imports() -> None: assert callable(promote_from_s3) assert isinstance(DEFAULT_LAKEHOUSE_KEY_PREFIX, str) assert callable(split_s3_path) + + +def test_download_notebook_imports() -> None: + """All download notebook imports resolve without error.""" + from cdm_data_loaders.pipelines.ncbi_ftp_download import ( # noqa: F401 + DEFAULT_STAGING_KEY_PREFIX, + download_and_stage, + ) + + assert callable(download_and_stage) + assert isinstance(DEFAULT_STAGING_KEY_PREFIX, str) diff --git a/tests/pipelines/test_ncbi_ftp_download.py b/tests/pipelines/test_ncbi_ftp_download.py index 90df299f..86d6f8e5 100644 --- a/tests/pipelines/test_ncbi_ftp_download.py +++ b/tests/pipelines/test_ncbi_ftp_download.py @@ -2,14 +2,22 @@ import json from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch +import boto3 import pytest +from moto import mock_aws from pydantic import ValidationError from cdm_data_loaders.ncbi_ftp.assembly import FTP_HOST from cdm_data_loaders.pipelines.cts_defaults import INPUT_MOUNT, OUTPUT_MOUNT -from cdm_data_loaders.pipelines.ncbi_ftp_download import DownloadSettings, download_batch +from cdm_data_loaders.pipelines.ncbi_ftp_download import ( + DEFAULT_STAGING_KEY_PREFIX, + DownloadSettings, + download_and_stage, + download_batch, +) +from cdm_data_loaders.utils.s3 import reset_s3_client _DEFAULT_THREADS = 4 _CUSTOM_THREADS = 8 @@ -228,3 +236,320 @@ def test_handles_download_failure(self, tmp_path: Path) -> None: assert report["failed"] == 1 assert report["succeeded"] == 0 + + +# ── Helpers shared by download_and_stage tests ─────────────────────────── + +_MANIFEST_CONTENT = ( + "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/\n" + "/genomes/all/GCF/000/001/405/GCF_000001405.40_GRCh38.p14/\n" +) +_TEST_BUCKET = "test-bucket" +_STAGING_PREFIX = "staging/run1/" + +_MOCK_REPORT = { + "timestamp": "2026-01-01T00:00:00+00:00", + "total_attempted": 2, + "succeeded": 2, + "failed": 0, + "failures": [], + "assembly_stats": [], +} + + +def _make_moto_s3(): + """Return a moto-backed S3 client with the test bucket created.""" + client = boto3.client("s3", region_name="us-east-1") + client.create_bucket(Bucket=_TEST_BUCKET) + return client + + +# ── download_and_stage — manifest source ──────────────────────────────── + + +@pytest.mark.parametrize( + ("manifest_s3_key", "use_local"), + [ + pytest.param("staging/input/transfer_manifest.txt", False, id="s3_source"), + pytest.param(None, True, id="local_source"), + ], +) +@mock_aws +def test_download_and_stage_manifest_source( + tmp_path: Path, + manifest_s3_key: str | None, + use_local: bool, +) -> None: + """Manifest lines are passed to download_batch regardless of source (S3 or local).""" + reset_s3_client() + s3 = _make_moto_s3() + + manifest_local: Path | None = None + if manifest_s3_key is not None: + s3.put_object(Bucket=_TEST_BUCKET, Key=manifest_s3_key, Body=_MANIFEST_CONTENT.encode()) + else: + manifest_local = tmp_path / "manifest.txt" + manifest_local.write_text(_MANIFEST_CONTENT) + + captured_content: list[str] = [] + + def _capturing_batch(manifest_path, output_dir, **kwargs): # noqa: ARG001 + captured_content.append(Path(manifest_path).read_text()) + return _MOCK_REPORT + + import cdm_data_loaders.utils.s3 as s3_mod + + with ( + patch.object(s3_mod, "get_s3_client", return_value=s3), + patch.object(s3_mod, "_s3_client", s3), + patch("cdm_data_loaders.pipelines.ncbi_ftp_download.get_s3_client", return_value=s3), + patch("cdm_data_loaders.pipelines.ncbi_ftp_download.download_batch", side_effect=_capturing_batch), + ): + download_and_stage( + bucket=_TEST_BUCKET, + staging_key_prefix=_STAGING_PREFIX, + manifest_s3_key=manifest_s3_key, + manifest_local_path=manifest_local, + dry_run=True, + ) + + assert captured_content == [_MANIFEST_CONTENT] + + reset_s3_client() + + +# ── download_and_stage — exactly one source required ──────────────────── + + +@pytest.mark.parametrize( + ("s3_key", "local_path", "should_raise"), + [ + pytest.param("s3/key", "local/path", True, id="both_provided_raises"), + pytest.param(None, None, True, id="neither_provided_raises"), + pytest.param("s3/key", None, False, id="s3_only_ok"), + pytest.param(None, "local/path", False, id="local_only_ok"), + ], +) +@mock_aws +def test_download_and_stage_exactly_one_source_required( + tmp_path: Path, + s3_key: str | None, + local_path: str | None, + should_raise: bool, +) -> None: + """ValueError is raised when both or neither manifest sources are given.""" + reset_s3_client() + + if should_raise: + with pytest.raises(ValueError, match="manifest"): + download_and_stage( + bucket=_TEST_BUCKET, + staging_key_prefix=_STAGING_PREFIX, + manifest_s3_key=s3_key, + manifest_local_path=local_path, + ) + else: + s3 = _make_moto_s3() + # For s3_only: seed the object; for local_only: create the file + if s3_key is not None: + s3.put_object(Bucket=_TEST_BUCKET, Key=s3_key, Body=_MANIFEST_CONTENT.encode()) + if local_path is not None: + real_local = tmp_path / "manifest.txt" + real_local.write_text(_MANIFEST_CONTENT) + local_path = real_local + + import cdm_data_loaders.utils.s3 as s3_mod + + with ( + patch.object(s3_mod, "get_s3_client", return_value=s3), + patch.object(s3_mod, "_s3_client", s3), + patch("cdm_data_loaders.pipelines.ncbi_ftp_download.get_s3_client", return_value=s3), + patch("cdm_data_loaders.pipelines.ncbi_ftp_download.download_batch", return_value=_MOCK_REPORT), + ): + result = download_and_stage( + bucket=_TEST_BUCKET, + staging_key_prefix=_STAGING_PREFIX, + manifest_s3_key=s3_key, + manifest_local_path=local_path, + dry_run=True, + ) + assert result["succeeded"] == _MOCK_REPORT["succeeded"] + + reset_s3_client() + + +# ── download_and_stage — uploads to staging ────────────────────────────── + + +@mock_aws +def test_download_and_stage_uploads_to_staging(tmp_path: Path) -> None: + """Files produced in raw_data/ and download_report.json are all uploaded to staging.""" + reset_s3_client() + s3 = _make_moto_s3() + + manifest_local = tmp_path / "manifest.txt" + manifest_local.write_text(_MANIFEST_CONTENT) + + assembly_rel = "raw_data/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT" + + def _fake_download_batch(manifest_path, output_dir, **kwargs): # noqa: ARG001 + out = Path(output_dir) + asm_dir = out / assembly_rel + asm_dir.mkdir(parents=True) + (asm_dir / "genomic.fna.gz").write_bytes(b"fasta_data") + (asm_dir / "genomic.fna.gz.md5").write_bytes(b"abc123") + report_path = out / "download_report.json" + report_path.write_text(json.dumps(_MOCK_REPORT)) + return _MOCK_REPORT + + import cdm_data_loaders.utils.s3 as s3_mod + + with ( + patch.object(s3_mod, "get_s3_client", return_value=s3), + patch.object(s3_mod, "_s3_client", s3), + patch("cdm_data_loaders.pipelines.ncbi_ftp_download.get_s3_client", return_value=s3), + patch("cdm_data_loaders.pipelines.ncbi_ftp_download.download_batch", side_effect=_fake_download_batch), + ): + report = download_and_stage( + bucket=_TEST_BUCKET, + staging_key_prefix=_STAGING_PREFIX, + manifest_local_path=manifest_local, + dry_run=False, + threads=1, + ) + + paginator = s3.get_paginator("list_objects_v2") + uploaded_keys = { + obj["Key"] + for page in paginator.paginate(Bucket=_TEST_BUCKET) + for obj in page.get("Contents", []) + } + + expected_keys = { + f"{_STAGING_PREFIX}{assembly_rel}/genomic.fna.gz", + f"{_STAGING_PREFIX}{assembly_rel}/genomic.fna.gz.md5", + f"{_STAGING_PREFIX}download_report.json", + } + assert uploaded_keys == expected_keys + assert report["staged_objects"] == len(expected_keys) + + reset_s3_client() + + +# ── download_and_stage — dry_run skips upload ──────────────────────────── + + +@mock_aws +def test_download_and_stage_dry_run_skips_upload(tmp_path: Path) -> None: + """dry_run=True leaves S3 empty and returns staged_objects=0.""" + reset_s3_client() + s3 = _make_moto_s3() + + manifest_local = tmp_path / "manifest.txt" + manifest_local.write_text(_MANIFEST_CONTENT) + + def _fake_download_batch(manifest_path, output_dir, **kwargs): # noqa: ARG001 + asm_dir = Path(output_dir) / "raw_data/GCF/000/001/215/GCF_000001215.4" + asm_dir.mkdir(parents=True) + (asm_dir / "genomic.fna.gz").write_bytes(b"fasta") + return _MOCK_REPORT + + import cdm_data_loaders.utils.s3 as s3_mod + + with ( + patch.object(s3_mod, "get_s3_client", return_value=s3), + patch.object(s3_mod, "_s3_client", s3), + patch("cdm_data_loaders.pipelines.ncbi_ftp_download.get_s3_client", return_value=s3), + patch("cdm_data_loaders.pipelines.ncbi_ftp_download.download_batch", side_effect=_fake_download_batch), + ): + report = download_and_stage( + bucket=_TEST_BUCKET, + staging_key_prefix=_STAGING_PREFIX, + manifest_local_path=manifest_local, + dry_run=True, + threads=1, + ) + + listed = s3.list_objects_v2(Bucket=_TEST_BUCKET) + assert listed.get("KeyCount", 0) == 0 + assert report["staged_objects"] == 0 + assert report["dry_run"] is True + + reset_s3_client() + + +# ── download_and_stage — limit forwarded ──────────────────────────────── + + +@pytest.mark.parametrize( + "limit", + [ + pytest.param(1, id="limit_1"), + pytest.param(10, id="limit_10"), + ], +) +@mock_aws +def test_download_and_stage_limit_forwarded(tmp_path: Path, limit: int) -> None: + """The limit parameter is forwarded verbatim to download_batch.""" + reset_s3_client() + s3 = _make_moto_s3() + + manifest_local = tmp_path / "manifest.txt" + manifest_local.write_text(_MANIFEST_CONTENT) + + import cdm_data_loaders.utils.s3 as s3_mod + + with ( + patch.object(s3_mod, "get_s3_client", return_value=s3), + patch.object(s3_mod, "_s3_client", s3), + patch("cdm_data_loaders.pipelines.ncbi_ftp_download.get_s3_client", return_value=s3), + patch("cdm_data_loaders.pipelines.ncbi_ftp_download.download_batch", return_value=_MOCK_REPORT) as mock_batch, + ): + download_and_stage( + bucket=_TEST_BUCKET, + staging_key_prefix=_STAGING_PREFIX, + manifest_local_path=manifest_local, + limit=limit, + dry_run=True, + ) + + assert mock_batch.call_args.kwargs["limit"] == limit + + reset_s3_client() + + +# ── download_and_stage — report shape ─────────────────────────────────── + + +@mock_aws +def test_download_and_stage_report_shape(tmp_path: Path) -> None: + """Return value includes all download_batch keys plus staged_objects, staging_key_prefix, dry_run.""" + reset_s3_client() + s3 = _make_moto_s3() + + manifest_local = tmp_path / "manifest.txt" + manifest_local.write_text(_MANIFEST_CONTENT) + + import cdm_data_loaders.utils.s3 as s3_mod + + with ( + patch.object(s3_mod, "get_s3_client", return_value=s3), + patch.object(s3_mod, "_s3_client", s3), + patch("cdm_data_loaders.pipelines.ncbi_ftp_download.get_s3_client", return_value=s3), + patch("cdm_data_loaders.pipelines.ncbi_ftp_download.download_batch", return_value=_MOCK_REPORT), + ): + report = download_and_stage( + bucket=_TEST_BUCKET, + staging_key_prefix=_STAGING_PREFIX, + manifest_local_path=manifest_local, + dry_run=True, + ) + + assert report == { + **_MOCK_REPORT, + "staged_objects": 0, + "staging_key_prefix": _STAGING_PREFIX, + "dry_run": True, + } + + reset_s3_client() From 395265571e53c87341efa3755d80430b15f545ff Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Tue, 28 Apr 2026 12:25:19 -0700 Subject: [PATCH 056/128] formatting --- src/cdm_data_loaders/pipelines/ncbi_ftp_download.py | 3 --- tests/pipelines/test_ncbi_ftp_download.py | 6 +----- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py index 85581a57..233c82b1 100644 --- a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py +++ b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py @@ -34,9 +34,6 @@ DEFAULT_STAGING_KEY_PREFIX = "staging/" - - - class DownloadSettings(BaseSettings): """Configuration for the NCBI FTP assembly download pipeline.""" diff --git a/tests/pipelines/test_ncbi_ftp_download.py b/tests/pipelines/test_ncbi_ftp_download.py index 86d6f8e5..f5046724 100644 --- a/tests/pipelines/test_ncbi_ftp_download.py +++ b/tests/pipelines/test_ncbi_ftp_download.py @@ -419,11 +419,7 @@ def _fake_download_batch(manifest_path, output_dir, **kwargs): # noqa: ARG001 ) paginator = s3.get_paginator("list_objects_v2") - uploaded_keys = { - obj["Key"] - for page in paginator.paginate(Bucket=_TEST_BUCKET) - for obj in page.get("Contents", []) - } + uploaded_keys = {obj["Key"] for page in paginator.paginate(Bucket=_TEST_BUCKET) for obj in page.get("Contents", [])} expected_keys = { f"{_STAGING_PREFIX}{assembly_rel}/genomic.fna.gz", From 991b40e50fdb0ac262f419384220a9bbb60126ee Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Tue, 28 Apr 2026 12:28:00 -0700 Subject: [PATCH 057/128] Potential fix for pull request finding 'Unused import' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- tests/pipelines/test_ncbi_ftp_download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipelines/test_ncbi_ftp_download.py b/tests/pipelines/test_ncbi_ftp_download.py index f5046724..e56430c6 100644 --- a/tests/pipelines/test_ncbi_ftp_download.py +++ b/tests/pipelines/test_ncbi_ftp_download.py @@ -2,7 +2,7 @@ import json from pathlib import Path -from unittest.mock import MagicMock, call, patch +from unittest.mock import MagicMock, patch import boto3 import pytest From 359901f5d7058c3e7e3786c8d0d24c6d422a47b6 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Tue, 28 Apr 2026 12:28:18 -0700 Subject: [PATCH 058/128] Potential fix for pull request finding 'Unused import' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- tests/pipelines/test_ncbi_ftp_download.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/pipelines/test_ncbi_ftp_download.py b/tests/pipelines/test_ncbi_ftp_download.py index e56430c6..cadfe559 100644 --- a/tests/pipelines/test_ncbi_ftp_download.py +++ b/tests/pipelines/test_ncbi_ftp_download.py @@ -12,7 +12,6 @@ from cdm_data_loaders.ncbi_ftp.assembly import FTP_HOST from cdm_data_loaders.pipelines.cts_defaults import INPUT_MOUNT, OUTPUT_MOUNT from cdm_data_loaders.pipelines.ncbi_ftp_download import ( - DEFAULT_STAGING_KEY_PREFIX, DownloadSettings, download_and_stage, download_batch, From 51a3680a98cb73b29ec1ef855ee26707973d8417 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 29 Apr 2026 09:51:29 -0700 Subject: [PATCH 059/128] update notebooks --- notebooks/ncbi_ftp_download.ipynb | 41 ++++++++++++++++-- notebooks/ncbi_ftp_manifest.ipynb | 69 ++++++++++--------------------- notebooks/ncbi_ftp_promote.ipynb | 20 +++++++++ 3 files changed, 78 insertions(+), 52 deletions(-) diff --git a/notebooks/ncbi_ftp_download.ipynb b/notebooks/ncbi_ftp_download.ipynb index 1a2af714..f2d25938 100644 --- a/notebooks/ncbi_ftp_download.ipynb +++ b/notebooks/ncbi_ftp_download.ipynb @@ -55,8 +55,7 @@ "from cdm_data_loaders.pipelines.ncbi_ftp_download import (\n", " DEFAULT_STAGING_KEY_PREFIX,\n", " download_and_stage,\n", - ")\n", - "from cdm_data_loaders.utils.s3 import get_s3_client" + ")" ] }, { @@ -101,7 +100,7 @@ "LIMIT: int | None = None\n", "\n", "# Dry-run mode — download locally but skip S3 uploads\n", - "DRY_RUN = True\n", + "DRY_RUN = False\n", "\n", "print(f\"Bucket: {STORE_BUCKET}\")\n", "print(f\"Manifest S3 key: {MANIFEST_S3_KEY}\")\n", @@ -112,6 +111,26 @@ "print(f\"Dry-run: {DRY_RUN}\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "51a857f9", + "metadata": {}, + "outputs": [], + "source": [ + "from cdm_data_loaders.utils.s3 import get_s3_client, reset_s3_client\n", + "\n", + "# Provide S3 credentials (use for local testing against MinIO test container)\n", + "PROVIDE_CREDENTIALS = False # Set to False to rely on environment credentials (e.g. IAM role)\n", + "if PROVIDE_CREDENTIALS:\n", + " reset_s3_client() # Clear any existing client to ensure new credentials are used\n", + " get_s3_client({\n", + " \"endpoint_url\": \"http://localhost:9000\",\n", + " \"aws_access_key_id\": \"minioadmin\",\n", + " \"aws_secret_access_key\": \"minioadmin\",\n", + " })" + ] + }, { "cell_type": "code", "execution_count": null, @@ -190,8 +209,22 @@ } ], "metadata": { + "kernelspec": { + "display_name": "cdm-data-loaders (3.13.11)", + "language": "python", + "name": "python3" + }, "language_info": { - "name": "python" + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.11" } }, "nbformat": 4, diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb index b4e9af5e..8083c6cb 100644 --- a/notebooks/ncbi_ftp_manifest.ipynb +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -66,6 +66,26 @@ "from cdm_data_loaders.utils.s3 import get_s3_client, split_s3_path" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "b196d5a3", + "metadata": {}, + "outputs": [], + "source": [ + "from cdm_data_loaders.utils.s3 import get_s3_client, reset_s3_client\n", + "\n", + "# Provide S3 credentials (use for local testing against MinIO test container)\n", + "PROVIDE_CREDENTIALS = False # Set to False to rely on environment credentials (e.g. IAM role)\n", + "if PROVIDE_CREDENTIALS:\n", + " reset_s3_client() # Clear any existing client to ensure new credentials are used\n", + " get_s3_client({\n", + " \"endpoint_url\": \"http://localhost:9000\",\n", + " \"aws_access_key_id\": \"minioadmin\",\n", + " \"aws_secret_access_key\": \"minioadmin\",\n", + " })" + ] + }, { "cell_type": "code", "execution_count": null, @@ -91,7 +111,7 @@ "\n", "# S3 location where the new snapshot will be uploaded after diffing\n", "# format: s3:// URI\n", - "SNAPSHOT_UPLOAD_URI: str | None = None\n", + "SNAPSHOT_UPLOAD_URI: str | None = \"s3://cdm-lake/tenant-general-warehouse/kbase/datasets/ncbi/assembly_summary_refseq.txt\"\n", "\n", "# Verify candidates against the S3 Lakehouse — prune assemblies already present.\n", "# Set STORE_BUCKET to your bucket name to enable, or None to skip.\n", @@ -113,53 +133,6 @@ "print(f\"Output dir: {OUTPUT_DIR}\")" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "be1fcf1c", - "metadata": {}, - "outputs": [], - "source": [ - "\"\"\"Validate S3 connectivity and bucket/prefix configuration.\"\"\"\n", - "\n", - "import boto3\n", - "from botocore.exceptions import ClientError, NoCredentialsError\n", - "\n", - "s3 = boto3.client(\"s3\")\n", - "\n", - "# Check credentials are present\n", - "try:\n", - " sts = boto3.client(\"sts\")\n", - " identity = sts.get_caller_identity()\n", - " print(f\"✓ Credentials valid — account: {identity['Account']}, arn: {identity['Arn']}\")\n", - "except NoCredentialsError:\n", - " print(\"✗ No AWS credentials found\")\n", - " raise\n", - "except ClientError as e:\n", - " if e.response[\"Error\"][\"Code\"] == \"InvalidParameterValue\":\n", - " print(\"✓ Credentials present (STS GetCallerIdentity not supported on this endpoint — skipping identity check)\")\n", - " else:\n", - " print(f\"✗ Credential check failed: {e}\")\n", - " raise\n", - "\n", - "# Check bucket access and prefix in one step — list_objects_v2 requires only\n", - "# s3:ListBucket on the prefix, which is less restrictive than HeadBucket.\n", - "_check_bucket = STORE_BUCKET if \"STORE_BUCKET\" in dir() and STORE_BUCKET else \"cdm-lake\"\n", - "_check_prefix = STORE_KEY_PREFIX if \"STORE_KEY_PREFIX\" in dir() else \"tenant-general-warehouse/kbase/datasets/ncbi/\"\n", - "try:\n", - " resp = s3.list_objects_v2(Bucket=_check_bucket, Prefix=_check_prefix, MaxKeys=1)\n", - " if resp.get(\"KeyCount\", 0) > 0:\n", - " print(f\"✓ Bucket accessible and prefix has objects: s3://{_check_bucket}/{_check_prefix}\")\n", - " else:\n", - " print(\n", - " f\"✓ Bucket accessible but no objects found under s3://{_check_bucket}/{_check_prefix} — check STORE_KEY_PREFIX\"\n", - " )\n", - "except ClientError as e:\n", - " code = e.response[\"Error\"][\"Code\"]\n", - " print(f\"✗ S3 access check failed (HTTP {code}): {e}\")\n", - " raise" - ] - }, { "cell_type": "code", "execution_count": null, diff --git a/notebooks/ncbi_ftp_promote.ipynb b/notebooks/ncbi_ftp_promote.ipynb index a1aebcdc..a23fc149 100644 --- a/notebooks/ncbi_ftp_promote.ipynb +++ b/notebooks/ncbi_ftp_promote.ipynb @@ -110,6 +110,26 @@ "print(f\"Dry-run: {DRY_RUN}\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "ccfd88d9", + "metadata": {}, + "outputs": [], + "source": [ + "from cdm_data_loaders.utils.s3 import get_s3_client, reset_s3_client\n", + "\n", + "# Provide S3 credentials (use for local testing against MinIO test container)\n", + "PROVIDE_CREDENTIALS = False # Set to False to rely on environment credentials (e.g. IAM role)\n", + "if PROVIDE_CREDENTIALS:\n", + " reset_s3_client() # Clear any existing client to ensure new credentials are used\n", + " get_s3_client({\n", + " \"endpoint_url\": \"http://localhost:9000\",\n", + " \"aws_access_key_id\": \"minioadmin\",\n", + " \"aws_secret_access_key\": \"minioadmin\",\n", + " })" + ] + }, { "cell_type": "code", "execution_count": null, From 8a411641c091e1c1162c8b4098163107190ab258 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 29 Apr 2026 11:19:36 -0700 Subject: [PATCH 060/128] allow separate staging and destination buckets --- docs/ncbi_ftp_e2e_walkthrough.md | 30 +++--- notebooks/ncbi_ftp_download.ipynb | 22 ++--- notebooks/ncbi_ftp_manifest.ipynb | 34 +++---- notebooks/ncbi_ftp_promote.ipynb | 46 +++++----- src/cdm_data_loaders/ncbi_ftp/promote.py | 52 ++++++----- tests/integration/conftest.py | 30 ++++++ tests/integration/test_full_pipeline.py | 27 +++--- tests/integration/test_promote_e2e.py | 111 +++++++++++++---------- tests/ncbi_ftp/test_promote.py | 18 ++-- 9 files changed, 213 insertions(+), 157 deletions(-) diff --git a/docs/ncbi_ftp_e2e_walkthrough.md b/docs/ncbi_ftp_e2e_walkthrough.md index 4a710046..83eb3e1e 100644 --- a/docs/ncbi_ftp_e2e_walkthrough.md +++ b/docs/ncbi_ftp_e2e_walkthrough.md @@ -49,8 +49,8 @@ Understanding this decomposition is the key to configuring the notebooks. ### Lakehouse object (final location) ``` -s3://{STORE_BUCKET}/{LAKEHOUSE_KEY_PREFIX}raw_data/{GCF|GCA}/{nnn}/{nnn}/{nnn}/{assembly_dir}/{filename} - └── bucket ──┘ └── key prefix ──────┘└── build_accession_path() ────────────────────────┘ +s3://{LAKEHOUSE_BUCKET}/{LAKEHOUSE_KEY_PREFIX}raw_data/{GCF|GCA}/{nnn}/{nnn}/{nnn}/{assembly_dir}/{filename} + └── bucket ─────┘ └── key prefix ──────┘└── build_accession_path() ────────────────────────┘ ``` Example: @@ -61,8 +61,8 @@ s3://cdm-lake/tenant-general-warehouse/kbase/datasets/ncbi/raw_data/GCF/900/000/ ### Staging object (Phase 2 output) ``` -s3://{STORE_BUCKET}/{STAGING_KEY_PREFIX}raw_data/{GCF|GCA}/{nnn}/{nnn}/{nnn}/{assembly_dir}/{filename} - └── bucket ──┘ └── key prefix ────┘└── build_accession_path() ────────────────────────┘ +s3://{STAGING_BUCKET}/{STAGING_KEY_PREFIX}raw_data/{GCF|GCA}/{nnn}/{nnn}/{nnn}/{assembly_dir}/{filename} + └── bucket ─────┘ └── key prefix ────┘└── build_accession_path() ────────────────────────┘ ``` ### Local output (Phase 1) @@ -102,6 +102,7 @@ included `scripts/s3_local.py` helper (requires no extra installs — only ```sh uv run python scripts/s3_local.py mb s3://cdm-lake +uv run python scripts/s3_local.py mb s3://cts ``` ### Lakehouse @@ -161,13 +162,13 @@ Open `notebooks/ncbi_ftp_manifest.ipynb` in JupyterLab or VS Code. | `LIMIT` | `10` | int | cap to 10 assemblies | | `PREVIOUS_SUMMARY_URI` | `None` | s3:// URI | first run — everything is "new" | | `SNAPSHOT_UPLOAD_URI` | `None` | s3:// URI | skip S3 upload for local testing | -| `STORE_BUCKET` | `"cdm-lake"` (or `None`) | bucket name | set to prune assemblies already in the Lakehouse | +| `LAKEHOUSE_BUCKET` | `"cdm-lake"` (or `None`) | bucket name | set to prune assemblies already in the Lakehouse | | `STORE_KEY_PREFIX` | `"tenant-general-warehouse/kbase/datasets/ncbi/"` | S3 key prefix | default Lakehouse path prefix | | `OUTPUT_DIR` | `Path("output")` | local path | keep as-is (local directory) | ### Initialise the S3 client for MinIO -If you set `PREVIOUS_SUMMARY_URI`, `SNAPSHOT_UPLOAD_URI`, `STORE_BUCKET`, +If you set `PREVIOUS_SUMMARY_URI`, `SNAPSHOT_UPLOAD_URI`, `LAKEHOUSE_BUCKET`, or `STAGING_URI` to point at your local MinIO, you must initialise the S3 client **before** running the cells that use them. Insert a new cell after Cell 1 (Imports) with: @@ -184,7 +185,7 @@ get_s3_client({ ``` If all three S3 variables are `None` (purely local testing), this cell can -be skipped — though on repeat runs you should set `STORE_BUCKET` so +be skipped — though on repeat runs you should set `LAKEHOUSE_BUCKET` so assemblies already promoted to the Lakehouse are pruned from the transfer manifest. @@ -202,7 +203,7 @@ checksums would take days. **How it works:** 1. Set `SCAN_STORE = True` in Cell 5 -2. The notebook scans all objects under `s3://{STORE_BUCKET}/{STORE_KEY_PREFIX}` +2. The notebook scans all objects under `s3://{LAKEHOUSE_BUCKET}/{STORE_KEY_PREFIX}` 3. For each unique assembly found, it extracts the accession and uses the earliest object `LastModified` as a conservative `seq_rel_date` 4. It saves the synthetic summary to `LOCAL_SYNTHETIC_SUMMARY` (default: @@ -213,7 +214,7 @@ checksums would take days. **Example (for a 500K-assembly store):** ```python SCAN_STORE = True -STORE_BUCKET = "cdm-lake" +LAKEHOUSE_BUCKET = "cdm-lake" STORE_KEY_PREFIX = "tenant-general-warehouse/kbase/datasets/ncbi/" LOCAL_SYNTHETIC_SUMMARY = Path("output/synthetic_summary_from_store.txt") @@ -350,13 +351,13 @@ The download step writes to the local filesystem. To feed Phase 3 we need to upload the staged files into MinIO under a staging prefix: ```sh -uv run python scripts/s3_local.py cp notebooks/staging/raw_data/ s3://cdm-lake/staging/run1/raw_data/ +uv run python scripts/s3_local.py cp notebooks/staging/raw_data/ s3://cts/staging/run1/raw_data/ ``` Verify the upload: ```sh -uv run python scripts/s3_local.py ls s3://cdm-lake/staging/run1/ +uv run python scripts/s3_local.py ls s3://cts/staging/run1/ ``` --- @@ -369,7 +370,8 @@ Open `notebooks/ncbi_ftp_promote.ipynb`. | Constant | Walkthrough value | Format | Why | |-------------------------|------------------------------------------------------|--------|---------------------------------------------| -| `STORE_BUCKET` | `"cdm-lake"` | bucket name | matches the bucket created in Step 1 | +| `STAGING_BUCKET` | `"cts"` | bucket name | CTS staging bucket (Phase 2 writes here) | +| `LAKEHOUSE_BUCKET` | `"cdm-lake"` | bucket name | final Lakehouse destination | | `STAGING_KEY_PREFIX` | `"staging/run1/"` | S3 key prefix | matches the upload prefix from Step 3d | | `REMOVED_MANIFEST_PATH` | `None` | local path | nothing to remove on first run | | `UPDATED_MANIFEST_PATH` | `None` | local path | nothing to archive on first run | @@ -436,7 +438,7 @@ uv run python scripts/s3_local.py head \ Each promoted assembly gets a [frictionless](https://framework.frictionlessdata.io/) data package descriptor stored at: ``` -s3://{STORE_BUCKET}/{LAKEHOUSE_KEY_PREFIX}metadata/{assembly_dir}_datapackage.json +s3://{LAKEHOUSE_BUCKET}/{LAKEHOUSE_KEY_PREFIX}metadata/{assembly_dir}_datapackage.json ``` For example: @@ -458,7 +460,7 @@ When an assembly is archived (updated or removed), its live descriptor is copied to: ``` -s3://{STORE_BUCKET}/{LAKEHOUSE_KEY_PREFIX}archive/{release_tag}/metadata/{assembly_dir}_datapackage.json +s3://{LAKEHOUSE_BUCKET}/{LAKEHOUSE_KEY_PREFIX}archive/{release_tag}/metadata/{assembly_dir}_datapackage.json ``` Use the last cell of `notebooks/ncbi_ftp_promote.ipynb` to list and preview diff --git a/notebooks/ncbi_ftp_download.ipynb b/notebooks/ncbi_ftp_download.ipynb index f2d25938..84ac4fbf 100644 --- a/notebooks/ncbi_ftp_download.ipynb +++ b/notebooks/ncbi_ftp_download.ipynb @@ -32,13 +32,13 @@ "\n", "| Suffix in variable name | Format | Example |\n", "|-------------------------|--------|---------|\n", - "| `_BUCKET` | bucket name only | `cdm-lake` |\n", + "| `_BUCKET` | bucket name only | `cts` |\n", "| `_KEY_PREFIX` | S3 key prefix (no scheme/bucket) | `staging/run1/` |\n", "| `_S3_KEY` | S3 object key (no scheme/bucket) | `staging/run1/input/transfer_manifest.txt` |\n", "| `_PATH` | local filesystem path | `output/transfer_manifest.txt` |\n", "\n", - "Staging object: `s3://{STORE_BUCKET}/{STAGING_KEY_PREFIX}raw_data/…/{filename}`\n", - "Report: `s3://{STORE_BUCKET}/{STAGING_KEY_PREFIX}download_report.json`" + "Staging object: `s3://{STAGING_BUCKET}/{STAGING_KEY_PREFIX}raw_data/…/{filename}`\n", + "Report: `s3://{STAGING_BUCKET}/{STAGING_KEY_PREFIX}download_report.json`" ] }, { @@ -77,12 +77,12 @@ "\n", "# S3 bucket where the manifest lives and where staged files will be written\n", "# format: bucket name (no s3:// scheme)\n", - "STORE_BUCKET = \"cdm-lake\"\n", + "STAGING_BUCKET = \"cts\"\n", "\n", "# S3 object key of the transfer manifest written by Phase 1\n", - "# format: S3 object key within STORE_BUCKET (no scheme, no bucket)\n", + "# format: S3 object key within STAGING_BUCKET (no scheme, no bucket)\n", "# Set to None to use MANIFEST_LOCAL_PATH instead\n", - "MANIFEST_S3_KEY: str | None = \"staging/run1/input/transfer_manifest.txt\"\n", + "MANIFEST_S3_KEY: str | None = \"io/matt-cohere/staging/run1/input/transfer_manifest.txt\"\n", "\n", "# Local path to the transfer manifest (alternative to MANIFEST_S3_KEY)\n", "# format: local filesystem path\n", @@ -90,8 +90,8 @@ "MANIFEST_LOCAL_PATH: str | None = None\n", "\n", "# S3 key prefix for staged output files (must match what Phase 3 expects)\n", - "# format: S3 key prefix within STORE_BUCKET (no scheme, no bucket)\n", - "STAGING_KEY_PREFIX = DEFAULT_STAGING_KEY_PREFIX + \"run1/\"\n", + "# format: S3 key prefix within STAGING_BUCKET (no scheme, no bucket)\n", + "STAGING_KEY_PREFIX = \"io/matt-cohere/staging/run1/output/\"\n", "\n", "# Number of parallel download and upload threads\n", "THREADS = 4\n", @@ -102,7 +102,7 @@ "# Dry-run mode — download locally but skip S3 uploads\n", "DRY_RUN = False\n", "\n", - "print(f\"Bucket: {STORE_BUCKET}\")\n", + "print(f\"Bucket: {STAGING_BUCKET}\")\n", "print(f\"Manifest S3 key: {MANIFEST_S3_KEY}\")\n", "print(f\"Manifest local: {MANIFEST_LOCAL_PATH}\")\n", "print(f\"Staging prefix: {STAGING_KEY_PREFIX}\")\n", @@ -142,7 +142,7 @@ "\n", "if MANIFEST_S3_KEY is not None:\n", " s3 = get_s3_client()\n", - " response = s3.get_object(Bucket=STORE_BUCKET, Key=MANIFEST_S3_KEY)\n", + " response = s3.get_object(Bucket=STAGING_BUCKET, Key=MANIFEST_S3_KEY)\n", " manifest_lines = response[\"Body\"].read().decode().splitlines()\n", "else:\n", " with open(MANIFEST_LOCAL_PATH) as f:\n", @@ -168,7 +168,7 @@ "\"\"\"Download assemblies from NCBI FTP and upload to S3 staging.\"\"\"\n", "\n", "report = download_and_stage(\n", - " bucket=STORE_BUCKET,\n", + " bucket=STAGING_BUCKET,\n", " staging_key_prefix=STAGING_KEY_PREFIX,\n", " manifest_s3_key=MANIFEST_S3_KEY,\n", " manifest_local_path=MANIFEST_LOCAL_PATH,\n", diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb index 8083c6cb..0ade0917 100644 --- a/notebooks/ncbi_ftp_manifest.ipynb +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -17,7 +17,7 @@ "All filtering (prefix range, limit) is applied here so downstream phases\n", "receive a final, pre-filtered manifest.\n", "\n", - "Optionally verifies candidates against the S3 Lakehouse (`STORE_BUCKET`) so\n", + "Optionally verifies candidates against the S3 Lakehouse (`LAKEHOUSE_BUCKET`) so\n", "assemblies that were already downloaded and promoted are pruned from the\n", "transfer manifest." ] @@ -36,7 +36,7 @@ "| `_KEY_PREFIX` | S3 key prefix (no scheme/bucket) | `tenant-general-warehouse/kbase/datasets/ncbi/` |\n", "| `_DIR` / `_PATH` | local filesystem path | `output/removed_manifest.txt` |\n", "\n", - "Lakehouse object: `s3://{STORE_BUCKET}/{STORE_KEY_PREFIX}raw_data/…/{filename}`" + "Lakehouse object: `s3://{LAKEHOUSE_BUCKET}/{STORE_KEY_PREFIX}raw_data/…/{filename}`" ] }, { @@ -114,11 +114,11 @@ "SNAPSHOT_UPLOAD_URI: str | None = \"s3://cdm-lake/tenant-general-warehouse/kbase/datasets/ncbi/assembly_summary_refseq.txt\"\n", "\n", "# Verify candidates against the S3 Lakehouse — prune assemblies already present.\n", - "# Set STORE_BUCKET to your bucket name to enable, or None to skip.\n", + "# Set LAKEHOUSE_BUCKET to your bucket name to enable, or None to skip.\n", "# STORE_KEY_PREFIX should point to the directory containing `raw_data/`.\n", "# format: bucket name (no s3:// scheme)\n", - "STORE_BUCKET: str | None = \"cdm-lake\"\n", - "# format: S3 key prefix within STORE_BUCKET (no scheme, no bucket)\n", + "LAKEHOUSE_BUCKET: str | None = \"cdm-lake\"\n", + "# format: S3 key prefix within LAKEHOUSE_BUCKET (no scheme, no bucket)\n", "STORE_KEY_PREFIX = \"tenant-general-warehouse/kbase/datasets/ncbi/\"\n", "\n", "# Local output directory for manifest files\n", @@ -129,7 +129,7 @@ "print(f\"Database: {DATABASE}\")\n", "print(f\"Prefix range: {PREFIX_FROM} -> {PREFIX_TO}\")\n", "print(f\"Limit: {LIMIT}\")\n", - "print(f\"Verify against S3: {STORE_BUCKET or 'disabled'}\")\n", + "print(f\"Verify against S3: {LAKEHOUSE_BUCKET or 'disabled'}\")\n", "print(f\"Output dir: {OUTPUT_DIR}\")" ] }, @@ -161,7 +161,7 @@ "for the diff.\n", "\n", "Set SCAN_STORE=True below to enable. The scan will:\n", - " 1. List all objects under STORE_BUCKET/STORE_KEY_PREFIX\n", + " 1. List all objects under LAKEHOUSE_BUCKET/STORE_KEY_PREFIX\n", " 2. Extract accessions matching the DATABASE type (GCF_ for refseq, GCA_ for genbank)\n", " 3. Apply user-provided SYNTHETIC_RELEASE_DATE to all records\n", " 4. Build AssemblyRecord for each assembly found\n", @@ -177,7 +177,7 @@ "SYNTHETIC_RELEASE_DATE = \"2025/10/31\" # YYYY/MM/DD applied to all synthetic records\n", "LOCAL_SYNTHETIC_SUMMARY = Path(\"output/synthetic_summary_from_store.txt\")\n", "\n", - "if SCAN_STORE and STORE_BUCKET:\n", + "if SCAN_STORE and LAKEHOUSE_BUCKET:\n", " if LOCAL_SYNTHETIC_SUMMARY.exists() and not FORCE_RESCAN:\n", " print(f\"Loading existing synthetic summary from {LOCAL_SYNTHETIC_SUMMARY} (set FORCE_RESCAN=True to rescan)\")\n", " previous = parse_assembly_summary(LOCAL_SYNTHETIC_SUMMARY)\n", @@ -187,7 +187,7 @@ " from cdm_data_loaders.ncbi_ftp.manifest import scan_store_to_synthetic_summary\n", " from tqdm.notebook import tqdm\n", "\n", - " print(f\"Scanning s3://{STORE_BUCKET}/{STORE_KEY_PREFIX} for existing {DATABASE} assemblies ...\")\n", + " print(f\"Scanning s3://{LAKEHOUSE_BUCKET}/{STORE_KEY_PREFIX} for existing {DATABASE} assemblies ...\")\n", " print(\"Note: large stores (500K+ assemblies) may take 15-30+ minutes.\")\n", " progress = tqdm(unit=\"assembly\", desc=\"Scanning store\", leave=True, mininterval=2.0)\n", "\n", @@ -203,7 +203,7 @@ " _last_refresh = now\n", "\n", " synthetic = scan_store_to_synthetic_summary(\n", - " STORE_BUCKET,\n", + " LAKEHOUSE_BUCKET,\n", " STORE_KEY_PREFIX,\n", " SYNTHETIC_RELEASE_DATE,\n", " database=DATABASE,\n", @@ -228,7 +228,7 @@ " previous = synthetic\n", "else:\n", " if SCAN_STORE:\n", - " print(\"SCAN_STORE=True but STORE_BUCKET not set. Skipping.\")\n", + " print(\"SCAN_STORE=True but LAKEHOUSE_BUCKET not set. Skipping.\")\n", " print(\"Skipping store scan (SCAN_STORE=False). Will load/use PREVIOUS_SUMMARY_URI instead.\")" ] }, @@ -304,13 +304,13 @@ "\"\"\"\n", "\n", "# -- Verify against Lakehouse --\n", - "if STORE_BUCKET:\n", + "if LAKEHOUSE_BUCKET:\n", " from cdm_data_loaders.ncbi_ftp.manifest import verify_transfer_candidates\n", " from tqdm.notebook import tqdm\n", "\n", " candidates = diff.new + diff.updated\n", " total = len(candidates)\n", - " print(f\"Verifying {total} candidates against s3://{STORE_BUCKET}/{STORE_KEY_PREFIX} ...\")\n", + " print(f\"Verifying {total} candidates against s3://{LAKEHOUSE_BUCKET}/{STORE_KEY_PREFIX} ...\")\n", "\n", " progress = tqdm(total=total, unit=\"assembly\", desc=\"Verifying checksums\", leave=True)\n", "\n", @@ -327,7 +327,7 @@ " verify_transfer_candidates(\n", " candidates,\n", " filtered,\n", - " STORE_BUCKET,\n", + " LAKEHOUSE_BUCKET,\n", " STORE_KEY_PREFIX,\n", " ftp_host=FTP_HOST,\n", " progress_callback=_update_progress,\n", @@ -342,7 +342,7 @@ " after = len(diff.new) + len(diff.updated)\n", " print(f\"Verified: {after} need downloading, {before - after} pruned (already in store)\")\n", "else:\n", - " print(\"Skipping S3 verification (STORE_BUCKET not set)\")\n", + " print(\"Skipping S3 verification (LAKEHOUSE_BUCKET not set)\")\n", "\n", "# -- Apply LIMIT --\n", "if LIMIT is not None:\n", @@ -407,7 +407,7 @@ "\"\"\"Upload manifests to S3 for CTS input staging (optional).\n", "\n", "Note: STAGING_URI is a full s3:// URI. The promote notebook splits this into\n", - "STORE_BUCKET + STAGING_KEY_PREFIX (separate bucket and key prefix parameters).\n", + "LAKEHOUSE_BUCKET + STAGING_KEY_PREFIX (separate bucket and key prefix parameters).\n", "\n", "This is for local testing. The CTS will stage the container's input folder in production.\n", "\"\"\"\n", @@ -415,7 +415,7 @@ "# S3 location where CTS will read input files from.\n", "# Set to None to skip upload (local-only testing).\n", "# format: s3:// URI (e.g. \"s3://cdm-lake/staging/run1/\")\n", - "STAGING_URI: str | None = \"s3://cdm-lake/staging/run1/input/\"\n", + "STAGING_URI: str | None = \"s3://cts/io/matt-cohere/staging/run1/input/\"\n", "\n", "if STAGING_URI:\n", " s3 = get_s3_client()\n", diff --git a/notebooks/ncbi_ftp_promote.ipynb b/notebooks/ncbi_ftp_promote.ipynb index a23fc149..cbf43a34 100644 --- a/notebooks/ncbi_ftp_promote.ipynb +++ b/notebooks/ncbi_ftp_promote.ipynb @@ -34,8 +34,8 @@ "| `_S3_KEY` | S3 object key (no scheme/bucket) | `staging/transfer_manifest.txt` |\n", "| `_PATH` | local filesystem path | `output/removed_manifest.txt` |\n", "\n", - "Lakehouse object: `s3://{STORE_BUCKET}/{LAKEHOUSE_KEY_PREFIX}raw_data/…/{filename}`\n", - "Staging object: `s3://{STORE_BUCKET}/{STAGING_KEY_PREFIX}raw_data/…/{filename}`" + "Lakehouse object: `s3://{LAKEHOUSE_BUCKET}/{LAKEHOUSE_KEY_PREFIX}raw_data/…/{filename}`\n", + "Staging object: `s3://{STAGING_BUCKET}/{STAGING_KEY_PREFIX}raw_data/…/{filename}`" ] }, { @@ -66,17 +66,21 @@ "\"\"\"Configure parameters.\n", "\n", "Path layout (how variables compose into a full S3 object path):\n", - " s3://{STORE_BUCKET}/{LAKEHOUSE_KEY_PREFIX}raw_data/{GCF|GCA}/{nnn}/{nnn}/{nnn}/{assembly_dir}/{file}\n", - " s3://{STORE_BUCKET}/{STAGING_KEY_PREFIX}raw_data/{assembly_dir}/{file}\n", + " s3://{LAKEHOUSE_BUCKET}/{LAKEHOUSE_KEY_PREFIX}raw_data/{GCF|GCA}/{nnn}/{nnn}/{nnn}/{assembly_dir}/{file}\n", + " s3://{STAGING_BUCKET}/{STAGING_KEY_PREFIX}raw_data/{assembly_dir}/{file}\n", "\"\"\"\n", "\n", - "# S3 bucket where staged files and final Lakehouse data live\n", + "# S3 bucket where CTS Phase 2 writes staged files\n", "# format: bucket name (no s3:// scheme)\n", - "STORE_BUCKET = \"cdm-lake\"\n", + "STAGING_BUCKET = \"cts\"\n", + "\n", + "# S3 bucket for the final Lakehouse destination\n", + "# format: bucket name (no s3:// scheme)\n", + "LAKEHOUSE_BUCKET = \"cdm-lake\"\n", "\n", "# Staging prefix written by CTS Phase 2\n", - "# format: S3 key prefix within STORE_BUCKET (no scheme, no bucket)\n", - "STAGING_KEY_PREFIX = \"staging/run1/\"\n", + "# format: S3 key prefix within STAGING_BUCKET (no scheme, no bucket)\n", + "STAGING_KEY_PREFIX = \"io/matt-cohere/staging/run1/output/\"\n", "\n", "# Local path to removed_manifest.txt from Phase 1 (or None to skip archiving)\n", "# format: local file path\n", @@ -91,17 +95,18 @@ "\n", "# S3 key of transfer_manifest.txt for trimming after promotion (or None to skip).\n", "# Only needed if the manifest was uploaded to S3 (e.g. via the staging cell in Phase 1).\n", - "# format: S3 object key within STORE_BUCKET (no scheme, no bucket)\n", - "MANIFEST_S3_KEY: str | None = \"staging/run1/input/transfer_manifest.txt\" # e.g. \"staging/transfer_manifest.txt\"\n", + "# format: S3 object key within STAGING_BUCKET (no scheme, no bucket)\n", + "MANIFEST_S3_KEY: str | None = \"io/matt-cohere/staging/run1/input/transfer_manifest.txt\" # e.g. \"staging/transfer_manifest.txt\"\n", "\n", "# Final Lakehouse path prefix\n", - "# format: S3 key prefix within STORE_BUCKET (no scheme, no bucket)\n", + "# format: S3 key prefix within LAKEHOUSE_BUCKET (no scheme, no bucket)\n", "LAKEHOUSE_KEY_PREFIX = DEFAULT_LAKEHOUSE_KEY_PREFIX\n", "\n", "# Dry-run mode — log actions without making changes\n", "DRY_RUN = False\n", "\n", - "print(f\"Bucket: {STORE_BUCKET}\")\n", + "print(f\"Staging bucket: {STAGING_BUCKET}\")\n", + "print(f\"Lakehouse bucket: {LAKEHOUSE_BUCKET}\")\n", "print(f\"Staging key prefix: {STAGING_KEY_PREFIX}\")\n", "print(f\"Updated manifest: {UPDATED_MANIFEST_PATH}\")\n", "print(f\"NCBI release: {NCBI_RELEASE}\")\n", @@ -143,7 +148,7 @@ "paginator = s3.get_paginator(\"list_objects_v2\")\n", "\n", "staged: list[str] = []\n", - "for page in paginator.paginate(Bucket=STORE_BUCKET, Prefix=STAGING_KEY_PREFIX):\n", + "for page in paginator.paginate(Bucket=STAGING_BUCKET, Prefix=STAGING_KEY_PREFIX):\n", " staged.extend(obj[\"Key\"] for obj in page.get(\"Contents\", []))\n", "\n", "sidecars = [k for k in staged if k.endswith((\".md5\", \".crc64nvme\"))]\n", @@ -172,7 +177,8 @@ "\n", "report = promote_from_s3(\n", " staging_key_prefix=STAGING_KEY_PREFIX,\n", - " bucket=STORE_BUCKET,\n", + " staging_bucket=STAGING_BUCKET,\n", + " lakehouse_bucket=LAKEHOUSE_BUCKET,\n", " removed_manifest_path=REMOVED_MANIFEST_PATH,\n", " updated_manifest_path=UPDATED_MANIFEST_PATH,\n", " ncbi_release=NCBI_RELEASE,\n", @@ -223,22 +229,16 @@ "paginator = s3.get_paginator(\"list_objects_v2\")\n", "\n", "descriptor_keys: list[str] = []\n", - "for page in paginator.paginate(Bucket=STORE_BUCKET, Prefix=LAKEHOUSE_KEY_PREFIX + \"metadata/\"):\n", + "for page in paginator.paginate(Bucket=LAKEHOUSE_BUCKET, Prefix=LAKEHOUSE_KEY_PREFIX + \"metadata/\"):\n", " descriptor_keys.extend(obj[\"Key\"] for obj in page.get(\"Contents\", []))\n", "\n", "print(f\"Found {len(descriptor_keys)} descriptor(s) in metadata/\")\n", "\n", "for key in descriptor_keys[:5]: # preview first 5\n", - " obj = s3.get_object(Bucket=STORE_BUCKET, Key=key)\n", + " obj = s3.get_object(Bucket=LAKEHOUSE_BUCKET, Key=key)\n", " descriptor = json.loads(obj[\"Body\"].read())\n", " print()\n", - " print(f\" Key: {key}\")\n", - " print(f\" Identifier: {descriptor.get('identifier')}\")\n", - " print(f\" Version: {descriptor.get('version')}\")\n", - " print(f\" Resources: {len(descriptor.get('resources', []))} file(s)\")\n", - "\n", - "if len(descriptor_keys) > 5:\n", - " print(f\" ... and {len(descriptor_keys) - 5} more\")" + " print(f\" Key: {key}\")" ] } ], diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index ab345ba3..2a127f19 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -39,7 +39,8 @@ def promote_from_s3( # noqa: PLR0913 staging_key_prefix: str, - bucket: str, + staging_bucket: str, + lakehouse_bucket: str, removed_manifest_path: str | Path | None = None, updated_manifest_path: str | Path | None = None, ncbi_release: str | None = None, @@ -54,7 +55,8 @@ def promote_from_s3( # noqa: PLR0913 with MD5 metadata from ``.md5`` sidecar files. :param staging_key_prefix: S3 key prefix where CTS output was written - :param bucket: S3 bucket name + :param staging_bucket: S3 bucket containing the staged files (e.g. ``"cts"``) + :param lakehouse_bucket: S3 bucket for the final Lakehouse destination (e.g. ``"cdm-lake"``) :param removed_manifest_path: local path to the removed_manifest file :param updated_manifest_path: local path to the updated_manifest file :param ncbi_release: NCBI release version tag for archiving @@ -69,7 +71,7 @@ def promote_from_s3( # noqa: PLR0913 # Collect all objects under the staging prefix staged_objects: list[str] = [] - for page in paginator.paginate(Bucket=bucket, Prefix=normalized_staging_key_prefix): + for page in paginator.paginate(Bucket=staging_bucket, Prefix=normalized_staging_key_prefix): staged_objects.extend(obj["Key"] for obj in page.get("Contents", [])) # Separate data files from sidecars @@ -87,7 +89,7 @@ def promote_from_s3( # noqa: PLR0913 if manifest_file and Path(str(manifest_file)).is_file(): archived += _archive_assemblies( str(manifest_file), - bucket=bucket, + lakehouse_bucket=lakehouse_bucket, ncbi_release=ncbi_release, lakehouse_key_prefix=lakehouse_key_prefix, archive_reason=reason, @@ -100,13 +102,14 @@ def promote_from_s3( # noqa: PLR0913 sidecars, normalized_staging_key_prefix, lakehouse_key_prefix, - bucket, + staging_bucket, + lakehouse_bucket, dry_run=dry_run, ) # Trim manifest for resumability if manifest_s3_key and promoted_accessions and not dry_run: - _trim_manifest(manifest_s3_key, bucket, promoted_accessions) + _trim_manifest(manifest_s3_key, staging_bucket, promoted_accessions) # Upload frictionless descriptors for each promoted assembly descriptors_written = 0 @@ -115,7 +118,7 @@ def promote_from_s3( # noqa: PLR0913 continue try: descriptor = create_descriptor(adir, acc, resources) - upload_descriptor(descriptor, adir, bucket, lakehouse_key_prefix, dry_run=dry_run) + upload_descriptor(descriptor, adir, lakehouse_bucket, lakehouse_key_prefix, dry_run=dry_run) descriptors_written += 1 except Exception: logger.exception("Failed to write descriptor for %s", adir) @@ -149,7 +152,8 @@ def _promote_data_files( # noqa: PLR0913, PLR0915 sidecars: set[str], normalized_staging_prefix: str, lakehouse_key_prefix: str, - bucket: str, + staging_bucket: str, + lakehouse_bucket: str, *, dry_run: bool, ) -> tuple[int, int, set[str], defaultdict[tuple[str, str], list[DescriptorResource]]]: @@ -181,19 +185,19 @@ def _promote_data_files( # noqa: PLR0913, PLR0915 with tempfile.NamedTemporaryFile(delete=False) as tmp: tmp_path = tmp.name try: - s3.download_file(Bucket=bucket, Key=staged_key, Filename=tmp_path) + s3.download_file(Bucket=staging_bucket, Key=staged_key, Filename=tmp_path) # Read MD5 from sidecar metadata: dict[str, str] = {} md5_key = staged_key + ".md5" if md5_key in sidecars: - md5_obj = s3.get_object(Bucket=bucket, Key=md5_key) + md5_obj = s3.get_object(Bucket=staging_bucket, Key=md5_key) metadata["md5"] = md5_obj["Body"].read().decode().strip() final_key_path = PurePosixPath(final_key) upload_succeeded = upload_file( tmp_path, - f"{bucket}/{final_key_path.parent}", + f"{lakehouse_bucket}/{final_key_path.parent}", metadata=metadata, object_name=final_key_path.name, ) @@ -240,7 +244,7 @@ def _promote_data_files( # noqa: PLR0913, PLR0915 def _archive_assemblies( # noqa: PLR0913 manifest_local_path: str, - bucket: str, + lakehouse_bucket: str, ncbi_release: str | None = None, lakehouse_key_prefix: str = DEFAULT_LAKEHOUSE_KEY_PREFIX, archive_reason: str = "unknown", @@ -256,7 +260,7 @@ def _archive_assemblies( # noqa: PLR0913 originals remain in place to be overwritten by the promote step. :param manifest_local_path: local path to a manifest file (one accession per line) - :param bucket: S3 bucket name + :param lakehouse_bucket: S3 bucket for the Lakehouse (source and archive destination) :param ncbi_release: release tag used in the archive path :param lakehouse_key_prefix: S3 key prefix for the Lakehouse dataset root :param archive_reason: metadata value describing why the object was archived @@ -284,7 +288,7 @@ def _archive_assemblies( # noqa: PLR0913 paginator = s3.get_paginator("list_objects_v2") matching_keys: list[str] = [] - for page in paginator.paginate(Bucket=bucket, Prefix=source_prefix): + for page in paginator.paginate(Bucket=lakehouse_bucket, Prefix=source_prefix): matching_keys.extend(obj["Key"] for obj in page.get("Contents", []) if accession in obj["Key"]) if not matching_keys: @@ -310,8 +314,8 @@ def _archive_assemblies( # noqa: PLR0913 try: copy_object( - f"{bucket}/{source_key}", - f"{bucket}/{archive_key}", + f"{lakehouse_bucket}/{source_key}", + f"{lakehouse_bucket}/{archive_key}", metadata={ "ncbi_last_release": release_tag, "archive_reason": archive_reason, @@ -319,7 +323,7 @@ def _archive_assemblies( # noqa: PLR0913 }, ) if delete_source: - delete_object(f"{bucket}/{source_key}") + delete_object(f"{lakehouse_bucket}/{source_key}") archived += 1 logger.debug(" Archived: %s -> %s", source_key, archive_key) except Exception: @@ -330,7 +334,7 @@ def _archive_assemblies( # noqa: PLR0913 try: archive_descriptor( assembly_dir, - bucket, + lakehouse_bucket, lakehouse_key_prefix, release_tag, archive_reason=archive_reason, @@ -346,11 +350,11 @@ def _archive_assemblies( # noqa: PLR0913 # ── Manifest trimming ─────────────────────────────────────────────────── -def _trim_manifest(manifest_s3_key: str, bucket: str, promoted_accessions: set[str]) -> None: +def _trim_manifest(manifest_s3_key: str, staging_bucket: str, promoted_accessions: set[str]) -> None: """Remove promoted accessions from the transfer manifest in S3. :param manifest_s3_key: S3 object key of the transfer_manifest.txt - :param bucket: S3 bucket name + :param staging_bucket: S3 bucket containing the transfer manifest :param promoted_accessions: set of accessions that were successfully promoted """ s3 = get_s3_client() @@ -360,13 +364,13 @@ def _trim_manifest(manifest_s3_key: str, bucket: str, promoted_accessions: set[s try: try: - s3.download_file(Bucket=bucket, Key=manifest_s3_key, Filename=tmp_path) + s3.download_file(Bucket=staging_bucket, Key=manifest_s3_key, Filename=tmp_path) except s3.exceptions.NoSuchKey: - logger.warning("Manifest not found in S3 (s3://%s/%s) — skipping trim", bucket, manifest_s3_key) + logger.warning("Manifest not found in S3 (s3://%s/%s) — skipping trim", staging_bucket, manifest_s3_key) return except botocore.exceptions.ClientError as e: if e.response["Error"]["Code"] == "404": - logger.warning("Manifest not found in S3 (s3://%s/%s) — skipping trim", bucket, manifest_s3_key) + logger.warning("Manifest not found in S3 (s3://%s/%s) — skipping trim", staging_bucket, manifest_s3_key) return raise @@ -378,7 +382,7 @@ def _trim_manifest(manifest_s3_key: str, bucket: str, promoted_accessions: set[s with Path(tmp_path).open("w") as f: f.writelines(remaining) - s3.upload_file(Filename=tmp_path, Bucket=bucket, Key=manifest_s3_key) + s3.upload_file(Filename=tmp_path, Bucket=staging_bucket, Key=manifest_s3_key) logger.info( "Trimmed manifest: %d -> %d entries (%d promoted)", len(lines), diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 0fe7384a..f27e4202 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -147,6 +147,36 @@ def test_bucket(minio_s3_client: botocore.client.BaseClient, request: pytest.Fix return bucket +@pytest.fixture +def staging_test_bucket(minio_s3_client: botocore.client.BaseClient, request: pytest.FixtureRequest) -> str: + """Create a per-test staging bucket in MinIO and return its name. + + Mirrors ``test_bucket`` but uses a ``staging-`` prefix so staging and + Lakehouse buckets are distinct within the same test. + """ + bucket = "staging-" + _bucket_name_from_node(request.node.nodeid) + if len(bucket) > _MAX_BUCKET_LEN: + suffix = hashlib.md5(bucket.encode()).hexdigest()[:6] # noqa: S324 + bucket = f"{bucket[: _MAX_BUCKET_LEN - 7]}-{suffix}" + s3 = minio_s3_client + + try: + s3.head_bucket(Bucket=bucket) + paginator = s3.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=bucket): + for obj in page.get("Contents", []): + s3.delete_object(Bucket=bucket, Key=obj["Key"]) + except s3.exceptions.NoSuchBucket: + s3.create_bucket(Bucket=bucket) + except botocore.exceptions.ClientError as e: + if e.response["Error"]["Code"] in ("404", "NoSuchBucket"): + s3.create_bucket(Bucket=bucket) + else: + raise + + return bucket + + # ── Helpers ───────────────────────────────────────────────────────────── diff --git a/tests/integration/test_full_pipeline.py b/tests/integration/test_full_pipeline.py index 2654215b..e8f6e4a6 100644 --- a/tests/integration/test_full_pipeline.py +++ b/tests/integration/test_full_pipeline.py @@ -29,7 +29,7 @@ from cdm_data_loaders.ncbi_ftp.promote import DEFAULT_LAKEHOUSE_KEY_PREFIX, promote_from_s3 from cdm_data_loaders.pipelines.ncbi_ftp_download import download_batch -from .conftest import get_object_metadata, list_all_keys, stage_files_to_minio +from .conftest import get_object_metadata, list_all_keys, stage_files_to_minio, staging_test_bucket # noqa: F401 STABLE_PREFIX = "900" STAGING_PREFIX = "staging/run1/" @@ -46,6 +46,7 @@ def test_full_pipeline_small_batch( self, minio_s3_client: object, test_bucket: str, + staging_test_bucket: str, tmp_path: Path, ) -> None: """Single assembly flows through all three phases into MinIO.""" @@ -76,13 +77,14 @@ def test_full_pipeline_small_batch( assert report["failed"] == 0 # ── Upload local output to MinIO staging ──────────────────────── - keys = stage_files_to_minio(s3, test_bucket, output_dir, STAGING_PREFIX) + keys = stage_files_to_minio(s3, staging_test_bucket, output_dir, STAGING_PREFIX) assert len(keys) > 0, "Expected files staged to MinIO" # ── Phase 3: Promote from staging to final path ───────────────── promote_report = promote_from_s3( staging_key_prefix=STAGING_PREFIX, - bucket=test_bucket, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, lakehouse_key_prefix=PATH_PREFIX, ) assert promote_report["promoted"] >= 1 @@ -112,6 +114,7 @@ def test_full_pipeline_incremental( self, minio_s3_client: object, test_bucket: str, + staging_test_bucket: str, tmp_path: Path, ) -> None: """Second sync archives the old version and promotes the new one.""" @@ -133,15 +136,16 @@ def test_full_pipeline_incremental( report1 = download_batch(str(manifest1), str(output1), threads=1, limit=1) assert report1["succeeded"] >= 1 - stage_files_to_minio(s3, test_bucket, output1, STAGING_PREFIX) + stage_files_to_minio(s3, staging_test_bucket, output1, STAGING_PREFIX) - # Upload manifest to MinIO for trimming + # Upload manifest to MinIO for trimming (manifest lives in staging bucket) manifest_key = "ncbi/transfer_manifest.txt" - s3.upload_file(Filename=str(manifest1), Bucket=test_bucket, Key=manifest_key) + s3.upload_file(Filename=str(manifest1), Bucket=staging_test_bucket, Key=manifest_key) promote1 = promote_from_s3( staging_key_prefix=STAGING_PREFIX, - bucket=test_bucket, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, manifest_s3_key=manifest_key, lakehouse_key_prefix=PATH_PREFIX, ) @@ -190,15 +194,16 @@ def test_full_pipeline_incremental( assert report2["succeeded"] >= 1 # Clean staging and re-stage - staging_keys = list_all_keys(s3, test_bucket, STAGING_PREFIX) + staging_keys = list_all_keys(s3, staging_test_bucket, STAGING_PREFIX) for key in staging_keys: - s3.delete_object(Bucket=test_bucket, Key=key) - stage_files_to_minio(s3, test_bucket, output2, STAGING_PREFIX) + s3.delete_object(Bucket=staging_test_bucket, Key=key) + stage_files_to_minio(s3, staging_test_bucket, output2, STAGING_PREFIX) # Phase 3 — promote with archival promote2 = promote_from_s3( staging_key_prefix=STAGING_PREFIX, - bucket=test_bucket, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, updated_manifest_path=str(updated_manifest), ncbi_release="test-incremental", lakehouse_key_prefix=PATH_PREFIX, diff --git a/tests/integration/test_promote_e2e.py b/tests/integration/test_promote_e2e.py index 46324603..67778c64 100644 --- a/tests/integration/test_promote_e2e.py +++ b/tests/integration/test_promote_e2e.py @@ -21,7 +21,7 @@ ) from cdm_data_loaders.ncbi_ftp.promote import DEFAULT_LAKEHOUSE_KEY_PREFIX, promote_from_s3 -from .conftest import get_object_metadata, list_all_keys, seed_lakehouse +from .conftest import get_object_metadata, list_all_keys, seed_lakehouse, staging_test_bucket # noqa: F401 from pathlib import Path @@ -82,14 +82,15 @@ def _write_manifest(tmp_path: Path, accessions: list[str], name: str) -> Path: class TestPromoteFromStaging: """Promote staged files to final Lakehouse paths.""" - def test_promote_from_staging(self, minio_s3_client: object, test_bucket: str) -> None: + def test_promote_from_staging(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: """Staged files appear at the final Lakehouse path with MD5 metadata.""" s3 = minio_s3_client - _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) + _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) report = promote_from_s3( staging_key_prefix=STAGING_PREFIX, - bucket=test_bucket, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, lakehouse_key_prefix=PATH_PREFIX, ) @@ -112,21 +113,23 @@ def test_promote_from_staging(self, minio_s3_client: object, test_bucket: str) - class TestPromoteIdempotent: """Promoting the same staging data twice should succeed without errors.""" - def test_promote_idempotent(self, minio_s3_client: object, test_bucket: str) -> None: + def test_promote_idempotent(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: """Second promote succeeds and produces the same final state.""" s3 = minio_s3_client - _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) + _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) report1 = promote_from_s3( staging_key_prefix=STAGING_PREFIX, - bucket=test_bucket, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, lakehouse_key_prefix=PATH_PREFIX, ) keys_after_first = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") report2 = promote_from_s3( staging_key_prefix=STAGING_PREFIX, - bucket=test_bucket, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, lakehouse_key_prefix=PATH_PREFIX, ) keys_after_second = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") @@ -142,7 +145,7 @@ def test_promote_idempotent(self, minio_s3_client: object, test_bucket: str) -> class TestPromoteArchiveUpdated: """Archive existing assemblies before overwriting with updated versions.""" - def test_archive_updated(self, minio_s3_client: object, test_bucket: str, tmp_path: Path) -> None: + def test_archive_updated(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path) -> None: """Updated assemblies are archived before being overwritten.""" s3 = minio_s3_client @@ -154,13 +157,14 @@ def test_archive_updated(self, minio_s3_client: object, test_bucket: str, tmp_pa seed_lakehouse(s3, test_bucket, ACCESSION_A, old_files, PATH_PREFIX, ASSEMBLY_DIR_A) # Stage "new" version - _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) + _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) updated_manifest = _write_manifest(tmp_path, [ACCESSION_A], "updated_manifest.txt") report = promote_from_s3( staging_key_prefix=STAGING_PREFIX, - bucket=test_bucket, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, updated_manifest_path=str(updated_manifest), ncbi_release="2024-01", lakehouse_key_prefix=PATH_PREFIX, @@ -186,7 +190,7 @@ def test_archive_updated(self, minio_s3_client: object, test_bucket: str, tmp_pa class TestPromoteArchiveRemoved: """Archive and delete replaced/suppressed assemblies.""" - def test_archive_removed(self, minio_s3_client: object, test_bucket: str, tmp_path: Path) -> None: + def test_archive_removed(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path) -> None: """Removed assemblies are archived and source objects are deleted.""" s3 = minio_s3_client @@ -201,7 +205,8 @@ def test_archive_removed(self, minio_s3_client: object, test_bucket: str, tmp_pa # Stage something (even empty staging is fine — promote won't find data files for this accession) report = promote_from_s3( staging_key_prefix=STAGING_PREFIX, - bucket=test_bucket, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, removed_manifest_path=str(removed_manifest), ncbi_release="2024-01", lakehouse_key_prefix=PATH_PREFIX, @@ -230,14 +235,15 @@ def test_archive_removed(self, minio_s3_client: object, test_bucket: str, tmp_pa class TestPromoteDryRun: """Dry-run mode should not create any objects.""" - def test_promote_dry_run(self, minio_s3_client: object, test_bucket: str) -> None: + def test_promote_dry_run(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: """Dry-run logs actions but creates no objects at the final path.""" s3 = minio_s3_client - _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) + _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) report = promote_from_s3( staging_key_prefix=STAGING_PREFIX, - bucket=test_bucket, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, lakehouse_key_prefix=PATH_PREFIX, dry_run=True, ) @@ -255,34 +261,35 @@ def test_promote_dry_run(self, minio_s3_client: object, test_bucket: str) -> Non class TestPromoteTrimsManifest: """Manifest trimming removes promoted accessions.""" - def test_trims_manifest(self, minio_s3_client: object, test_bucket: str, tmp_path: Path) -> None: + def test_trims_manifest(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path) -> None: """Transfer manifest in MinIO is trimmed to exclude promoted accessions.""" s3 = minio_s3_client - # Upload a transfer manifest with 3 entries to MinIO + # Upload a transfer manifest with 3 entries to MinIO (manifest lives in staging) manifest_key = "ncbi/transfer_manifest.txt" manifest_lines = [ "/genomes/all/GCF/900/000/001/GCF_900000001.1_FakeAssemblyA/\n", "/genomes/all/GCF/900/000/002/GCF_900000002.1_FakeAssemblyB/\n", "/genomes/all/GCF/900/000/003/GCF_900000003.1_FakeAssemblyC/\n", ] - s3.put_object(Bucket=test_bucket, Key=manifest_key, Body="".join(manifest_lines).encode()) + s3.put_object(Bucket=staging_test_bucket, Key=manifest_key, Body="".join(manifest_lines).encode()) # Stage only assemblies A and B (not C) - _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) - _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_B) + _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) + _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_B) report = promote_from_s3( staging_key_prefix=STAGING_PREFIX, - bucket=test_bucket, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, manifest_s3_key=manifest_key, lakehouse_key_prefix=PATH_PREFIX, ) assert report["failed"] == 0 - # Read back the manifest from MinIO - resp = s3.get_object(Bucket=test_bucket, Key=manifest_key) + # Read back the manifest from MinIO (it lives in staging) + resp = s3.get_object(Bucket=staging_test_bucket, Key=manifest_key) remaining = resp["Body"].read().decode() remaining_lines = [line.strip() for line in remaining.strip().splitlines() if line.strip()] @@ -296,7 +303,7 @@ def test_trims_manifest(self, minio_s3_client: object, test_bucket: str, tmp_pat class TestPromoteIncompleteStaging: """Incomplete staging (sidecar only, no data) should not promote anything.""" - def test_incomplete_staging(self, minio_s3_client: object, test_bucket: str) -> None: + def test_incomplete_staging(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: """Only .md5 sidecars staged → nothing promoted.""" s3 = minio_s3_client @@ -305,11 +312,12 @@ def test_incomplete_staging(self, minio_s3_client: object, test_bucket: str) -> base = f"{STAGING_PREFIX}{rel}" fname = f"{ASSEMBLY_DIR_A}_genomic.fna.gz" md5_key = f"{base}{fname}.md5" - s3.put_object(Bucket=test_bucket, Key=md5_key, Body=_md5(FAKE_GENOMIC).encode()) + s3.put_object(Bucket=staging_test_bucket, Key=md5_key, Body=_md5(FAKE_GENOMIC).encode()) report = promote_from_s3( staging_key_prefix=STAGING_PREFIX, - bucket=test_bucket, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, lakehouse_key_prefix=PATH_PREFIX, ) @@ -327,14 +335,15 @@ def test_incomplete_staging(self, minio_s3_client: object, test_bucket: str) -> class TestPromoteCreatesDescriptor: """Promote step writes a frictionless descriptor for each promoted assembly.""" - def test_descriptor_created(self, minio_s3_client: object, test_bucket: str) -> None: + def test_descriptor_created(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: """After promote, a JSON descriptor exists under ``metadata/``.""" s3 = minio_s3_client - _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) + _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) promote_from_s3( staging_key_prefix=STAGING_PREFIX, - bucket=test_bucket, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, lakehouse_key_prefix=PATH_PREFIX, ) @@ -345,14 +354,15 @@ def test_descriptor_created(self, minio_s3_client: object, test_bucket: str) -> assert body["identifier"] == f"NCBI:{ACCESSION_A}" assert body["resource_type"] == "dataset" - def test_descriptor_resources_include_promoted_files(self, minio_s3_client: object, test_bucket: str) -> None: + def test_descriptor_resources_include_promoted_files(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: """Descriptor's ``resources`` list references the final Lakehouse key.""" s3 = minio_s3_client - _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) + _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) promote_from_s3( staging_key_prefix=STAGING_PREFIX, - bucket=test_bucket, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, lakehouse_key_prefix=PATH_PREFIX, ) @@ -363,14 +373,15 @@ def test_descriptor_resources_include_promoted_files(self, minio_s3_client: obje resource_paths = [r["path"] for r in body["resources"]] assert any(PATH_PREFIX + "raw_data/" in p for p in resource_paths) - def test_descriptor_resources_have_md5(self, minio_s3_client: object, test_bucket: str) -> None: + def test_descriptor_resources_have_md5(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: """Resources with .md5 sidecars include the hash value.""" s3 = minio_s3_client - _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) + _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) promote_from_s3( staging_key_prefix=STAGING_PREFIX, - bucket=test_bucket, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, lakehouse_key_prefix=PATH_PREFIX, ) @@ -382,15 +393,16 @@ def test_descriptor_resources_have_md5(self, minio_s3_client: object, test_bucke for resource in body["resources"]: assert "hash" in resource, f"Expected hash in resource: {resource}" - def test_multiple_assemblies_get_separate_descriptors(self, minio_s3_client: object, test_bucket: str) -> None: + def test_multiple_assemblies_get_separate_descriptors(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: """Each assembly gets its own descriptor file.""" s3 = minio_s3_client - _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) - _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_B) + _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) + _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_B) promote_from_s3( staging_key_prefix=STAGING_PREFIX, - bucket=test_bucket, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, lakehouse_key_prefix=PATH_PREFIX, ) @@ -406,7 +418,7 @@ def test_multiple_assemblies_get_separate_descriptors(self, minio_s3_client: obj class TestPromoteArchiveUpdatedIncludesDescriptor: """Archiving updated assemblies also archives the descriptor.""" - def test_archive_copies_descriptor(self, minio_s3_client: object, test_bucket: str, tmp_path: Path) -> None: + def test_archive_copies_descriptor(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path) -> None: """After archiving an updated assembly, the descriptor appears under archive/.""" s3 = minio_s3_client @@ -419,12 +431,13 @@ def test_archive_copies_descriptor(self, minio_s3_client: object, test_bucket: s descriptor_key = build_descriptor_key(ASSEMBLY_DIR_A, PATH_PREFIX) s3.put_object(Bucket=test_bucket, Key=descriptor_key, Body=json.dumps(descriptor).encode()) - _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) + _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) updated_manifest = _write_manifest(tmp_path, [ACCESSION_A], "updated_manifest.txt") promote_from_s3( staging_key_prefix=STAGING_PREFIX, - bucket=test_bucket, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, updated_manifest_path=str(updated_manifest), ncbi_release="2024-01", lakehouse_key_prefix=PATH_PREFIX, @@ -441,7 +454,7 @@ def test_archive_copies_descriptor(self, minio_s3_client: object, test_bucket: s class TestPromoteArchiveRemovedIncludesDescriptor: """Archiving removed assemblies also archives the descriptor.""" - def test_archive_removed_copies_descriptor(self, minio_s3_client: object, test_bucket: str, tmp_path: Path) -> None: + def test_archive_removed_copies_descriptor(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path) -> None: """After archiving a removed assembly, the descriptor is under archive/.""" s3 = minio_s3_client @@ -457,7 +470,8 @@ def test_archive_removed_copies_descriptor(self, minio_s3_client: object, test_b promote_from_s3( staging_key_prefix=STAGING_PREFIX, - bucket=test_bucket, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, removed_manifest_path=str(removed_manifest), ncbi_release="2024-01", lakehouse_key_prefix=PATH_PREFIX, @@ -473,14 +487,15 @@ def test_archive_removed_copies_descriptor(self, minio_s3_client: object, test_b class TestPromoteDryRunNoDescriptor: """Dry-run must not write any descriptor files.""" - def test_dry_run_no_descriptor(self, minio_s3_client: object, test_bucket: str) -> None: + def test_dry_run_no_descriptor(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: """Dry-run does not upload a descriptor to the metadata/ prefix.""" s3 = minio_s3_client - _stage_assembly(s3, test_bucket, ASSEMBLY_DIR_A) + _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) promote_from_s3( staging_key_prefix=STAGING_PREFIX, - bucket=test_bucket, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, lakehouse_key_prefix=PATH_PREFIX, dry_run=True, ) diff --git a/tests/ncbi_ftp/test_promote.py b/tests/ncbi_ftp/test_promote.py index fdb180db..73777b9b 100644 --- a/tests/ncbi_ftp/test_promote.py +++ b/tests/ncbi_ftp/test_promote.py @@ -31,7 +31,7 @@ def test_promote_dry_run_no_writes(mock_s3_client_no_checksum: botocore.client.B prefix = "staging/run1/" _stage_files(mock_s3_client_no_checksum, prefix) - report = promote_from_s3(staging_key_prefix=prefix, bucket=TEST_BUCKET, dry_run=True) + report = promote_from_s3(staging_key_prefix=prefix, staging_bucket=TEST_BUCKET, lakehouse_bucket=TEST_BUCKET, dry_run=True) assert report["promoted"] == 1 assert report["dry_run"] is True @@ -45,7 +45,7 @@ def test_promote_with_metadata(mock_s3_client_no_checksum: botocore.client.BaseC prefix = "staging/run1/" _stage_files(mock_s3_client_no_checksum, prefix) - report = promote_from_s3(staging_key_prefix=prefix, bucket=TEST_BUCKET) + report = promote_from_s3(staging_key_prefix=prefix, staging_bucket=TEST_BUCKET, lakehouse_bucket=TEST_BUCKET) assert report["promoted"] == 1 # only .fna.gz, not download_report.json assert report["failed"] == 0 @@ -106,7 +106,7 @@ def test_archive_assemblies_removed(mock_s3_client_no_checksum: botocore.client. assert ( _archive_assemblies( str(manifest), - bucket=TEST_BUCKET, + lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="replaced_or_suppressed", delete_source=True, @@ -137,7 +137,7 @@ def test_archive_assemblies_updated_no_delete( assert ( _archive_assemblies( - str(manifest), bucket=TEST_BUCKET, ncbi_release="2024-06", archive_reason="updated", delete_source=False + str(manifest), lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-06", archive_reason="updated", delete_source=False ) == 1 ) @@ -164,9 +164,9 @@ def test_archive_assemblies_multiple_releases_no_collision( manifest = tmp_path / "updated.txt" manifest.write_text(f"{accession}\n") - _archive_assemblies(str(manifest), bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated") + _archive_assemblies(str(manifest), lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated") mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"v2-data") - _archive_assemblies(str(manifest), bucket=TEST_BUCKET, ncbi_release="2024-06", archive_reason="updated") + _archive_assemblies(str(manifest), lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-06", archive_reason="updated") archive_key_1 = ( f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" @@ -191,7 +191,7 @@ def test_archive_assemblies_dry_run(mock_s3_client_no_checksum: botocore.client. assert ( _archive_assemblies( str(manifest), - bucket=TEST_BUCKET, + lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="replaced_or_suppressed", delete_source=True, @@ -212,7 +212,7 @@ def test_archive_assemblies_no_objects_skips( """Accessions with no existing S3 objects are silently skipped.""" manifest = tmp_path / "updated.txt" manifest.write_text("GCF_000001215.4\n") - assert _archive_assemblies(str(manifest), bucket=TEST_BUCKET, ncbi_release="2024-01") == 0 + assert _archive_assemblies(str(manifest), lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01") == 0 @pytest.mark.s3 @@ -228,7 +228,7 @@ def test_archive_assemblies_unknown_release_fallback( manifest = tmp_path / "updated.txt" manifest.write_text(f"{accession}\n") - assert _archive_assemblies(str(manifest), bucket=TEST_BUCKET, ncbi_release=None) == 1 + assert _archive_assemblies(str(manifest), lakehouse_bucket=TEST_BUCKET, ncbi_release=None) == 1 archive_key = ( f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/unknown/raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" From 1da3d05629999c69cc7b1d1c9861a98e4bc5465b Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 29 Apr 2026 12:29:48 -0700 Subject: [PATCH 061/128] consolidate progress bars --- notebooks/ncbi_ftp_download.ipynb | 18 +++++--- notebooks/ncbi_ftp_manifest.ipynb | 16 ++++--- notebooks/ncbi_ftp_promote.ipynb | 17 ++++--- src/cdm_data_loaders/ncbi_ftp/assembly.py | 4 +- src/cdm_data_loaders/ncbi_ftp/metadata.py | 2 +- src/cdm_data_loaders/ncbi_ftp/promote.py | 13 +++--- .../pipelines/ncbi_ftp_download.py | 35 ++++++++------- src/cdm_data_loaders/utils/s3.py | 45 +++++++++++++------ tests/integration/test_promote_e2e.py | 32 +++++++++---- tests/ncbi_ftp/test_promote.py | 10 ++++- tests/utils/test_s3.py | 12 +++-- 11 files changed, 132 insertions(+), 72 deletions(-) diff --git a/notebooks/ncbi_ftp_download.ipynb b/notebooks/ncbi_ftp_download.ipynb index 84ac4fbf..c8e10603 100644 --- a/notebooks/ncbi_ftp_download.ipynb +++ b/notebooks/ncbi_ftp_download.ipynb @@ -124,11 +124,13 @@ "PROVIDE_CREDENTIALS = False # Set to False to rely on environment credentials (e.g. IAM role)\n", "if PROVIDE_CREDENTIALS:\n", " reset_s3_client() # Clear any existing client to ensure new credentials are used\n", - " get_s3_client({\n", - " \"endpoint_url\": \"http://localhost:9000\",\n", - " \"aws_access_key_id\": \"minioadmin\",\n", - " \"aws_secret_access_key\": \"minioadmin\",\n", - " })" + " get_s3_client(\n", + " {\n", + " \"endpoint_url\": \"http://localhost:9000\",\n", + " \"aws_access_key_id\": \"minioadmin\",\n", + " \"aws_secret_access_key\": \"minioadmin\",\n", + " }\n", + " )" ] }, { @@ -187,6 +189,8 @@ "source": [ "\"\"\"Display download and staging report.\"\"\"\n", "\n", + "FAILURE_PREVIEW = 10\n", + "\n", "print(\"=\" * 50)\n", "print(\"DOWNLOAD & STAGE REPORT\")\n", "print(\"=\" * 50)\n", @@ -200,8 +204,10 @@ "\n", "if report[\"failed\"] > 0:\n", " print(\"\\nFailed assemblies:\")\n", - " for failure in report[\"failures\"]:\n", + " for failure in report[\"failures\"][:FAILURE_PREVIEW]:\n", " print(f\" {failure['path']}: {failure['error']}\")\n", + " if report[\"failed\"] > FAILURE_PREVIEW:\n", + " print(f\" ... and {report['failed'] - FAILURE_PREVIEW} more\")\n", "\n", "if report[\"dry_run\"]:\n", " print(\"\\nThis was a dry-run. Set DRY_RUN = False and re-run to upload to S3.\")" diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb index 0ade0917..49045123 100644 --- a/notebooks/ncbi_ftp_manifest.ipynb +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -79,11 +79,13 @@ "PROVIDE_CREDENTIALS = False # Set to False to rely on environment credentials (e.g. IAM role)\n", "if PROVIDE_CREDENTIALS:\n", " reset_s3_client() # Clear any existing client to ensure new credentials are used\n", - " get_s3_client({\n", - " \"endpoint_url\": \"http://localhost:9000\",\n", - " \"aws_access_key_id\": \"minioadmin\",\n", - " \"aws_secret_access_key\": \"minioadmin\",\n", - " })" + " get_s3_client(\n", + " {\n", + " \"endpoint_url\": \"http://localhost:9000\",\n", + " \"aws_access_key_id\": \"minioadmin\",\n", + " \"aws_secret_access_key\": \"minioadmin\",\n", + " }\n", + " )" ] }, { @@ -111,7 +113,9 @@ "\n", "# S3 location where the new snapshot will be uploaded after diffing\n", "# format: s3:// URI\n", - "SNAPSHOT_UPLOAD_URI: str | None = \"s3://cdm-lake/tenant-general-warehouse/kbase/datasets/ncbi/assembly_summary_refseq.txt\"\n", + "SNAPSHOT_UPLOAD_URI: str | None = (\n", + " \"s3://cdm-lake/tenant-general-warehouse/kbase/datasets/ncbi/assembly_summary_refseq.txt\"\n", + ")\n", "\n", "# Verify candidates against the S3 Lakehouse — prune assemblies already present.\n", "# Set LAKEHOUSE_BUCKET to your bucket name to enable, or None to skip.\n", diff --git a/notebooks/ncbi_ftp_promote.ipynb b/notebooks/ncbi_ftp_promote.ipynb index cbf43a34..1be65156 100644 --- a/notebooks/ncbi_ftp_promote.ipynb +++ b/notebooks/ncbi_ftp_promote.ipynb @@ -96,7 +96,9 @@ "# S3 key of transfer_manifest.txt for trimming after promotion (or None to skip).\n", "# Only needed if the manifest was uploaded to S3 (e.g. via the staging cell in Phase 1).\n", "# format: S3 object key within STAGING_BUCKET (no scheme, no bucket)\n", - "MANIFEST_S3_KEY: str | None = \"io/matt-cohere/staging/run1/input/transfer_manifest.txt\" # e.g. \"staging/transfer_manifest.txt\"\n", + "MANIFEST_S3_KEY: str | None = (\n", + " \"io/matt-cohere/staging/run1/input/transfer_manifest.txt\" # e.g. \"staging/transfer_manifest.txt\"\n", + ")\n", "\n", "# Final Lakehouse path prefix\n", "# format: S3 key prefix within LAKEHOUSE_BUCKET (no scheme, no bucket)\n", @@ -128,11 +130,13 @@ "PROVIDE_CREDENTIALS = False # Set to False to rely on environment credentials (e.g. IAM role)\n", "if PROVIDE_CREDENTIALS:\n", " reset_s3_client() # Clear any existing client to ensure new credentials are used\n", - " get_s3_client({\n", - " \"endpoint_url\": \"http://localhost:9000\",\n", - " \"aws_access_key_id\": \"minioadmin\",\n", - " \"aws_secret_access_key\": \"minioadmin\",\n", - " })" + " get_s3_client(\n", + " {\n", + " \"endpoint_url\": \"http://localhost:9000\",\n", + " \"aws_access_key_id\": \"minioadmin\",\n", + " \"aws_secret_access_key\": \"minioadmin\",\n", + " }\n", + " )" ] }, { @@ -218,6 +222,7 @@ { "cell_type": "code", "execution_count": null, + "id": "7fb27b941602401d91542211134fc71a", "metadata": {}, "outputs": [], "source": [ diff --git a/src/cdm_data_loaders/ncbi_ftp/assembly.py b/src/cdm_data_loaders/ncbi_ftp/assembly.py index cd2981ef..b424702b 100644 --- a/src/cdm_data_loaders/ncbi_ftp/assembly.py +++ b/src/cdm_data_loaders/ncbi_ftp/assembly.py @@ -163,7 +163,7 @@ def download_assembly_to_local( dest_dir = Path(output_dir) / rel_path dest_dir.mkdir(parents=True, exist_ok=True) - logger.info("Downloading %s -> %s", accession, dest_dir) + logger.debug("Downloading %s -> %s", accession, dest_dir) owns_ftp = ftp is None if owns_ftp: @@ -196,7 +196,7 @@ def download_assembly_to_local( for filename in target_files: last_activity = _download_and_verify(ftp, filename, dest_dir, md5_checksums, stats, last_activity) - logger.info(" %s: %d files downloaded", accession, stats["files_downloaded"]) + logger.debug(" %s: %d files downloaded", accession, stats["files_downloaded"]) finally: if owns_ftp: diff --git a/src/cdm_data_loaders/ncbi_ftp/metadata.py b/src/cdm_data_loaders/ncbi_ftp/metadata.py index 42cfe865..43a9c04f 100644 --- a/src/cdm_data_loaders/ncbi_ftp/metadata.py +++ b/src/cdm_data_loaders/ncbi_ftp/metadata.py @@ -194,7 +194,7 @@ def upload_descriptor( try: s3.upload_file(Filename=tmp_path, Bucket=bucket, Key=key) - logger.info("Uploaded descriptor: s3://%s/%s", bucket, key) + logger.debug("Uploaded descriptor: s3://%s/%s", bucket, key) finally: Path(tmp_path).unlink() diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index 2a127f19..f449ee7b 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -14,6 +14,7 @@ from typing import Any import botocore.exceptions +import tqdm from cdm_data_loaders.ncbi_ftp.metadata import ( DescriptorResource, @@ -113,9 +114,8 @@ def promote_from_s3( # noqa: PLR0913 # Upload frictionless descriptors for each promoted assembly descriptors_written = 0 - for (adir, acc), resources in assembly_resources.items(): - if not resources: - continue + non_empty = [(k, v) for k, v in assembly_resources.items() if v] + for (adir, acc), resources in tqdm.tqdm(non_empty, unit="descriptor", desc="Writing descriptors"): try: descriptor = create_descriptor(adir, acc, resources) upload_descriptor(descriptor, adir, lakehouse_bucket, lakehouse_key_prefix, dry_run=dry_run) @@ -167,7 +167,7 @@ def _promote_data_files( # noqa: PLR0913, PLR0915 promoted_accessions: set[str] = set() assembly_resources: defaultdict[tuple[str, str], list[DescriptorResource]] = defaultdict(list) - for staged_key in data_files: + for staged_key in tqdm.tqdm(data_files, unit="file", desc="Promoting"): if staged_key.endswith("download_report.json"): continue @@ -177,7 +177,7 @@ def _promote_data_files( # noqa: PLR0913, PLR0915 final_key = lakehouse_key_prefix + rel_path if dry_run: - logger.info("[dry-run] would promote: %s -> %s", staged_key, final_key) + logger.debug("[dry-run] would promote: %s -> %s", staged_key, final_key) promoted += 1 continue @@ -200,6 +200,7 @@ def _promote_data_files( # noqa: PLR0913, PLR0915 f"{lakehouse_bucket}/{final_key_path.parent}", metadata=metadata, object_name=final_key_path.name, + show_progress=False, ) if not upload_succeeded: logger.error("Failed to upload promoted file %s to %s", staged_key, final_key) @@ -276,7 +277,7 @@ def _archive_assemblies( # noqa: PLR0913 with Path(manifest_local_path).open() as f: accessions = [line.strip() for line in f if line.strip()] - for accession in accessions: + for accession in tqdm.tqdm(accessions, unit="accession", desc="Archiving"): m = re.match(r"(GC[AF])_(\d{3})(\d{3})(\d{3})\.\d+", accession) if not m: logger.warning("Cannot parse accession for archival: %s", accession) diff --git a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py index 233c82b1..9c628942 100644 --- a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py +++ b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py @@ -15,6 +15,7 @@ from pathlib import Path from typing import Any +import tqdm from pydantic import AliasChoices, Field from tenacity import before_sleep_log, retry, retry_if_exception_type, stop_after_attempt, wait_fixed from pydantic_settings import BaseSettings, SettingsConfigDict @@ -126,14 +127,16 @@ def _attempt() -> dict[str, Any]: return path, None try: - with ThreadPoolExecutor(max_workers=threads) as executor: - futures = {executor.submit(_download_one, p): p for p in assembly_paths} - for future in as_completed(futures): - path, error = future.result() - if error: - logger.error("FAILED: %s: %s", path, error) - with lock: - failed.append({"path": path, "error": str(error)}) + with tqdm.tqdm(total=len(assembly_paths), unit="assembly", desc="Downloading from NCBI FTP") as pbar: + with ThreadPoolExecutor(max_workers=threads) as executor: + futures = {executor.submit(_download_one, p): p for p in assembly_paths} + for future in as_completed(futures): + path, error = future.result() + if error: + logger.error("FAILED: %s: %s", path, error) + with lock: + failed.append({"path": path, "error": str(error)}) + pbar.update(1) finally: pool.close_all() @@ -263,13 +266,15 @@ def download_and_stage( def _upload(task: tuple[Path, str]) -> None: local_file, dest = task - upload_file(local_file, dest) - - with ThreadPoolExecutor(max_workers=threads) as executor: - futures = [executor.submit(_upload, t) for t in upload_tasks] - for future in as_completed(futures): - future.result() - staged_objects += 1 + upload_file(local_file, dest, show_progress=False) + + with tqdm.tqdm(total=len(upload_tasks), unit="file", desc="Staging to S3") as pbar: + with ThreadPoolExecutor(max_workers=threads) as executor: + futures = [executor.submit(_upload, t) for t in upload_tasks] + for future in as_completed(futures): + future.result() + staged_objects += 1 + pbar.update(1) logger.info("Staged %d objects to s3://%s/%s", staged_objects, bucket, staging_key_prefix) diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index c010404f..07c9a47d 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -201,6 +201,7 @@ def upload_file( destination_dir: str, object_name: str | None = None, metadata: dict[str, str] | None = None, + show_progress: bool = True, ) -> bool: """Upload an object to an S3 bucket. @@ -232,7 +233,7 @@ def upload_file( s3_path = f"{destination_dir.removesuffix('/')}/{object_name}" if metadata is None and object_exists(s3_path): - logger.info("File already present: %s", s3_path) + logger.debug("File already present: %s", s3_path) return True s3 = get_s3_client() @@ -241,21 +242,29 @@ def upload_file( extra_args = {**DEFAULT_EXTRA_ARGS, **(({"Metadata": metadata}) if metadata is not None else {})} # Upload the file - file_size = local_file_path.stat().st_size - with tqdm.tqdm(total=file_size, unit="B", unit_scale=True, desc=str(local_file_path)) as pbar: - logger.info("uploading %s to %s", str(local_file_path), s3_path) - try: + logger.debug("uploading %s to %s", str(local_file_path), s3_path) + try: + if show_progress: + file_size = local_file_path.stat().st_size + with tqdm.tqdm(total=file_size, unit="B", unit_scale=True, desc=str(local_file_path)) as pbar: + s3.upload_file( + Filename=str(local_file_path), + Bucket=bucket, + Key=key, + Callback=pbar.update, + ExtraArgs=extra_args, + ) + else: s3.upload_file( Filename=str(local_file_path), Bucket=bucket, Key=key, - Callback=pbar.update, ExtraArgs=extra_args, ) - except Exception as e: # noqa: BLE001 - logger.exception("Error uploading to s3") - return False - return True + except Exception as e: # noqa: BLE001 + logger.exception("Error uploading to s3") + return False + return True def stream_to_s3(url: str, s3_path: str, requests: ModuleType) -> str: @@ -287,7 +296,9 @@ def stream_to_s3(url: str, s3_path: str, requests: ModuleType) -> str: return f"{bucket}/{key}" -def download_file(s3_path: str, local_file_path: str | Path, version_id: str | None = None) -> None: +def download_file( + s3_path: str, local_file_path: str | Path, version_id: str | None = None, show_progress: bool = True +) -> None: """Download an object from s3. WARNING: will overwrite existing files but will not overwrite a file whilst trying to make a directory @@ -333,13 +344,21 @@ def download_file(s3_path: str, local_file_path: str | Path, version_id: str | N # set ``unit_scale=True`` so tqdm uses SI unit prefixes # ``unit="B"`` means it adds the string "B" as a suffix # progress is reported as (e.g.) "14.5kB/s". - with tqdm.tqdm(total=object_size, unit="B", unit_scale=True, desc=str(local_file_path)) as pbar: + if show_progress: + with tqdm.tqdm(total=object_size, unit="B", unit_scale=True, desc=str(local_file_path)) as pbar: + s3.download_file( + Bucket=bucket, + Key=key, + ExtraArgs=extra_args, + Filename=str(local_file_path), + Callback=pbar.update, + ) + else: s3.download_file( Bucket=bucket, Key=key, ExtraArgs=extra_args, Filename=str(local_file_path), - Callback=pbar.update, ) diff --git a/tests/integration/test_promote_e2e.py b/tests/integration/test_promote_e2e.py index 67778c64..7e28c796 100644 --- a/tests/integration/test_promote_e2e.py +++ b/tests/integration/test_promote_e2e.py @@ -145,7 +145,9 @@ def test_promote_idempotent(self, minio_s3_client: object, test_bucket: str, sta class TestPromoteArchiveUpdated: """Archive existing assemblies before overwriting with updated versions.""" - def test_archive_updated(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path) -> None: + def test_archive_updated( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + ) -> None: """Updated assemblies are archived before being overwritten.""" s3 = minio_s3_client @@ -190,7 +192,9 @@ def test_archive_updated(self, minio_s3_client: object, test_bucket: str, stagin class TestPromoteArchiveRemoved: """Archive and delete replaced/suppressed assemblies.""" - def test_archive_removed(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path) -> None: + def test_archive_removed( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + ) -> None: """Removed assemblies are archived and source objects are deleted.""" s3 = minio_s3_client @@ -261,7 +265,9 @@ def test_promote_dry_run(self, minio_s3_client: object, test_bucket: str, stagin class TestPromoteTrimsManifest: """Manifest trimming removes promoted accessions.""" - def test_trims_manifest(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path) -> None: + def test_trims_manifest( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + ) -> None: """Transfer manifest in MinIO is trimmed to exclude promoted accessions.""" s3 = minio_s3_client @@ -354,7 +360,9 @@ def test_descriptor_created(self, minio_s3_client: object, test_bucket: str, sta assert body["identifier"] == f"NCBI:{ACCESSION_A}" assert body["resource_type"] == "dataset" - def test_descriptor_resources_include_promoted_files(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: + def test_descriptor_resources_include_promoted_files( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + ) -> None: """Descriptor's ``resources`` list references the final Lakehouse key.""" s3 = minio_s3_client _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) @@ -373,7 +381,9 @@ def test_descriptor_resources_include_promoted_files(self, minio_s3_client: obje resource_paths = [r["path"] for r in body["resources"]] assert any(PATH_PREFIX + "raw_data/" in p for p in resource_paths) - def test_descriptor_resources_have_md5(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: + def test_descriptor_resources_have_md5( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + ) -> None: """Resources with .md5 sidecars include the hash value.""" s3 = minio_s3_client _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) @@ -393,7 +403,9 @@ def test_descriptor_resources_have_md5(self, minio_s3_client: object, test_bucke for resource in body["resources"]: assert "hash" in resource, f"Expected hash in resource: {resource}" - def test_multiple_assemblies_get_separate_descriptors(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: + def test_multiple_assemblies_get_separate_descriptors( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + ) -> None: """Each assembly gets its own descriptor file.""" s3 = minio_s3_client _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) @@ -418,7 +430,9 @@ def test_multiple_assemblies_get_separate_descriptors(self, minio_s3_client: obj class TestPromoteArchiveUpdatedIncludesDescriptor: """Archiving updated assemblies also archives the descriptor.""" - def test_archive_copies_descriptor(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path) -> None: + def test_archive_copies_descriptor( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + ) -> None: """After archiving an updated assembly, the descriptor appears under archive/.""" s3 = minio_s3_client @@ -454,7 +468,9 @@ def test_archive_copies_descriptor(self, minio_s3_client: object, test_bucket: s class TestPromoteArchiveRemovedIncludesDescriptor: """Archiving removed assemblies also archives the descriptor.""" - def test_archive_removed_copies_descriptor(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path) -> None: + def test_archive_removed_copies_descriptor( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + ) -> None: """After archiving a removed assembly, the descriptor is under archive/.""" s3 = minio_s3_client diff --git a/tests/ncbi_ftp/test_promote.py b/tests/ncbi_ftp/test_promote.py index 73777b9b..db19bcdb 100644 --- a/tests/ncbi_ftp/test_promote.py +++ b/tests/ncbi_ftp/test_promote.py @@ -31,7 +31,9 @@ def test_promote_dry_run_no_writes(mock_s3_client_no_checksum: botocore.client.B prefix = "staging/run1/" _stage_files(mock_s3_client_no_checksum, prefix) - report = promote_from_s3(staging_key_prefix=prefix, staging_bucket=TEST_BUCKET, lakehouse_bucket=TEST_BUCKET, dry_run=True) + report = promote_from_s3( + staging_key_prefix=prefix, staging_bucket=TEST_BUCKET, lakehouse_bucket=TEST_BUCKET, dry_run=True + ) assert report["promoted"] == 1 assert report["dry_run"] is True @@ -137,7 +139,11 @@ def test_archive_assemblies_updated_no_delete( assert ( _archive_assemblies( - str(manifest), lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-06", archive_reason="updated", delete_source=False + str(manifest), + lakehouse_bucket=TEST_BUCKET, + ncbi_release="2024-06", + archive_reason="updated", + delete_source=False, ) == 1 ) diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index 59287cd5..7e68acc7 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -393,16 +393,14 @@ def test_upload_file_uses_custom_object_name(mock_s3_client: Any, sample_file: P @pytest.mark.s3 -def test_upload_file_skips_when_already_present( - mock_s3_client: Any, sample_file: Path, caplog: pytest.LogCaptureFixture -) -> None: - """Verify that uploading a file that already exists is skipped and returns True.""" +def test_upload_file_skips_when_already_present(mock_s3_client: Any, sample_file: Path) -> None: + """Verify that uploading a file that already exists is skipped, returns True, and leaves the object unchanged.""" mock_s3_client.put_object(Bucket=CDM_LAKE_BUCKET, Key=f"uploads/{sample_file.name}", Body=b"old") result = upload_file(sample_file, f"{CDM_LAKE_BUCKET}/uploads") assert result is True - last_log_message = caplog.records[-1] - assert "File already present" in last_log_message.message - assert last_log_message.levelno == logging.INFO + # The existing object must not have been overwritten + obj = mock_s3_client.get_object(Bucket=CDM_LAKE_BUCKET, Key=f"uploads/{sample_file.name}") + assert obj["Body"].read() == b"old" @pytest.mark.usefixtures("mock_s3_client") From 0d3aa68d979ac589a4cdca2e1b102ea013fde4a7 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 29 Apr 2026 13:56:07 -0700 Subject: [PATCH 062/128] consolidate download with staging and promotion with deletion from staging and metadata generation Co-authored-by: Copilot --- src/cdm_data_loaders/ncbi_ftp/promote.py | 185 ++++++++++-------- .../pipelines/ncbi_ftp_download.py | 153 ++++++++++++--- tests/pipelines/test_ncbi_ftp_download.py | 108 ++++++---- 3 files changed, 303 insertions(+), 143 deletions(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index f449ee7b..10532cfd 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -98,7 +98,7 @@ def promote_from_s3( # noqa: PLR0913 dry_run=dry_run, ) - promoted, failed, promoted_accessions, assembly_resources = _promote_data_files( + promoted, failed, descriptors_written, promoted_accessions = _promote_data_files( data_files, sidecars, normalized_staging_key_prefix, @@ -112,17 +112,6 @@ def promote_from_s3( # noqa: PLR0913 if manifest_s3_key and promoted_accessions and not dry_run: _trim_manifest(manifest_s3_key, staging_bucket, promoted_accessions) - # Upload frictionless descriptors for each promoted assembly - descriptors_written = 0 - non_empty = [(k, v) for k, v in assembly_resources.items() if v] - for (adir, acc), resources in tqdm.tqdm(non_empty, unit="descriptor", desc="Writing descriptors"): - try: - descriptor = create_descriptor(adir, acc, resources) - upload_descriptor(descriptor, adir, lakehouse_bucket, lakehouse_key_prefix, dry_run=dry_run) - descriptors_written += 1 - except Exception: - logger.exception("Failed to write descriptor for %s", adir) - if descriptors_written: logger.info("Wrote %d frictionless descriptor(s)", descriptors_written) @@ -156,88 +145,128 @@ def _promote_data_files( # noqa: PLR0913, PLR0915 lakehouse_bucket: str, *, dry_run: bool, -) -> tuple[int, int, set[str], defaultdict[tuple[str, str], list[DescriptorResource]]]: +) -> tuple[int, int, int, set[str]]: """Promote each data file from staging to the final Lakehouse path. - :return: (promoted_count, failed_count, promoted_accessions, assembly_resources) + Files are grouped by assembly. When all files for an assembly are promoted + successfully, the frictionless descriptor is written immediately and the staged + files (including sidecars) are deleted from staging. This prevents staging + accumulation across runs and ensures partial runs leave descriptors for all + completed assemblies. + + :return: (promoted_count, failed_count, descriptors_written, promoted_accessions) """ s3 = get_s3_client() promoted = 0 failed = 0 + descriptors_written = 0 promoted_accessions: set[str] = set() - assembly_resources: defaultdict[tuple[str, str], list[DescriptorResource]] = defaultdict(list) - for staged_key in tqdm.tqdm(data_files, unit="file", desc="Promoting"): + # Group files by assembly; skip download_report.json and non-raw_data paths + assembly_files: defaultdict[tuple[str, str], list[str]] = defaultdict(list) + for staged_key in data_files: if staged_key.endswith("download_report.json"): continue - rel_path = staged_key[len(normalized_staging_prefix) :] if not rel_path.startswith("raw_data/"): continue - final_key = lakehouse_key_prefix + rel_path - - if dry_run: - logger.debug("[dry-run] would promote: %s -> %s", staged_key, final_key) - promoted += 1 - continue - - try: - with tempfile.NamedTemporaryFile(delete=False) as tmp: - tmp_path = tmp.name - try: - s3.download_file(Bucket=staging_bucket, Key=staged_key, Filename=tmp_path) - - # Read MD5 from sidecar - metadata: dict[str, str] = {} - md5_key = staged_key + ".md5" - if md5_key in sidecars: - md5_obj = s3.get_object(Bucket=staging_bucket, Key=md5_key) - metadata["md5"] = md5_obj["Body"].read().decode().strip() - + acc_match = re.search(r"(GC[AF]_\d{9}\.\d+)", staged_key) + adir_match = re.search(r"raw_data/GC[AF]/\d+/\d+/\d+/([^/]+)/", staged_key) + if acc_match and adir_match: + assembly_files[(adir_match.group(1), acc_match.group(1))].append(staged_key) + + total_files = sum(len(v) for v in assembly_files.values()) + with tqdm.tqdm(total=total_files, unit="file", desc="Promoting") as pbar: + for (adir, acc), files in assembly_files.items(): + assembly_failed = 0 + resources: list[DescriptorResource] = [] + promoted_keys: list[str] = [] + + for staged_key in files: + rel_path = staged_key[len(normalized_staging_prefix) :] + final_key = lakehouse_key_prefix + rel_path final_key_path = PurePosixPath(final_key) - upload_succeeded = upload_file( - tmp_path, - f"{lakehouse_bucket}/{final_key_path.parent}", - metadata=metadata, - object_name=final_key_path.name, - show_progress=False, - ) - if not upload_succeeded: - logger.error("Failed to upload promoted file %s to %s", staged_key, final_key) - failed += 1 + + if dry_run: + logger.debug("[dry-run] would promote: %s -> %s", staged_key, final_key) + promoted += 1 + pbar.update(1) continue - promoted += 1 - - # Track promoted accession for manifest trimming - acc_match = re.search(r"(GC[AF]_\d{9}\.\d+)", staged_key) - if acc_match: - promoted_accessions.add(acc_match.group(1)) - - # Track resources for frictionless descriptor creation - adir_match = re.search(r"raw_data/GC[AF]/\d+/\d+/\d+/([^/]+)/", final_key) - if adir_match and acc_match: - adir = adir_match.group(1) - acc = acc_match.group(1) - fname = final_key_path.name - ext = fname.rsplit(".", 1)[-1] if "." in fname else "" - md5_hash = metadata.get("md5") - resource: DescriptorResource = { - "name": fname.lower(), - "path": final_key, - "format": ext, - "bytes": Path(tmp_path).stat().st_size, - "hash": md5_hash, - } - assembly_resources[(adir, acc)].append(resource) - - finally: - Path(tmp_path).unlink() - except Exception: - logger.exception("Failed to promote %s", staged_key) - failed += 1 - - return promoted, failed, promoted_accessions, assembly_resources + file_promoted = False + try: + with tempfile.NamedTemporaryFile(delete=False) as tmp: + tmp_path = tmp.name + try: + s3.download_file(Bucket=staging_bucket, Key=staged_key, Filename=tmp_path) + + # Read MD5 from sidecar + metadata: dict[str, str] = {} + md5_key = staged_key + ".md5" + if md5_key in sidecars: + md5_obj = s3.get_object(Bucket=staging_bucket, Key=md5_key) + metadata["md5"] = md5_obj["Body"].read().decode().strip() + + upload_succeeded = upload_file( + tmp_path, + f"{lakehouse_bucket}/{final_key_path.parent}", + metadata=metadata, + object_name=final_key_path.name, + show_progress=False, + ) + if not upload_succeeded: + logger.error("Failed to upload promoted file %s to %s", staged_key, final_key) + else: + promoted += 1 + promoted_keys.append(staged_key) + promoted_accessions.add(acc) + file_promoted = True + + fname = final_key_path.name + ext = fname.rsplit(".", 1)[-1] if "." in fname else "" + resource: DescriptorResource = { + "name": fname.lower(), + "path": final_key, + "format": ext, + "bytes": Path(tmp_path).stat().st_size, + "hash": metadata.get("md5"), + } + resources.append(resource) + + finally: + Path(tmp_path).unlink() + except Exception: + logger.exception("Failed to promote %s", staged_key) + + if not file_promoted: + assembly_failed += 1 + pbar.update(1) + + failed += assembly_failed + + # Write descriptor and delete staged files immediately after a fully successful assembly + if assembly_failed == 0 and promoted_keys: + try: + descriptor = create_descriptor(adir, acc, resources) + upload_descriptor(descriptor, adir, lakehouse_bucket, lakehouse_key_prefix, dry_run=False) + descriptors_written += 1 + except Exception: + logger.exception("Failed to write descriptor for %s", adir) + + for staged_key in promoted_keys: + try: + delete_object(f"{staging_bucket}/{staged_key}") + except Exception: + logger.warning("Failed to delete staged file %s", staged_key) + for sidecar_ext in (".md5", ".crc64nvme"): + sidecar_key = staged_key + sidecar_ext + if sidecar_key in sidecars: + try: + delete_object(f"{staging_bucket}/{sidecar_key}") + except Exception: + logger.warning("Failed to delete staged sidecar %s", sidecar_key) + + return promoted, failed, descriptors_written, promoted_accessions # ── Archive assemblies ────────────────────────────────────────────────── diff --git a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py index 9c628942..57a73154 100644 --- a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py +++ b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py @@ -7,6 +7,7 @@ import json import logging +import shutil import tempfile import threading from concurrent.futures import ThreadPoolExecutor, as_completed @@ -20,7 +21,12 @@ from tenacity import before_sleep_log, retry, retry_if_exception_type, stop_after_attempt, wait_fixed from pydantic_settings import BaseSettings, SettingsConfigDict -from cdm_data_loaders.ncbi_ftp.assembly import FTP_HOST, download_assembly_to_local +from cdm_data_loaders.ncbi_ftp.assembly import ( + FTP_HOST, + build_accession_path, + download_assembly_to_local, + parse_assembly_path, +) from cdm_data_loaders.pipelines.core import run_cli from cdm_data_loaders.pipelines.cts_defaults import DEFAULT_SETTINGS_CONFIG_DICT, INPUT_MOUNT, OUTPUT_MOUNT from cdm_data_loaders.utils.cdm_logger import get_cdm_logger @@ -70,6 +76,41 @@ class DownloadSettings(BaseSettings): ) +# ── Private helpers ───────────────────────────────────────────────────── + + +def _upload_assembly_dir( + assembly_dir: Path, + tmp_root: Path, + bucket: str, + staging_key_prefix: str, +) -> int: + """Upload all files under *assembly_dir* to S3, deleting each file immediately after upload. + + Empty directories are removed after all files are uploaded. If the + directory does not exist (e.g. the assembly had no files) the function + returns zero without raising. + + :param assembly_dir: local directory for one assembly + :param tmp_root: root of the temp directory (used to compute relative S3 paths) + :param bucket: destination S3 bucket + :param staging_key_prefix: S3 key prefix within *bucket* + :return: number of files uploaded + """ + if not assembly_dir.exists(): + return 0 + count = 0 + for f in sorted(assembly_dir.rglob("*")): + if f.is_file(): + relative = f.relative_to(tmp_root) + dest_prefix = f"{bucket}/{staging_key_prefix.rstrip('/')}/{relative.parent}" + upload_file(f, dest_prefix, show_progress=False) + f.unlink() + count += 1 + shutil.rmtree(assembly_dir, ignore_errors=True) + return count + + # ── Batch download ─────────────────────────────────────────────────────── @@ -208,13 +249,18 @@ def download_and_stage( Exactly one of *manifest_s3_key* or *manifest_local_path* must be given. + Downloads and uploads are pipelined per assembly: each worker downloads one + assembly, immediately uploads its files to S3, then deletes the local copies + before picking up the next assembly. At most *threads* assembly directories + exist on disk simultaneously, preventing disk exhaustion on large batches. + :param bucket: destination S3 bucket name :param staging_key_prefix: key prefix inside the bucket (e.g. ``"staging/run1/"``) :param manifest_s3_key: S3 object key of the transfer manifest within *bucket* :param manifest_local_path: local path to the transfer manifest file - :param threads: number of parallel download **and** upload threads + :param threads: number of parallel download-and-upload workers :param ftp_host: NCBI FTP hostname - :param limit: optional limit for testing (pass to :func:`download_batch`) + :param limit: optional limit for testing :param dry_run: when ``True``, download but skip all S3 uploads :return: download report extended with ``staged_objects``, ``staging_key_prefix``, ``dry_run`` """ @@ -238,46 +284,89 @@ def download_and_stage( manifest_dest.write_bytes(Path(manifest_local_path).read_bytes()) logger.info("Manifest read from local path: %s", manifest_local_path) - report = download_batch( - manifest_path=manifest_dest, - output_dir=tmp, - threads=threads, - ftp_host=ftp_host, - limit=limit, - ) + with manifest_dest.open() as f: + assembly_paths = [line.strip() for line in f if line.strip() and not line.startswith("#")] - staged_objects = 0 + if limit: + assembly_paths = assembly_paths[:limit] - if not dry_run: - raw_data_dir = tmp / "raw_data" - report_json = tmp / "download_report.json" + logger.info("Starting download & stage of %d assemblies with %d threads", len(assembly_paths), threads) - upload_tasks: list[tuple[Path, str]] = [] - - if raw_data_dir.exists(): - for local_file in sorted(raw_data_dir.rglob("*")): - if local_file.is_file(): - relative = local_file.relative_to(tmp) - dest_prefix = f"{bucket}/{staging_key_prefix.rstrip('/')}/{relative.parent}" - upload_tasks.append((local_file, dest_prefix)) - - if report_json.exists(): - upload_tasks.append((report_json, f"{bucket}/{staging_key_prefix.rstrip('/')}")) + pool = ThreadLocalFTP(ftp_host) + lock = threading.Lock() + success_count = 0 + staged_objects = 0 + failed: list[dict[str, str]] = [] + all_stats: list[dict[str, Any]] = [] + + def _download_upload_one(path: str) -> tuple[str, Exception | None]: + nonlocal success_count, staged_objects + + @retry( + retry=retry_if_exception_type(error_temp), + stop=stop_after_attempt(3), + wait=wait_fixed(5), + reraise=True, + before_sleep=before_sleep_log(logger, logging.WARNING), + ) + def _attempt() -> dict[str, Any]: + return download_assembly_to_local(path, tmp, ftp_host=ftp_host, ftp=pool.get()) + + try: + stats = _attempt() + except Exception as e: # noqa: BLE001 + return path, e + + if not dry_run: + _db, assembly_dir_name, _accession = parse_assembly_path(path) + assembly_local_dir = tmp / build_accession_path(assembly_dir_name) + count = _upload_assembly_dir(assembly_local_dir, tmp, bucket, staging_key_prefix) + with lock: + staged_objects += count - def _upload(task: tuple[Path, str]) -> None: - local_file, dest = task - upload_file(local_file, dest, show_progress=False) + with lock: + success_count += 1 + all_stats.append(stats) + return path, None - with tqdm.tqdm(total=len(upload_tasks), unit="file", desc="Staging to S3") as pbar: + try: + with tqdm.tqdm(total=len(assembly_paths), unit="assembly", desc="Downloading & staging") as pbar: with ThreadPoolExecutor(max_workers=threads) as executor: - futures = [executor.submit(_upload, t) for t in upload_tasks] + futures = {executor.submit(_download_upload_one, p): p for p in assembly_paths} for future in as_completed(futures): - future.result() - staged_objects += 1 + path, error = future.result() + if error: + logger.error("FAILED: %s: %s", path, error) + with lock: + failed.append({"path": path, "error": str(error)}) pbar.update(1) + finally: + pool.close_all() + + report: dict[str, Any] = { + "timestamp": datetime.now(UTC).isoformat(), + "total_attempted": len(assembly_paths), + "succeeded": success_count, + "failed": len(failed), + "failures": failed, + "assembly_stats": all_stats, + } + if not dry_run: + report_path = tmp / "download_report.json" + with report_path.open("w") as f: + json.dump(report, f, indent=2) + upload_file(report_path, f"{bucket}/{staging_key_prefix.rstrip('/')}", show_progress=False) + staged_objects += 1 logger.info("Staged %d objects to s3://%s/%s", staged_objects, bucket, staging_key_prefix) + logger.info( + "SUMMARY: %d attempted, %d succeeded, %d failed", + len(assembly_paths), + success_count, + len(failed), + ) + return { **report, "staged_objects": staged_objects, diff --git a/tests/pipelines/test_ncbi_ftp_download.py b/tests/pipelines/test_ncbi_ftp_download.py index cadfe559..7c45b476 100644 --- a/tests/pipelines/test_ncbi_ftp_download.py +++ b/tests/pipelines/test_ncbi_ftp_download.py @@ -18,6 +18,21 @@ ) from cdm_data_loaders.utils.s3 import reset_s3_client +_MOCK_STATS = { + "accession": "GCF_000001215.4", + "assembly_dir": "GCF_000001215.4_Release_6_plus_ISO1_MT", + "files_downloaded": 0, + "files_skipped_checksum_mismatch": 0, + "files_without_checksum": 0, +} +_MOCK_STATS_2 = { + "accession": "GCF_000001405.40", + "assembly_dir": "GCF_000001405.40_GRCh38.p14", + "files_downloaded": 0, + "files_skipped_checksum_mismatch": 0, + "files_without_checksum": 0, +} + _DEFAULT_THREADS = 4 _CUSTOM_THREADS = 8 _ALIAS_THREADS = 16 @@ -279,7 +294,7 @@ def test_download_and_stage_manifest_source( manifest_s3_key: str | None, use_local: bool, ) -> None: - """Manifest lines are passed to download_batch regardless of source (S3 or local).""" + """Assembly paths from the manifest are processed regardless of source (S3 or local).""" reset_s3_client() s3 = _make_moto_s3() @@ -290,11 +305,11 @@ def test_download_and_stage_manifest_source( manifest_local = tmp_path / "manifest.txt" manifest_local.write_text(_MANIFEST_CONTENT) - captured_content: list[str] = [] + called_paths: list[str] = [] - def _capturing_batch(manifest_path, output_dir, **kwargs): # noqa: ARG001 - captured_content.append(Path(manifest_path).read_text()) - return _MOCK_REPORT + def _fake_download(path, output_dir, **kwargs): # noqa: ARG001 + called_paths.append(path) + return _MOCK_STATS import cdm_data_loaders.utils.s3 as s3_mod @@ -302,7 +317,11 @@ def _capturing_batch(manifest_path, output_dir, **kwargs): # noqa: ARG001 patch.object(s3_mod, "get_s3_client", return_value=s3), patch.object(s3_mod, "_s3_client", s3), patch("cdm_data_loaders.pipelines.ncbi_ftp_download.get_s3_client", return_value=s3), - patch("cdm_data_loaders.pipelines.ncbi_ftp_download.download_batch", side_effect=_capturing_batch), + patch("cdm_data_loaders.pipelines.ncbi_ftp_download.ThreadLocalFTP"), + patch( + "cdm_data_loaders.pipelines.ncbi_ftp_download.download_assembly_to_local", + side_effect=_fake_download, + ), ): download_and_stage( bucket=_TEST_BUCKET, @@ -310,9 +329,11 @@ def _capturing_batch(manifest_path, output_dir, **kwargs): # noqa: ARG001 manifest_s3_key=manifest_s3_key, manifest_local_path=manifest_local, dry_run=True, + threads=1, ) - assert captured_content == [_MANIFEST_CONTENT] + expected_paths = [l for l in _MANIFEST_CONTENT.splitlines() if l.strip()] + assert sorted(called_paths) == sorted(expected_paths) reset_s3_client() @@ -363,7 +384,11 @@ def test_download_and_stage_exactly_one_source_required( patch.object(s3_mod, "get_s3_client", return_value=s3), patch.object(s3_mod, "_s3_client", s3), patch("cdm_data_loaders.pipelines.ncbi_ftp_download.get_s3_client", return_value=s3), - patch("cdm_data_loaders.pipelines.ncbi_ftp_download.download_batch", return_value=_MOCK_REPORT), + patch("cdm_data_loaders.pipelines.ncbi_ftp_download.ThreadLocalFTP"), + patch( + "cdm_data_loaders.pipelines.ncbi_ftp_download.download_assembly_to_local", + return_value=_MOCK_STATS, + ), ): result = download_and_stage( bucket=_TEST_BUCKET, @@ -372,7 +397,7 @@ def test_download_and_stage_exactly_one_source_required( manifest_local_path=local_path, dry_run=True, ) - assert result["succeeded"] == _MOCK_REPORT["succeeded"] + assert result["succeeded"] == _EXPECTED_ATTEMPTED reset_s3_client() @@ -382,24 +407,22 @@ def test_download_and_stage_exactly_one_source_required( @mock_aws def test_download_and_stage_uploads_to_staging(tmp_path: Path) -> None: - """Files produced in raw_data/ and download_report.json are all uploaded to staging.""" + """Files produced by download_assembly_to_local and download_report.json are all staged to S3.""" reset_s3_client() s3 = _make_moto_s3() manifest_local = tmp_path / "manifest.txt" - manifest_local.write_text(_MANIFEST_CONTENT) + # Single assembly so the fake download writes exactly the files we expect + manifest_local.write_text("/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/\n") assembly_rel = "raw_data/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT" - def _fake_download_batch(manifest_path, output_dir, **kwargs): # noqa: ARG001 - out = Path(output_dir) - asm_dir = out / assembly_rel + def _fake_download(path, output_dir, **kwargs): # noqa: ARG001 + asm_dir = Path(output_dir) / assembly_rel asm_dir.mkdir(parents=True) (asm_dir / "genomic.fna.gz").write_bytes(b"fasta_data") (asm_dir / "genomic.fna.gz.md5").write_bytes(b"abc123") - report_path = out / "download_report.json" - report_path.write_text(json.dumps(_MOCK_REPORT)) - return _MOCK_REPORT + return {**_MOCK_STATS, "files_downloaded": 2} import cdm_data_loaders.utils.s3 as s3_mod @@ -407,7 +430,11 @@ def _fake_download_batch(manifest_path, output_dir, **kwargs): # noqa: ARG001 patch.object(s3_mod, "get_s3_client", return_value=s3), patch.object(s3_mod, "_s3_client", s3), patch("cdm_data_loaders.pipelines.ncbi_ftp_download.get_s3_client", return_value=s3), - patch("cdm_data_loaders.pipelines.ncbi_ftp_download.download_batch", side_effect=_fake_download_batch), + patch("cdm_data_loaders.pipelines.ncbi_ftp_download.ThreadLocalFTP"), + patch( + "cdm_data_loaders.pipelines.ncbi_ftp_download.download_assembly_to_local", + side_effect=_fake_download, + ), ): report = download_and_stage( bucket=_TEST_BUCKET, @@ -443,11 +470,11 @@ def test_download_and_stage_dry_run_skips_upload(tmp_path: Path) -> None: manifest_local = tmp_path / "manifest.txt" manifest_local.write_text(_MANIFEST_CONTENT) - def _fake_download_batch(manifest_path, output_dir, **kwargs): # noqa: ARG001 + def _fake_download(path, output_dir, **kwargs): # noqa: ARG001 asm_dir = Path(output_dir) / "raw_data/GCF/000/001/215/GCF_000001215.4" - asm_dir.mkdir(parents=True) + asm_dir.mkdir(parents=True, exist_ok=True) (asm_dir / "genomic.fna.gz").write_bytes(b"fasta") - return _MOCK_REPORT + return _MOCK_STATS import cdm_data_loaders.utils.s3 as s3_mod @@ -455,7 +482,11 @@ def _fake_download_batch(manifest_path, output_dir, **kwargs): # noqa: ARG001 patch.object(s3_mod, "get_s3_client", return_value=s3), patch.object(s3_mod, "_s3_client", s3), patch("cdm_data_loaders.pipelines.ncbi_ftp_download.get_s3_client", return_value=s3), - patch("cdm_data_loaders.pipelines.ncbi_ftp_download.download_batch", side_effect=_fake_download_batch), + patch("cdm_data_loaders.pipelines.ncbi_ftp_download.ThreadLocalFTP"), + patch( + "cdm_data_loaders.pipelines.ncbi_ftp_download.download_assembly_to_local", + side_effect=_fake_download, + ), ): report = download_and_stage( bucket=_TEST_BUCKET, @@ -485,7 +516,7 @@ def _fake_download_batch(manifest_path, output_dir, **kwargs): # noqa: ARG001 ) @mock_aws def test_download_and_stage_limit_forwarded(tmp_path: Path, limit: int) -> None: - """The limit parameter is forwarded verbatim to download_batch.""" + """The limit parameter truncates the number of assemblies processed.""" reset_s3_client() s3 = _make_moto_s3() @@ -498,7 +529,11 @@ def test_download_and_stage_limit_forwarded(tmp_path: Path, limit: int) -> None: patch.object(s3_mod, "get_s3_client", return_value=s3), patch.object(s3_mod, "_s3_client", s3), patch("cdm_data_loaders.pipelines.ncbi_ftp_download.get_s3_client", return_value=s3), - patch("cdm_data_loaders.pipelines.ncbi_ftp_download.download_batch", return_value=_MOCK_REPORT) as mock_batch, + patch("cdm_data_loaders.pipelines.ncbi_ftp_download.ThreadLocalFTP"), + patch( + "cdm_data_loaders.pipelines.ncbi_ftp_download.download_assembly_to_local", + return_value=_MOCK_STATS, + ) as mock_dl, ): download_and_stage( bucket=_TEST_BUCKET, @@ -508,7 +543,9 @@ def test_download_and_stage_limit_forwarded(tmp_path: Path, limit: int) -> None: dry_run=True, ) - assert mock_batch.call_args.kwargs["limit"] == limit + # The manifest has 2 entries; limit caps how many were processed + expected_calls = min(limit, _EXPECTED_ATTEMPTED) + assert mock_dl.call_count == expected_calls reset_s3_client() @@ -518,7 +555,7 @@ def test_download_and_stage_limit_forwarded(tmp_path: Path, limit: int) -> None: @mock_aws def test_download_and_stage_report_shape(tmp_path: Path) -> None: - """Return value includes all download_batch keys plus staged_objects, staging_key_prefix, dry_run.""" + """Return value contains all expected keys including staged_objects, staging_key_prefix, dry_run.""" reset_s3_client() s3 = _make_moto_s3() @@ -531,7 +568,11 @@ def test_download_and_stage_report_shape(tmp_path: Path) -> None: patch.object(s3_mod, "get_s3_client", return_value=s3), patch.object(s3_mod, "_s3_client", s3), patch("cdm_data_loaders.pipelines.ncbi_ftp_download.get_s3_client", return_value=s3), - patch("cdm_data_loaders.pipelines.ncbi_ftp_download.download_batch", return_value=_MOCK_REPORT), + patch("cdm_data_loaders.pipelines.ncbi_ftp_download.ThreadLocalFTP"), + patch( + "cdm_data_loaders.pipelines.ncbi_ftp_download.download_assembly_to_local", + return_value=_MOCK_STATS, + ), ): report = download_and_stage( bucket=_TEST_BUCKET, @@ -540,11 +581,12 @@ def test_download_and_stage_report_shape(tmp_path: Path) -> None: dry_run=True, ) - assert report == { - **_MOCK_REPORT, - "staged_objects": 0, - "staging_key_prefix": _STAGING_PREFIX, - "dry_run": True, - } + for key in ("timestamp", "total_attempted", "succeeded", "failed", "failures", "assembly_stats"): + assert key in report + assert report["staged_objects"] == 0 + assert report["staging_key_prefix"] == _STAGING_PREFIX + assert report["dry_run"] is True + assert report["total_attempted"] == _EXPECTED_ATTEMPTED + assert report["succeeded"] == _EXPECTED_ATTEMPTED reset_s3_client() From c823283ffb3ec94c499f766c754eb35080e42de9 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 29 Apr 2026 13:58:56 -0700 Subject: [PATCH 063/128] Potential fix for pull request finding 'Unused global variable' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- tests/pipelines/test_ncbi_ftp_download.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/pipelines/test_ncbi_ftp_download.py b/tests/pipelines/test_ncbi_ftp_download.py index 7c45b476..6c016804 100644 --- a/tests/pipelines/test_ncbi_ftp_download.py +++ b/tests/pipelines/test_ncbi_ftp_download.py @@ -25,13 +25,6 @@ "files_skipped_checksum_mismatch": 0, "files_without_checksum": 0, } -_MOCK_STATS_2 = { - "accession": "GCF_000001405.40", - "assembly_dir": "GCF_000001405.40_GRCh38.p14", - "files_downloaded": 0, - "files_skipped_checksum_mismatch": 0, - "files_without_checksum": 0, -} _DEFAULT_THREADS = 4 _CUSTOM_THREADS = 8 From cf930134c16f616e9386eb7ef660fb4c28b63c85 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 29 Apr 2026 13:59:12 -0700 Subject: [PATCH 064/128] Potential fix for pull request finding 'Unused global variable' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- tests/pipelines/test_ncbi_ftp_download.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/pipelines/test_ncbi_ftp_download.py b/tests/pipelines/test_ncbi_ftp_download.py index 6c016804..99a9ebfd 100644 --- a/tests/pipelines/test_ncbi_ftp_download.py +++ b/tests/pipelines/test_ncbi_ftp_download.py @@ -254,15 +254,6 @@ def test_handles_download_failure(self, tmp_path: Path) -> None: _TEST_BUCKET = "test-bucket" _STAGING_PREFIX = "staging/run1/" -_MOCK_REPORT = { - "timestamp": "2026-01-01T00:00:00+00:00", - "total_attempted": 2, - "succeeded": 2, - "failed": 0, - "failures": [], - "assembly_stats": [], -} - def _make_moto_s3(): """Return a moto-backed S3 client with the test bucket created.""" From 2b45416831ff05e4b7e1ba75f62bc21098f2eea6 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 29 Apr 2026 14:13:18 -0700 Subject: [PATCH 065/128] fix tests --- src/cdm_data_loaders/ncbi_ftp/promote.py | 12 +++++++++--- tests/integration/test_promote_e2e.py | 10 ++++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index 10532cfd..3be925fe 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -19,6 +19,7 @@ from cdm_data_loaders.ncbi_ftp.metadata import ( DescriptorResource, archive_descriptor, + build_descriptor_key, create_descriptor, upload_descriptor, ) @@ -27,6 +28,7 @@ copy_object, delete_object, get_s3_client, + object_exists, upload_file, ) @@ -247,9 +249,13 @@ def _promote_data_files( # noqa: PLR0913, PLR0915 # Write descriptor and delete staged files immediately after a fully successful assembly if assembly_failed == 0 and promoted_keys: try: - descriptor = create_descriptor(adir, acc, resources) - upload_descriptor(descriptor, adir, lakehouse_bucket, lakehouse_key_prefix, dry_run=False) - descriptors_written += 1 + descriptor_key = build_descriptor_key(adir, lakehouse_key_prefix) + if object_exists(f"{lakehouse_bucket}/{descriptor_key}"): + logger.debug("Descriptor already exists, skipping: %s", descriptor_key) + else: + descriptor = create_descriptor(adir, acc, resources) + upload_descriptor(descriptor, adir, lakehouse_bucket, lakehouse_key_prefix, dry_run=False) + descriptors_written += 1 except Exception: logger.exception("Failed to write descriptor for %s", adir) diff --git a/tests/integration/test_promote_e2e.py b/tests/integration/test_promote_e2e.py index 7e28c796..ed3f5dd1 100644 --- a/tests/integration/test_promote_e2e.py +++ b/tests/integration/test_promote_e2e.py @@ -114,7 +114,12 @@ class TestPromoteIdempotent: """Promoting the same staging data twice should succeed without errors.""" def test_promote_idempotent(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: - """Second promote succeeds and produces the same final state.""" + """Second promote on empty staging succeeds and leaves the lakehouse unchanged. + + After the first promote, staged files are deleted. A second run therefore + finds nothing to promote — which is correct and expected. The lakehouse + contents must be identical after both runs. + """ s3 = minio_s3_client _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) @@ -135,8 +140,9 @@ def test_promote_idempotent(self, minio_s3_client: object, test_bucket: str, sta keys_after_second = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") assert report1["failed"] == 0 + assert report1["promoted"] >= 1 assert report2["failed"] == 0 - assert report2["promoted"] >= 1 + assert report2["promoted"] == 0 # staging was cleared by the first run assert keys_after_first == keys_after_second From f3c09842d13c2294367688ca549d3788cd7e6918 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 29 Apr 2026 15:16:12 -0700 Subject: [PATCH 066/128] keep ftp connection alive Co-authored-by: Copilot --- .../pipelines/ncbi_ftp_download.py | 14 ++++++------- src/cdm_data_loaders/utils/ftp_client.py | 20 ++++++++++++++++++- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py index 57a73154..4a2597df 100644 --- a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py +++ b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py @@ -18,7 +18,7 @@ import tqdm from pydantic import AliasChoices, Field -from tenacity import before_sleep_log, retry, retry_if_exception_type, stop_after_attempt, wait_fixed +from tenacity import before_sleep_log, retry, retry_if_exception_type, stop_after_attempt, wait_fixed, wait_exponential from pydantic_settings import BaseSettings, SettingsConfigDict from cdm_data_loaders.ncbi_ftp.assembly import ( @@ -148,9 +148,9 @@ def _download_one(path: str) -> tuple[str, Exception | None]: nonlocal success_count @retry( - retry=retry_if_exception_type(error_temp), - stop=stop_after_attempt(3), - wait=wait_fixed(5), + retry=retry_if_exception_type((error_temp, BrokenPipeError, ConnectionResetError, EOFError)), + stop=stop_after_attempt(5), + wait=wait_exponential(multiplier=1, min=5, max=60), reraise=True, before_sleep=before_sleep_log(logger, logging.WARNING), ) @@ -303,9 +303,9 @@ def _download_upload_one(path: str) -> tuple[str, Exception | None]: nonlocal success_count, staged_objects @retry( - retry=retry_if_exception_type(error_temp), - stop=stop_after_attempt(3), - wait=wait_fixed(5), + retry=retry_if_exception_type((error_temp, BrokenPipeError, ConnectionResetError, EOFError)), + stop=stop_after_attempt(5), + wait=wait_exponential(multiplier=1, min=5, max=60), reraise=True, before_sleep=before_sleep_log(logger, logging.WARNING), ) diff --git a/src/cdm_data_loaders/utils/ftp_client.py b/src/cdm_data_loaders/utils/ftp_client.py index 902779e5..bd372924 100644 --- a/src/cdm_data_loaders/utils/ftp_client.py +++ b/src/cdm_data_loaders/utils/ftp_client.py @@ -153,8 +153,26 @@ def __init__(self, host: str, timeout: int = DEFAULT_TIMEOUT) -> None: self._connections: list[FTP] = [] def get(self) -> FTP: - """Return the FTP connection for the current thread, creating one if needed.""" + """Return the FTP connection for the current thread, reconnecting if stale. + + Sends a NOOP to verify the connection is still alive before returning it. + If the server has closed the connection (e.g. after an idle timeout or + session limit), the dead socket is discarded and a fresh connection is + established transparently. + """ ftp = getattr(self._local, "ftp", None) + if ftp is not None: + try: + ftp.voidcmd("NOOP") + except Exception: + # Connection is stale — discard it and reconnect below + with contextlib.suppress(Exception): + ftp.quit() + with self._lock: + with contextlib.suppress(ValueError): + self._connections.remove(ftp) + ftp = None + self._local.ftp = None if ftp is None: ftp = connect_ftp(self._host, self._timeout) self._local.ftp = ftp From b6f0bddcc3edb1a2f401ceff14ee326db027980c Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 29 Apr 2026 15:21:45 -0700 Subject: [PATCH 067/128] Potential fix for pull request finding 'Unused import' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- src/cdm_data_loaders/pipelines/ncbi_ftp_download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py index 4a2597df..2cd6a0f4 100644 --- a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py +++ b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py @@ -18,7 +18,7 @@ import tqdm from pydantic import AliasChoices, Field -from tenacity import before_sleep_log, retry, retry_if_exception_type, stop_after_attempt, wait_fixed, wait_exponential +from tenacity import before_sleep_log, retry, retry_if_exception_type, stop_after_attempt, wait_exponential from pydantic_settings import BaseSettings, SettingsConfigDict from cdm_data_loaders.ncbi_ftp.assembly import ( From 5fcada9b48241fe9ae6caec1381908d0d2f64168 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 29 Apr 2026 15:41:14 -0700 Subject: [PATCH 068/128] smooth progress bars --- src/cdm_data_loaders/pipelines/ncbi_ftp_download.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py index 2cd6a0f4..062575f6 100644 --- a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py +++ b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py @@ -168,7 +168,9 @@ def _attempt() -> dict[str, Any]: return path, None try: - with tqdm.tqdm(total=len(assembly_paths), unit="assembly", desc="Downloading from NCBI FTP") as pbar: + with tqdm.tqdm( + total=len(assembly_paths), unit="assembly", desc="Downloading from NCBI FTP", smoothing=0.01 + ) as pbar: with ThreadPoolExecutor(max_workers=threads) as executor: futures = {executor.submit(_download_one, p): p for p in assembly_paths} for future in as_completed(futures): @@ -330,7 +332,9 @@ def _attempt() -> dict[str, Any]: return path, None try: - with tqdm.tqdm(total=len(assembly_paths), unit="assembly", desc="Downloading & staging") as pbar: + with tqdm.tqdm( + total=len(assembly_paths), unit="assembly", desc="Downloading & staging", smoothing=0.01 + ) as pbar: with ThreadPoolExecutor(max_workers=threads) as executor: futures = {executor.submit(_download_upload_one, p): p for p in assembly_paths} for future in as_completed(futures): From 6b78cb01832b51c4ec57a6b4cf698edbea52e97d Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 30 Apr 2026 08:25:55 -0700 Subject: [PATCH 069/128] add promote progress bar and reduce logger output on dry runs Co-authored-by: Copilot --- notebooks/ncbi_ftp_promote.ipynb | 43 +++++++++++++++++++++-- src/cdm_data_loaders/ncbi_ftp/metadata.py | 4 +-- src/cdm_data_loaders/ncbi_ftp/promote.py | 14 ++++++-- 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/notebooks/ncbi_ftp_promote.ipynb b/notebooks/ncbi_ftp_promote.ipynb index 1be65156..2691251f 100644 --- a/notebooks/ncbi_ftp_promote.ipynb +++ b/notebooks/ncbi_ftp_promote.ipynb @@ -100,6 +100,11 @@ " \"io/matt-cohere/staging/run1/input/transfer_manifest.txt\" # e.g. \"staging/transfer_manifest.txt\"\n", ")\n", "\n", + "# Local path to transfer_manifest.txt (used when the manifest has not been uploaded to S3).\n", + "# Used only for the object-count estimate in the scan step; set to None to skip.\n", + "# format: local file path\n", + "MANIFEST_LOCAL_PATH: str | None = None # e.g. \"output/transfer_manifest.txt\"\n", + "\n", "# Final Lakehouse path prefix\n", "# format: S3 key prefix within LAKEHOUSE_BUCKET (no scheme, no bucket)\n", "LAKEHOUSE_KEY_PREFIX = DEFAULT_LAKEHOUSE_KEY_PREFIX\n", @@ -113,6 +118,7 @@ "print(f\"Updated manifest: {UPDATED_MANIFEST_PATH}\")\n", "print(f\"NCBI release: {NCBI_RELEASE}\")\n", "print(f\"Manifest S3 key: {MANIFEST_S3_KEY}\")\n", + "print(f\"Manifest local path: {MANIFEST_LOCAL_PATH}\")\n", "print(f\"Lakehouse prefix: {LAKEHOUSE_KEY_PREFIX}\")\n", "print(f\"Dry-run: {DRY_RUN}\")" ] @@ -148,12 +154,45 @@ "source": [ "\"\"\"Scan staged files and display summary.\"\"\"\n", "\n", + "import tqdm\n", + "\n", "s3 = get_s3_client()\n", "paginator = s3.get_paginator(\"list_objects_v2\")\n", "\n", + "# Estimate total objects from manifest so tqdm can show a percentage.\n", + "# Each assembly typically produces ~11 data files + ~10 .md5 sidecars = ~21 objects.\n", + "_FILES_PER_ASSEMBLY_EST = 21\n", + "estimated_total = None\n", + "if MANIFEST_S3_KEY:\n", + " try:\n", + " _resp = s3.get_object(Bucket=STAGING_BUCKET, Key=MANIFEST_S3_KEY)\n", + " _lines = [\n", + " ln.strip() for ln in _resp[\"Body\"].read().decode().splitlines() if ln.strip() and not ln.startswith(\"#\")\n", + " ]\n", + " estimated_total = len(_lines) * _FILES_PER_ASSEMBLY_EST\n", + " print(f\"Manifest (S3) has {len(_lines)} assemblies → estimated ~{estimated_total} staged objects\")\n", + " except Exception as e:\n", + " print(f\"Could not read S3 manifest for estimate: {e}\")\n", + "elif MANIFEST_LOCAL_PATH:\n", + " try:\n", + " from pathlib import Path\n", + "\n", + " _lines = [\n", + " ln.strip()\n", + " for ln in Path(MANIFEST_LOCAL_PATH).read_text().splitlines()\n", + " if ln.strip() and not ln.startswith(\"#\")\n", + " ]\n", + " estimated_total = len(_lines) * _FILES_PER_ASSEMBLY_EST\n", + " print(f\"Manifest (local) has {len(_lines)} assemblies → estimated ~{estimated_total} staged objects\")\n", + " except Exception as e:\n", + " print(f\"Could not read local manifest for estimate: {e}\")\n", + "\n", "staged: list[str] = []\n", - "for page in paginator.paginate(Bucket=STAGING_BUCKET, Prefix=STAGING_KEY_PREFIX):\n", - " staged.extend(obj[\"Key\"] for obj in page.get(\"Contents\", []))\n", + "with tqdm.tqdm(total=estimated_total, unit=\"obj\", desc=\"Scanning staging prefix\", dynamic_ncols=True) as pbar:\n", + " for page in paginator.paginate(Bucket=STAGING_BUCKET, Prefix=STAGING_KEY_PREFIX):\n", + " keys = [obj[\"Key\"] for obj in page.get(\"Contents\", [])]\n", + " staged.extend(keys)\n", + " pbar.update(len(keys))\n", "\n", "sidecars = [k for k in staged if k.endswith((\".md5\", \".crc64nvme\"))]\n", "data_files = [k for k in staged if k not in set(sidecars)]\n", diff --git a/src/cdm_data_loaders/ncbi_ftp/metadata.py b/src/cdm_data_loaders/ncbi_ftp/metadata.py index 43a9c04f..a10b3c76 100644 --- a/src/cdm_data_loaders/ncbi_ftp/metadata.py +++ b/src/cdm_data_loaders/ncbi_ftp/metadata.py @@ -182,7 +182,7 @@ def upload_descriptor( key = build_descriptor_key(assembly_dir, key_prefix) if dry_run: - logger.info("[dry-run] would upload descriptor: s3://%s/%s", bucket, key) + logger.debug("[dry-run] would upload descriptor: s3://%s/%s", bucket, key) return key s3 = get_s3_client() @@ -227,7 +227,7 @@ def archive_descriptor( # noqa: PLR0913 archive_key = build_archive_descriptor_key(assembly_dir, release_tag, key_prefix) if dry_run: - logger.info("[dry-run] would archive descriptor: s3://%s/%s -> %s", bucket, source_key, archive_key) + logger.debug("[dry-run] would archive descriptor: s3://%s/%s -> %s", bucket, source_key, archive_key) return True s3 = get_s3_client() diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index 3be925fe..3f6ceebf 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -178,6 +178,7 @@ def _promote_data_files( # noqa: PLR0913, PLR0915 assembly_files[(adir_match.group(1), acc_match.group(1))].append(staged_key) total_files = sum(len(v) for v in assembly_files.values()) + _dry_run_log_count = 0 with tqdm.tqdm(total=total_files, unit="file", desc="Promoting") as pbar: for (adir, acc), files in assembly_files.items(): assembly_failed = 0 @@ -190,7 +191,11 @@ def _promote_data_files( # noqa: PLR0913, PLR0915 final_key_path = PurePosixPath(final_key) if dry_run: - logger.debug("[dry-run] would promote: %s -> %s", staged_key, final_key) + if _dry_run_log_count < 10: + logger.info("[dry-run] would promote: %s -> %s", staged_key, final_key) + else: + logger.debug("[dry-run] would promote: %s -> %s", staged_key, final_key) + _dry_run_log_count += 1 promoted += 1 pbar.update(1) continue @@ -312,6 +317,7 @@ def _archive_assemblies( # noqa: PLR0913 with Path(manifest_local_path).open() as f: accessions = [line.strip() for line in f if line.strip()] + _dry_run_log_count = 0 for accession in tqdm.tqdm(accessions, unit="accession", desc="Archiving"): m = re.match(r"(GC[AF])_(\d{3})(\d{3})(\d{3})\.\d+", accession) if not m: @@ -344,7 +350,11 @@ def _archive_assemblies( # noqa: PLR0913 archive_key = f"{lakehouse_key_prefix}archive/{release_tag}/{rel}" if dry_run: - logger.info("[dry-run] would archive: %s -> %s", source_key, archive_key) + if _dry_run_log_count < 10: + logger.info("[dry-run] would archive: %s -> %s", source_key, archive_key) + else: + logger.debug("[dry-run] would archive: %s -> %s", source_key, archive_key) + _dry_run_log_count += 1 archived += 1 continue From afcae5c6fc4419edc0083b1866371a816b7c070e Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 30 Apr 2026 08:45:42 -0700 Subject: [PATCH 070/128] print removed manifest path --- notebooks/ncbi_ftp_promote.ipynb | 1 + 1 file changed, 1 insertion(+) diff --git a/notebooks/ncbi_ftp_promote.ipynb b/notebooks/ncbi_ftp_promote.ipynb index 2691251f..e7995dc8 100644 --- a/notebooks/ncbi_ftp_promote.ipynb +++ b/notebooks/ncbi_ftp_promote.ipynb @@ -115,6 +115,7 @@ "print(f\"Staging bucket: {STAGING_BUCKET}\")\n", "print(f\"Lakehouse bucket: {LAKEHOUSE_BUCKET}\")\n", "print(f\"Staging key prefix: {STAGING_KEY_PREFIX}\")\n", + "print(f\"Removed manifest: {REMOVED_MANIFEST_PATH}\")\n", "print(f\"Updated manifest: {UPDATED_MANIFEST_PATH}\")\n", "print(f\"NCBI release: {NCBI_RELEASE}\")\n", "print(f\"Manifest S3 key: {MANIFEST_S3_KEY}\")\n", From 2e8da0ce23b17624a5306344de7c399d36788a83 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 30 Apr 2026 08:52:21 -0700 Subject: [PATCH 071/128] optimize array creation --- notebooks/ncbi_ftp_promote.ipynb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/notebooks/ncbi_ftp_promote.ipynb b/notebooks/ncbi_ftp_promote.ipynb index e7995dc8..6e16e1a4 100644 --- a/notebooks/ncbi_ftp_promote.ipynb +++ b/notebooks/ncbi_ftp_promote.ipynb @@ -195,8 +195,13 @@ " staged.extend(keys)\n", " pbar.update(len(keys))\n", "\n", - "sidecars = [k for k in staged if k.endswith((\".md5\", \".crc64nvme\"))]\n", - "data_files = [k for k in staged if k not in set(sidecars)]\n", + "sidecars = []\n", + "data_files = []\n", + "for k in staged:\n", + " if k.endswith((\".md5\", \".crc64nvme\")):\n", + " sidecars.append(k)\n", + " else:\n", + " data_files.append(k)\n", "\n", "print(f\"Staged objects: {len(staged)}\")\n", "print(f\" Data files: {len(data_files)}\")\n", From ae929164cf41200d905a212d1b80d6c5514afeca Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 30 Apr 2026 09:47:59 -0700 Subject: [PATCH 072/128] remove checksum type from copy headers --- src/cdm_data_loaders/utils/s3.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index 07c9a47d..105f8d73 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -476,7 +476,6 @@ def copy_object( Bucket=new_s3_bucket, Key=new_s3_key, **extra, - **DEFAULT_EXTRA_ARGS, ) @@ -529,7 +528,6 @@ def copy_directory(current_s3_path: str, new_s3_path: str) -> tuple[dict[str, st CopySource={"Bucket": current_s3_bucket, "Key": current_key}, Bucket=new_s3_bucket, Key=new_key, - **DEFAULT_EXTRA_ARGS, ) if resp["ResponseMetadata"]["HTTPStatusCode"] == SUCCESS_RESPONSE: successes[source_path] = dest_path From 1a8f54a00b25b782dab533f6b0f7c4bbdd1bcde2 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 30 Apr 2026 11:24:07 -0700 Subject: [PATCH 073/128] use tags for metadata on copy to avoid boto errors Co-authored-by: Copilot --- src/cdm_data_loaders/ncbi_ftp/metadata.py | 2 +- src/cdm_data_loaders/ncbi_ftp/promote.py | 4 +-- src/cdm_data_loaders/utils/s3.py | 44 +++++++++++++---------- tests/integration/conftest.py | 20 +++++++++-- tests/integration/test_full_pipeline.py | 6 ++-- tests/integration/test_promote_e2e.py | 12 +++---- tests/ncbi_ftp/test_promote.py | 7 ++-- tests/utils/test_s3.py | 30 +++++++++------- 8 files changed, 78 insertions(+), 47 deletions(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/metadata.py b/src/cdm_data_loaders/ncbi_ftp/metadata.py index a10b3c76..8098e9c4 100644 --- a/src/cdm_data_loaders/ncbi_ftp/metadata.py +++ b/src/cdm_data_loaders/ncbi_ftp/metadata.py @@ -247,7 +247,7 @@ def archive_descriptor( # noqa: PLR0913 copy_object( f"{bucket}/{source_key}", f"{bucket}/{archive_key}", - metadata={ + tags={ "ncbi_last_release": release_tag, "archive_reason": archive_reason, "archive_date": datestamp, diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index 3f6ceebf..7920d7b7 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -217,7 +217,7 @@ def _promote_data_files( # noqa: PLR0913, PLR0915 upload_succeeded = upload_file( tmp_path, f"{lakehouse_bucket}/{final_key_path.parent}", - metadata=metadata, + tags=metadata, object_name=final_key_path.name, show_progress=False, ) @@ -362,7 +362,7 @@ def _archive_assemblies( # noqa: PLR0913 copy_object( f"{lakehouse_bucket}/{source_key}", f"{lakehouse_bucket}/{archive_key}", - metadata={ + tags={ "ncbi_last_release": release_tag, "archive_reason": archive_reason, "archive_date": datestamp, diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index 105f8d73..0766f248 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -200,7 +200,7 @@ def upload_file( local_file_path: Path | str, destination_dir: str, object_name: str | None = None, - metadata: dict[str, str] | None = None, + tags: dict[str, str] | None = None, show_progress: bool = True, ) -> bool: """Upload an object to an S3 bucket. @@ -232,14 +232,14 @@ def upload_file( object_name = local_file_path.name s3_path = f"{destination_dir.removesuffix('/')}/{object_name}" - if metadata is None and object_exists(s3_path): + if tags is None and object_exists(s3_path): logger.debug("File already present: %s", s3_path) return True s3 = get_s3_client() (bucket, key) = split_s3_path(s3_path) - extra_args = {**DEFAULT_EXTRA_ARGS, **(({"Metadata": metadata}) if metadata is not None else {})} + extra_args = {**DEFAULT_EXTRA_ARGS, **(({"Metadata": tags}) if tags is not None else {})} # Upload the file logger.debug("uploading %s to %s", str(local_file_path), s3_path) @@ -439,13 +439,18 @@ def upload_dir( def copy_object( current_s3_path: str, new_s3_path: str, - metadata: dict[str, str] | None = None, + tags: dict[str, str] | None = None, ) -> dict[str, Any]: - """Copy an object from one place to another, adding in a CRC64NVME checksum. + """Copy an object from one place to another, inheriting the source user metadata. - When *metadata* is supplied the destination object carries exactly those - key/value pairs (``MetadataDirective='REPLACE'``). When *metadata* is - ``None`` (the default) the source metadata is inherited. + Source user metadata (e.g. ``md5``) is preserved on the destination because + ``MetadataDirective`` is omitted, which defaults to ``COPY``. + + When *metadata* is supplied it is applied as S3 object **tags** via a + separate ``put_object_tagging`` call rather than as user metadata. This + avoids passing ``MetadataDirective=REPLACE`` to ``CopyObject``, which causes + botocore to inject an unsigned checksum header that AWS rejects with + AccessDenied. A successful copy operation will return a response where resp["ResponseMetadata"]["HTTPStatusCode"] == 200 @@ -457,27 +462,30 @@ def copy_object( :type current_s3_path: str :param new_s3_path: the desired new file path on s3, INCLUDING the bucket name :type new_s3_path: str - :param metadata: user metadata to set on the destination object; when provided the source metadata is replaced - :type metadata: dict[str, str] | None - :return: dictionary containing response + :param tags: key-value pairs to set as S3 object tags on the destination + :type tags: dict[str, str] | None + :return: dictionary containing response from the copy operation :rtype: dict[str, Any] """ s3 = get_s3_client() (current_s3_bucket, current_s3_key) = split_s3_path(current_s3_path) (new_s3_bucket, new_s3_key) = split_s3_path(new_s3_path) - extra: dict[str, Any] = {} - if metadata is not None: - extra["Metadata"] = metadata - extra["MetadataDirective"] = "REPLACE" - - return s3.copy_object( + resp = s3.copy_object( CopySource={"Bucket": current_s3_bucket, "Key": current_s3_key}, Bucket=new_s3_bucket, Key=new_s3_key, - **extra, ) + if tags: + s3.put_object_tagging( + Bucket=new_s3_bucket, + Key=new_s3_key, + Tagging={"TagSet": [{"Key": k, "Value": v} for k, v in tags.items()]}, + ) + + return resp + def copy_directory(current_s3_path: str, new_s3_path: str) -> tuple[dict[str, str], dict[str, Any]]: """Copy all objects under a given S3 prefix to a new prefix. diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index f27e4202..08e9fe00 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -252,12 +252,28 @@ def list_all_keys(s3: botocore.client.BaseClient, bucket: str, prefix: str = "") def get_object_metadata(s3: botocore.client.BaseClient, bucket: str, key: str) -> dict[str, Any]: - """Return the user metadata dict for an S3 object. + """Return the S3 user metadata dict for an S3 object (from HeadObject). :param s3: boto3 S3 client :param bucket: bucket name :param key: object key - :return: metadata dict + :return: user metadata dict """ resp = s3.head_object(Bucket=bucket, Key=key) return resp.get("Metadata", {}) + + +def get_object_tags(s3: botocore.client.BaseClient, bucket: str, key: str) -> dict[str, Any]: + """Return the S3 object tags dict for an S3 object (from GetObjectTagging). + + Archive attributes (e.g. ``archive_reason``) are stored as S3 object tags + rather than user metadata to avoid the botocore unsigned-header bug with + ``MetadataDirective=REPLACE``. + + :param s3: boto3 S3 client + :param bucket: bucket name + :param key: object key + :return: tags dict + """ + resp = s3.get_object_tagging(Bucket=bucket, Key=key) + return {t["Key"]: t["Value"] for t in resp.get("TagSet", [])} diff --git a/tests/integration/test_full_pipeline.py b/tests/integration/test_full_pipeline.py index e8f6e4a6..af8f164b 100644 --- a/tests/integration/test_full_pipeline.py +++ b/tests/integration/test_full_pipeline.py @@ -29,7 +29,7 @@ from cdm_data_loaders.ncbi_ftp.promote import DEFAULT_LAKEHOUSE_KEY_PREFIX, promote_from_s3 from cdm_data_loaders.pipelines.ncbi_ftp_download import download_batch -from .conftest import get_object_metadata, list_all_keys, stage_files_to_minio, staging_test_bucket # noqa: F401 +from .conftest import get_object_metadata, get_object_tags, list_all_keys, stage_files_to_minio, staging_test_bucket # noqa: F401 STABLE_PREFIX = "900" STAGING_PREFIX = "staging/run1/" @@ -215,8 +215,8 @@ def test_full_pipeline_incremental( if promote2["archived"] > 0: assert len(archive_keys) >= 1 for key in archive_keys: - meta = get_object_metadata(s3, test_bucket, key) - assert meta.get("archive_reason") == "updated" + tags = get_object_tags(s3, test_bucket, key) + assert tags.get("archive_reason") == "updated" # Final Lakehouse path should still have files final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") diff --git a/tests/integration/test_promote_e2e.py b/tests/integration/test_promote_e2e.py index ed3f5dd1..7b9b8d48 100644 --- a/tests/integration/test_promote_e2e.py +++ b/tests/integration/test_promote_e2e.py @@ -21,7 +21,7 @@ ) from cdm_data_loaders.ncbi_ftp.promote import DEFAULT_LAKEHOUSE_KEY_PREFIX, promote_from_s3 -from .conftest import get_object_metadata, list_all_keys, seed_lakehouse, staging_test_bucket # noqa: F401 +from .conftest import get_object_metadata, get_object_tags, list_all_keys, seed_lakehouse, staging_test_bucket # noqa: F401 from pathlib import Path @@ -188,9 +188,9 @@ def test_archive_updated( # Verify archive metadata for key in archive_keys: - meta = get_object_metadata(s3, test_bucket, key) - assert meta.get("archive_reason") == "updated" - assert meta.get("ncbi_last_release") == "2024-01" + tags = get_object_tags(s3, test_bucket, key) + assert tags.get("archive_reason") == "updated" + assert tags.get("ncbi_last_release") == "2024-01" @pytest.mark.integration @@ -231,8 +231,8 @@ def test_archive_removed( # Verify archive metadata for key in archive_keys: - meta = get_object_metadata(s3, test_bucket, key) - assert meta.get("archive_reason") == "replaced_or_suppressed" + tags = get_object_tags(s3, test_bucket, key) + assert tags.get("archive_reason") == "replaced_or_suppressed" # Verify source objects are deleted rel = build_accession_path(ASSEMBLY_DIR_A) diff --git a/tests/ncbi_ftp/test_promote.py b/tests/ncbi_ftp/test_promote.py index db19bcdb..9c61ca6e 100644 --- a/tests/ncbi_ftp/test_promote.py +++ b/tests/ncbi_ftp/test_promote.py @@ -153,8 +153,11 @@ def test_archive_assemblies_updated_no_delete( f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-06/raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" ) resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=archive_key) - assert resp["Metadata"]["archive_reason"] == "updated" - assert resp["Metadata"]["ncbi_last_release"] == "2024-06" + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + tag_resp = mock_s3_client_no_checksum.get_object_tagging(Bucket=TEST_BUCKET, Key=archive_key) + tags = {t["Key"]: t["Value"] for t in tag_resp["TagSet"]} + assert tags["archive_reason"] == "updated" + assert tags["ncbi_last_release"] == "2024-06" @pytest.mark.s3 diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index 7e68acc7..c88bbadd 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -899,7 +899,7 @@ def test_delete_object_removes_object(mock_s3_client: Any, bucket: str, protocol def test_upload_file_with_metadata_attaches_metadata(mock_s3_client: Any, sample_file: Path, bucket: str) -> None: """Verify that upload_file with metadata stores user metadata on the uploaded object.""" metadata = {"md5": "abc123", "source": "ncbi"} - result = upload_file(sample_file, f"{bucket}/uploads", metadata=metadata) + result = upload_file(sample_file, f"{bucket}/uploads", tags=metadata) assert result is True resp = mock_s3_client.head_object(Bucket=bucket, Key=f"uploads/{sample_file.name}") @@ -910,7 +910,7 @@ def test_upload_file_with_metadata_attaches_metadata(mock_s3_client: Any, sample @pytest.mark.s3 def test_upload_file_with_metadata_custom_object_name(mock_s3_client: Any, sample_file: Path) -> None: """Verify that the object_name parameter overrides the filename.""" - result = upload_file(sample_file, f"{CDM_LAKE_BUCKET}/uploads", metadata={"k": "v"}, object_name="renamed.txt") + result = upload_file(sample_file, f"{CDM_LAKE_BUCKET}/uploads", tags={"k": "v"}, object_name="renamed.txt") assert result is True obj = mock_s3_client.get_object(Bucket=CDM_LAKE_BUCKET, Key="uploads/renamed.txt") assert obj["Body"].read() == b"hello s3" @@ -920,7 +920,7 @@ def test_upload_file_with_metadata_custom_object_name(mock_s3_client: Any, sampl def test_upload_file_with_metadata_overwrites_existing(mock_s3_client: Any, sample_file: Path) -> None: """Verify that upload_file with metadata uploads even when the object already exists.""" mock_s3_client.put_object(Bucket=CDM_LAKE_BUCKET, Key=f"uploads/{sample_file.name}", Body=b"old") - result = upload_file(sample_file, f"{CDM_LAKE_BUCKET}/uploads", metadata={"new": "true"}) + result = upload_file(sample_file, f"{CDM_LAKE_BUCKET}/uploads", tags={"new": "true"}) assert result is True obj = mock_s3_client.get_object(Bucket=CDM_LAKE_BUCKET, Key=f"uploads/{sample_file.name}") assert obj["Body"].read() == b"hello s3" @@ -931,7 +931,7 @@ def test_upload_file_with_metadata_overwrites_existing(mock_s3_client: Any, samp def test_upload_file_with_metadata_raises_on_empty_destination(sample_file: Path) -> None: """Verify ValueError when destination_dir is empty.""" with pytest.raises(ValueError, match="No destination directory"): - upload_file(sample_file, "", metadata={"k": "v"}) + upload_file(sample_file, "", tags={"k": "v"}) @pytest.mark.usefixtures("mock_s3_client") @@ -939,7 +939,7 @@ def test_upload_file_with_metadata_raises_on_empty_destination(sample_file: Path @pytest.mark.s3 def test_upload_file_with_metadata_accepts_str_and_path(sample_file: Path, path_type: type[str] | type[Path]) -> None: """Verify that upload_file with metadata accepts both str and Path.""" - result = upload_file(path_type(sample_file), f"{CDM_LAKE_BUCKET}/uploads", metadata={}) + result = upload_file(path_type(sample_file), f"{CDM_LAKE_BUCKET}/uploads", tags={}) assert result is True @@ -978,23 +978,27 @@ def test_head_object_with_protocols(mock_s3_client: Any, protocol: str) -> None: @pytest.mark.parametrize("destination", BUCKETS) @pytest.mark.s3 def test_copy_object_with_metadata_replaces_metadata(mocked_s3_client_no_checksum: Any, destination: str) -> None: - """Verify that copy_object with metadata copies and replaces metadata.""" + """Verify that copy_object with tags copies the object and sets S3 object tags on the destination.""" mocked_s3_client_no_checksum.put_object( Bucket=CDM_LAKE_BUCKET, Key="src/file.txt", Body=b"archive me", Metadata={"old_key": "old_val"} ) - new_metadata = {"archive_reason": "replaced", "archive_date": "2026-04-16"} + new_tags = {"archive_reason": "replaced", "archive_date": "2026-04-16"} response = copy_object( f"{CDM_LAKE_BUCKET}/src/file.txt", f"{destination}/archive/file.txt", - metadata=new_metadata, + tags=new_tags, ) assert response["ResponseMetadata"]["HTTPStatusCode"] == HTTP_STATUS_OK - # verify the destination has the new metadata, not the old + # archive fields are stored as S3 tags, not user metadata + tag_resp = mocked_s3_client_no_checksum.get_object_tagging(Bucket=destination, Key="archive/file.txt") + tag_dict = {t["Key"]: t["Value"] for t in tag_resp["TagSet"]} + assert tag_dict["archive_reason"] == "replaced" + assert tag_dict["archive_date"] == "2026-04-16" + + # source user metadata is preserved (MetadataDirective=COPY) resp = mocked_s3_client_no_checksum.head_object(Bucket=destination, Key="archive/file.txt") - assert resp["Metadata"]["archive_reason"] == "replaced" - assert resp["Metadata"]["archive_date"] == "2026-04-16" - assert "old_key" not in resp["Metadata"] + assert resp["Metadata"].get("old_key") == "old_val" # verify source still exists assert object_exists(f"{CDM_LAKE_BUCKET}/src/file.txt") @@ -1007,7 +1011,7 @@ def test_copy_object_with_metadata_preserves_content(mocked_s3_client_no_checksu copy_object( f"{CDM_LAKE_BUCKET}/src/data.bin", f"{CDM_LAKE_BUCKET}/dst/data.bin", - metadata={"tag": "value"}, + tags={"tag": "value"}, ) obj = mocked_s3_client_no_checksum.get_object(Bucket=CDM_LAKE_BUCKET, Key="dst/data.bin") assert obj["Body"].read() == b"binary data" From fa3df736c93281831f44cc8db119a8de455c0814 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 30 Apr 2026 11:53:48 -0700 Subject: [PATCH 074/128] remove tagging option on s3 copy Co-authored-by: Copilot --- src/cdm_data_loaders/ncbi_ftp/metadata.py | 17 +++++++---------- src/cdm_data_loaders/ncbi_ftp/promote.py | 8 +------- src/cdm_data_loaders/utils/s3.py | 20 +------------------- tests/integration/conftest.py | 16 ---------------- tests/integration/test_full_pipeline.py | 5 ++--- tests/integration/test_promote_e2e.py | 14 ++++++-------- tests/ncbi_ftp/test_metadata.py | 6 +++--- tests/ncbi_ftp/test_promote.py | 22 +++++----------------- tests/utils/test_s3.py | 21 ++++++--------------- 9 files changed, 31 insertions(+), 98 deletions(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/metadata.py b/src/cdm_data_loaders/ncbi_ftp/metadata.py index 8098e9c4..e616c9b5 100644 --- a/src/cdm_data_loaders/ncbi_ftp/metadata.py +++ b/src/cdm_data_loaders/ncbi_ftp/metadata.py @@ -67,16 +67,19 @@ def build_descriptor_key(assembly_dir: str, key_prefix: str) -> str: return f"{prefix}metadata/{assembly_dir}_datapackage.json" -def build_archive_descriptor_key(assembly_dir: str, release_tag: str, key_prefix: str) -> str: +def build_archive_descriptor_key( + assembly_dir: str, release_tag: str, key_prefix: str, archive_reason: str = "unknown" +) -> str: """Return the S3 key for the archived descriptor of *assembly_dir*. :param assembly_dir: full assembly directory name :param release_tag: NCBI release tag used in the archive path, e.g. ``"2024-01"`` :param key_prefix: Lakehouse key prefix - :return: S3 key under ``archive/{release_tag}/metadata/`` + :param archive_reason: reason for archival, encoded as a path segment + :return: S3 key under ``archive/{release_tag}/{archive_reason}/metadata/`` """ prefix = key_prefix.rstrip("/") + "/" - return f"{prefix}archive/{release_tag}/metadata/{assembly_dir}_datapackage.json" + return f"{prefix}archive/{release_tag}/{archive_reason}/metadata/{assembly_dir}_datapackage.json" def create_descriptor( @@ -224,7 +227,7 @@ def archive_descriptor( # noqa: PLR0913 :return: ``True`` if the descriptor was (or would be) archived; ``False`` if not found """ source_key = build_descriptor_key(assembly_dir, key_prefix) - archive_key = build_archive_descriptor_key(assembly_dir, release_tag, key_prefix) + archive_key = build_archive_descriptor_key(assembly_dir, release_tag, key_prefix, archive_reason) if dry_run: logger.debug("[dry-run] would archive descriptor: s3://%s/%s -> %s", bucket, source_key, archive_key) @@ -243,15 +246,9 @@ def archive_descriptor( # noqa: PLR0913 return False raise - datestamp = datetime.now(UTC).strftime("%Y-%m-%d") copy_object( f"{bucket}/{source_key}", f"{bucket}/{archive_key}", - tags={ - "ncbi_last_release": release_tag, - "archive_reason": archive_reason, - "archive_date": datestamp, - }, ) logger.debug("Archived descriptor: s3://%s/%s -> %s", bucket, source_key, archive_key) return True diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index 7920d7b7..cb5fa6c9 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -311,7 +311,6 @@ def _archive_assemblies( # noqa: PLR0913 """ s3 = get_s3_client() release_tag = ncbi_release or "unknown" - datestamp = datetime.now(UTC).strftime("%Y-%m-%d") archived = 0 with Path(manifest_local_path).open() as f: @@ -347,7 +346,7 @@ def _archive_assemblies( # noqa: PLR0913 for source_key in matching_keys: rel = source_key[len(lakehouse_key_prefix) :] - archive_key = f"{lakehouse_key_prefix}archive/{release_tag}/{rel}" + archive_key = f"{lakehouse_key_prefix}archive/{release_tag}/{archive_reason}/{rel}" if dry_run: if _dry_run_log_count < 10: @@ -362,11 +361,6 @@ def _archive_assemblies( # noqa: PLR0913 copy_object( f"{lakehouse_bucket}/{source_key}", f"{lakehouse_bucket}/{archive_key}", - tags={ - "ncbi_last_release": release_tag, - "archive_reason": archive_reason, - "archive_date": datestamp, - }, ) if delete_source: delete_object(f"{lakehouse_bucket}/{source_key}") diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index 0766f248..a6d85976 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -439,19 +439,12 @@ def upload_dir( def copy_object( current_s3_path: str, new_s3_path: str, - tags: dict[str, str] | None = None, ) -> dict[str, Any]: """Copy an object from one place to another, inheriting the source user metadata. Source user metadata (e.g. ``md5``) is preserved on the destination because ``MetadataDirective`` is omitted, which defaults to ``COPY``. - When *metadata* is supplied it is applied as S3 object **tags** via a - separate ``put_object_tagging`` call rather than as user metadata. This - avoids passing ``MetadataDirective=REPLACE`` to ``CopyObject``, which causes - botocore to inject an unsigned checksum header that AWS rejects with - AccessDenied. - A successful copy operation will return a response where resp["ResponseMetadata"]["HTTPStatusCode"] == 200 @@ -462,8 +455,6 @@ def copy_object( :type current_s3_path: str :param new_s3_path: the desired new file path on s3, INCLUDING the bucket name :type new_s3_path: str - :param tags: key-value pairs to set as S3 object tags on the destination - :type tags: dict[str, str] | None :return: dictionary containing response from the copy operation :rtype: dict[str, Any] """ @@ -471,21 +462,12 @@ def copy_object( (current_s3_bucket, current_s3_key) = split_s3_path(current_s3_path) (new_s3_bucket, new_s3_key) = split_s3_path(new_s3_path) - resp = s3.copy_object( + return s3.copy_object( CopySource={"Bucket": current_s3_bucket, "Key": current_s3_key}, Bucket=new_s3_bucket, Key=new_s3_key, ) - if tags: - s3.put_object_tagging( - Bucket=new_s3_bucket, - Key=new_s3_key, - Tagging={"TagSet": [{"Key": k, "Value": v} for k, v in tags.items()]}, - ) - - return resp - def copy_directory(current_s3_path: str, new_s3_path: str) -> tuple[dict[str, str], dict[str, Any]]: """Copy all objects under a given S3 prefix to a new prefix. diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 08e9fe00..b8bdf9ba 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -261,19 +261,3 @@ def get_object_metadata(s3: botocore.client.BaseClient, bucket: str, key: str) - """ resp = s3.head_object(Bucket=bucket, Key=key) return resp.get("Metadata", {}) - - -def get_object_tags(s3: botocore.client.BaseClient, bucket: str, key: str) -> dict[str, Any]: - """Return the S3 object tags dict for an S3 object (from GetObjectTagging). - - Archive attributes (e.g. ``archive_reason``) are stored as S3 object tags - rather than user metadata to avoid the botocore unsigned-header bug with - ``MetadataDirective=REPLACE``. - - :param s3: boto3 S3 client - :param bucket: bucket name - :param key: object key - :return: tags dict - """ - resp = s3.get_object_tagging(Bucket=bucket, Key=key) - return {t["Key"]: t["Value"] for t in resp.get("TagSet", [])} diff --git a/tests/integration/test_full_pipeline.py b/tests/integration/test_full_pipeline.py index af8f164b..0c948764 100644 --- a/tests/integration/test_full_pipeline.py +++ b/tests/integration/test_full_pipeline.py @@ -29,7 +29,7 @@ from cdm_data_loaders.ncbi_ftp.promote import DEFAULT_LAKEHOUSE_KEY_PREFIX, promote_from_s3 from cdm_data_loaders.pipelines.ncbi_ftp_download import download_batch -from .conftest import get_object_metadata, get_object_tags, list_all_keys, stage_files_to_minio, staging_test_bucket # noqa: F401 +from .conftest import get_object_metadata, list_all_keys, stage_files_to_minio, staging_test_bucket # noqa: F401 STABLE_PREFIX = "900" STAGING_PREFIX = "staging/run1/" @@ -215,8 +215,7 @@ def test_full_pipeline_incremental( if promote2["archived"] > 0: assert len(archive_keys) >= 1 for key in archive_keys: - tags = get_object_tags(s3, test_bucket, key) - assert tags.get("archive_reason") == "updated" + assert "/updated/" in key # Final Lakehouse path should still have files final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") diff --git a/tests/integration/test_promote_e2e.py b/tests/integration/test_promote_e2e.py index 7b9b8d48..b4aabe6c 100644 --- a/tests/integration/test_promote_e2e.py +++ b/tests/integration/test_promote_e2e.py @@ -21,7 +21,7 @@ ) from cdm_data_loaders.ncbi_ftp.promote import DEFAULT_LAKEHOUSE_KEY_PREFIX, promote_from_s3 -from .conftest import get_object_metadata, get_object_tags, list_all_keys, seed_lakehouse, staging_test_bucket # noqa: F401 +from .conftest import get_object_metadata, list_all_keys, seed_lakehouse, staging_test_bucket # noqa: F401 from pathlib import Path @@ -188,9 +188,8 @@ def test_archive_updated( # Verify archive metadata for key in archive_keys: - tags = get_object_tags(s3, test_bucket, key) - assert tags.get("archive_reason") == "updated" - assert tags.get("ncbi_last_release") == "2024-01" + assert "/updated/" in key + assert "/2024-01/" in key @pytest.mark.integration @@ -231,8 +230,7 @@ def test_archive_removed( # Verify archive metadata for key in archive_keys: - tags = get_object_tags(s3, test_bucket, key) - assert tags.get("archive_reason") == "replaced_or_suppressed" + assert "/replaced_or_suppressed/" in key # Verify source objects are deleted rel = build_accession_path(ASSEMBLY_DIR_A) @@ -463,7 +461,7 @@ def test_archive_copies_descriptor( lakehouse_key_prefix=PATH_PREFIX, ) - archive_key = build_archive_descriptor_key(ASSEMBLY_DIR_A, "2024-01", PATH_PREFIX) + archive_key = build_archive_descriptor_key(ASSEMBLY_DIR_A, "2024-01", PATH_PREFIX, "updated") # Confirm the archive descriptor object exists resp = s3.head_object(Bucket=test_bucket, Key=archive_key) assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 @@ -499,7 +497,7 @@ def test_archive_removed_copies_descriptor( lakehouse_key_prefix=PATH_PREFIX, ) - archive_key = build_archive_descriptor_key(ASSEMBLY_DIR_A, "2024-01", PATH_PREFIX) + archive_key = build_archive_descriptor_key(ASSEMBLY_DIR_A, "2024-01", PATH_PREFIX, "replaced_or_suppressed") resp = s3.head_object(Bucket=test_bucket, Key=archive_key) assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 diff --git a/tests/ncbi_ftp/test_metadata.py b/tests/ncbi_ftp/test_metadata.py index 0b471b48..df1df775 100644 --- a/tests/ncbi_ftp/test_metadata.py +++ b/tests/ncbi_ftp/test_metadata.py @@ -77,9 +77,9 @@ def test_build_descriptor_key(prefix: str) -> None: ], ) def test_build_archive_descriptor_key(prefix: str, tag: str) -> None: - """Archive key includes tag and has no double slash; trailing slash on prefix is normalized.""" - key = build_archive_descriptor_key(_ASSEMBLY_DIR, tag, prefix) - assert key == f"{_KEY_PREFIX}archive/{tag}/metadata/{_ASSEMBLY_DIR}_datapackage.json" + """Archive key includes tag and reason segment; no double slash; prefix trailing slash normalized.""" + key = build_archive_descriptor_key(_ASSEMBLY_DIR, tag, prefix, "updated") + assert key == f"{_KEY_PREFIX}archive/{tag}/updated/metadata/{_ASSEMBLY_DIR}_datapackage.json" assert "//" not in key diff --git a/tests/ncbi_ftp/test_promote.py b/tests/ncbi_ftp/test_promote.py index 9c61ca6e..3c7bc05e 100644 --- a/tests/ncbi_ftp/test_promote.py +++ b/tests/ncbi_ftp/test_promote.py @@ -118,7 +118,7 @@ def test_archive_assemblies_removed(mock_s3_client_no_checksum: botocore.client. assert mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=key).get("KeyCount", 0) == 0 archive_key = ( - f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/" + f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/replaced_or_suppressed/" f"raw_data/GCF/000/005/845/{accession}_ASM584v2/{accession}_genomic.fna.gz" ) assert mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_key).get("KeyCount", 0) == 1 @@ -149,15 +149,9 @@ def test_archive_assemblies_updated_no_delete( ) assert mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=key).get("KeyCount", 0) == 1 - archive_key = ( - f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-06/raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" - ) + archive_key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-06/updated/raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=archive_key) assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 - tag_resp = mock_s3_client_no_checksum.get_object_tagging(Bucket=TEST_BUCKET, Key=archive_key) - tags = {t["Key"]: t["Value"] for t in tag_resp["TagSet"]} - assert tags["archive_reason"] == "updated" - assert tags["ncbi_last_release"] == "2024-06" @pytest.mark.s3 @@ -177,12 +171,8 @@ def test_archive_assemblies_multiple_releases_no_collision( mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"v2-data") _archive_assemblies(str(manifest), lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-06", archive_reason="updated") - archive_key_1 = ( - f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" - ) - archive_key_2 = ( - f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-06/raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" - ) + archive_key_1 = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/updated/raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" + archive_key_2 = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-06/updated/raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" assert mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=archive_key_1)["Body"].read() == b"v1-data" assert mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=archive_key_2)["Body"].read() == b"v2-data" @@ -239,7 +229,5 @@ def test_archive_assemblies_unknown_release_fallback( assert _archive_assemblies(str(manifest), lakehouse_bucket=TEST_BUCKET, ncbi_release=None) == 1 - archive_key = ( - f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/unknown/raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" - ) + archive_key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/unknown/unknown/raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" assert mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_key).get("KeyCount", 0) == 1 diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index c88bbadd..36dd9205 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -974,44 +974,35 @@ def test_head_object_with_protocols(mock_s3_client: Any, protocol: str) -> None: assert result["size"] == SIZE_DATA -# copy_object with metadata +# copy_object @pytest.mark.parametrize("destination", BUCKETS) @pytest.mark.s3 -def test_copy_object_with_metadata_replaces_metadata(mocked_s3_client_no_checksum: Any, destination: str) -> None: - """Verify that copy_object with tags copies the object and sets S3 object tags on the destination.""" +def test_copy_object_preserves_user_metadata(mocked_s3_client_no_checksum: Any, destination: str) -> None: + """copy_object preserves source user metadata (MetadataDirective=COPY default).""" mocked_s3_client_no_checksum.put_object( - Bucket=CDM_LAKE_BUCKET, Key="src/file.txt", Body=b"archive me", Metadata={"old_key": "old_val"} + Bucket=CDM_LAKE_BUCKET, Key="src/file.txt", Body=b"archive me", Metadata={"md5": "abc123"} ) - new_tags = {"archive_reason": "replaced", "archive_date": "2026-04-16"} response = copy_object( f"{CDM_LAKE_BUCKET}/src/file.txt", f"{destination}/archive/file.txt", - tags=new_tags, ) assert response["ResponseMetadata"]["HTTPStatusCode"] == HTTP_STATUS_OK - # archive fields are stored as S3 tags, not user metadata - tag_resp = mocked_s3_client_no_checksum.get_object_tagging(Bucket=destination, Key="archive/file.txt") - tag_dict = {t["Key"]: t["Value"] for t in tag_resp["TagSet"]} - assert tag_dict["archive_reason"] == "replaced" - assert tag_dict["archive_date"] == "2026-04-16" - # source user metadata is preserved (MetadataDirective=COPY) resp = mocked_s3_client_no_checksum.head_object(Bucket=destination, Key="archive/file.txt") - assert resp["Metadata"].get("old_key") == "old_val" + assert resp["Metadata"].get("md5") == "abc123" # verify source still exists assert object_exists(f"{CDM_LAKE_BUCKET}/src/file.txt") @pytest.mark.s3 -def test_copy_object_with_metadata_preserves_content(mocked_s3_client_no_checksum: Any) -> None: +def test_copy_object_preserves_content(mocked_s3_client_no_checksum: Any) -> None: """Verify that the content of the copied object matches the original.""" mocked_s3_client_no_checksum.put_object(Bucket=CDM_LAKE_BUCKET, Key="src/data.bin", Body=b"binary data") copy_object( f"{CDM_LAKE_BUCKET}/src/data.bin", f"{CDM_LAKE_BUCKET}/dst/data.bin", - tags={"tag": "value"}, ) obj = mocked_s3_client_no_checksum.get_object(Bucket=CDM_LAKE_BUCKET, Key="dst/data.bin") assert obj["Body"].read() == b"binary data" From 4c9c1c8c057e444777780945468fcc8a4b06e80b Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 30 Apr 2026 12:26:48 -0700 Subject: [PATCH 075/128] parallelize copies and batch deletes Co-authored-by: Copilot --- src/cdm_data_loaders/ncbi_ftp/promote.py | 58 +++- src/cdm_data_loaders/utils/s3.py | 25 ++ tests/integration/test_promote_e2e.py | 422 +++++++++++++++++++++++ tests/ncbi_ftp/test_promote.py | 321 +++++++++++++++++ tests/utils/test_s3.py | 32 ++ 5 files changed, 841 insertions(+), 17 deletions(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index cb5fa6c9..52a71612 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -9,6 +9,7 @@ import re import tempfile from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import UTC, datetime from pathlib import Path, PurePosixPath from typing import Any @@ -27,6 +28,7 @@ from cdm_data_loaders.utils.s3 import ( copy_object, delete_object, + delete_objects, get_s3_client, object_exists, upload_file, @@ -344,30 +346,52 @@ def _archive_assemblies( # noqa: PLR0913 assembly_dir = adir_match.group(1) break - for source_key in matching_keys: - rel = source_key[len(lakehouse_key_prefix) :] - archive_key = f"{lakehouse_key_prefix}archive/{release_tag}/{archive_reason}/{rel}" + key_pairs = [ + ( + source_key, + f"{lakehouse_key_prefix}archive/{release_tag}/{archive_reason}/{source_key[len(lakehouse_key_prefix) :]}", + ) + for source_key in matching_keys + ] - if dry_run: + if dry_run: + for source_key, archive_key in key_pairs: if _dry_run_log_count < 10: logger.info("[dry-run] would archive: %s -> %s", source_key, archive_key) else: logger.debug("[dry-run] would archive: %s -> %s", source_key, archive_key) _dry_run_log_count += 1 - archived += 1 - continue + archived += len(key_pairs) + continue - try: - copy_object( - f"{lakehouse_bucket}/{source_key}", - f"{lakehouse_bucket}/{archive_key}", - ) - if delete_source: - delete_object(f"{lakehouse_bucket}/{source_key}") - archived += 1 - logger.debug(" Archived: %s -> %s", source_key, archive_key) - except Exception: - logger.exception("Failed to archive %s", source_key) + # Copy all files for this accession concurrently + keys_to_delete: list[str] = [] + n_workers = min(32, len(key_pairs)) + with ThreadPoolExecutor(max_workers=n_workers) as executor: + futures = { + executor.submit( + copy_object, + f"{lakehouse_bucket}/{src}", + f"{lakehouse_bucket}/{arch}", + ): src + for src, arch in key_pairs + } + for future in as_completed(futures): + src = futures[future] + try: + future.result() + archived += 1 + if delete_source: + keys_to_delete.append(src) + logger.debug(" Archived: %s", src) + except Exception: + logger.exception("Failed to archive %s", src) + + # Batch-delete source keys in a single API call + if keys_to_delete: + del_errors = delete_objects(lakehouse_bucket, keys_to_delete) + for err in del_errors: + logger.warning("Failed to delete %s: %s", err.get("Key"), err.get("Message")) # Archive the frictionless descriptor alongside raw data if assembly_dir: diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index a6d85976..738309e8 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -547,3 +547,28 @@ def delete_object(s3_path: str) -> dict[str, Any]: s3 = get_s3_client() (bucket, key) = split_s3_path(s3_path) return s3.delete_object(Bucket=bucket, Key=key) + + +def delete_objects(bucket: str, keys: list[str]) -> list[dict[str, Any]]: + """Delete multiple objects from an S3 bucket in a single API call. + + Splits into batches of 1000 (the S3 API maximum per request). + + :param bucket: S3 bucket name (no protocol prefix) + :param keys: list of S3 keys to delete + :return: list of per-key error dicts returned by S3 (empty if all succeeded) + :rtype: list[dict[str, Any]] + """ + if not keys: + return [] + + s3 = get_s3_client() + errors: list[dict[str, Any]] = [] + for i in range(0, len(keys), 1000): + batch = keys[i : i + 1000] + resp = s3.delete_objects( + Bucket=bucket, + Delete={"Objects": [{"Key": k} for k in batch], "Quiet": False}, + ) + errors.extend(resp.get("Errors", [])) + return errors diff --git a/tests/integration/test_promote_e2e.py b/tests/integration/test_promote_e2e.py index b4aabe6c..b805f03c 100644 --- a/tests/integration/test_promote_e2e.py +++ b/tests/integration/test_promote_e2e.py @@ -522,3 +522,425 @@ def test_dry_run_no_descriptor(self, minio_s3_client: object, test_bucket: str, metadata_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "metadata/") assert len(metadata_keys) == 0, f"Dry-run should not create descriptor files, found: {metadata_keys}" + + +# ── Parallel archiving tests ───────────────────────────────────────────── + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestArchiveMultiFileConcurrent: + """Verify parallel copy archives all files correctly with correct content.""" + + def test_all_files_archived_with_correct_content( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + ) -> None: + """Every file is archived with byte-identical content when copied concurrently.""" + from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies + + s3 = minio_s3_client + + # Seed many files for assembly A at final Lakehouse path + many_files = { + f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"GENOMIC_CONTENT", + f"{ASSEMBLY_DIR_A}_protein.faa.gz": b"PROTEIN_CONTENT", + f"{ASSEMBLY_DIR_A}_rna.fna.gz": b"RNA_CONTENT", + f"{ASSEMBLY_DIR_A}_assembly_report.txt": b"ASSEMBLY_REPORT", + f"{ASSEMBLY_DIR_A}_assembly_stats.txt": b"ASSEMBLY_STATS", + f"{ASSEMBLY_DIR_A}_cds_from_genomic.fna.gz": b"CDS_CONTENT", + } + seed_lakehouse(s3, test_bucket, ACCESSION_A, many_files, PATH_PREFIX, ASSEMBLY_DIR_A) + + updated_manifest = _write_manifest(tmp_path, [ACCESSION_A], "updated_manifest.txt") + + archived = _archive_assemblies( + str(updated_manifest), + lakehouse_bucket=test_bucket, + ncbi_release="2024-01", + archive_reason="updated", + lakehouse_key_prefix=PATH_PREFIX, + delete_source=False, + ) + + assert archived == len(many_files) + + # Verify every archived file has correct content + rel = build_accession_path(ASSEMBLY_DIR_A) + for fname, expected_body in many_files.items(): + archive_key = f"{PATH_PREFIX}archive/2024-01/updated/{rel}{fname}" + obj = s3.get_object(Bucket=test_bucket, Key=archive_key) + actual_body = obj["Body"].read() + assert actual_body == expected_body, f"Content mismatch for {fname}" + + def test_archive_key_paths_are_correct( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + ) -> None: + """Archived keys follow the exact ``archive/{release}/{reason}/{rel_path}`` pattern.""" + from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies + + s3 = minio_s3_client + files = {f"{ASSEMBLY_DIR_B}_genomic.fna.gz": b"content"} + seed_lakehouse(s3, test_bucket, ACCESSION_B, files, PATH_PREFIX, ASSEMBLY_DIR_B) + + removed_manifest = _write_manifest(tmp_path, [ACCESSION_B], "removed_manifest.txt") + _archive_assemblies( + str(removed_manifest), + lakehouse_bucket=test_bucket, + ncbi_release="2024-02", + archive_reason="replaced_or_suppressed", + lakehouse_key_prefix=PATH_PREFIX, + delete_source=False, + ) + + rel = build_accession_path(ASSEMBLY_DIR_B) + expected_key = f"{PATH_PREFIX}archive/2024-02/replaced_or_suppressed/{rel}{ASSEMBLY_DIR_B}_genomic.fna.gz" + resp = s3.head_object(Bucket=test_bucket, Key=expected_key) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestArchiveDeleteSourceBatch: + """Verify batch delete removes all source objects after concurrent copy.""" + + def test_all_sources_deleted_after_archive( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + ) -> None: + """After archive with delete_source=True, no source objects remain.""" + from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies + + s3 = minio_s3_client + many_files = { + f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"genomic", + f"{ASSEMBLY_DIR_A}_protein.faa.gz": b"protein", + f"{ASSEMBLY_DIR_A}_rna.fna.gz": b"rna", + f"{ASSEMBLY_DIR_A}_assembly_report.txt": b"report", + } + source_keys = seed_lakehouse(s3, test_bucket, ACCESSION_A, many_files, PATH_PREFIX, ASSEMBLY_DIR_A) + + removed_manifest = _write_manifest(tmp_path, [ACCESSION_A], "removed_manifest.txt") + archived = _archive_assemblies( + str(removed_manifest), + lakehouse_bucket=test_bucket, + ncbi_release="2024-01", + archive_reason="replaced_or_suppressed", + lakehouse_key_prefix=PATH_PREFIX, + delete_source=True, + ) + + assert archived == len(many_files) + # Source keys must all be gone + for key in source_keys: + remaining = list_all_keys(s3, test_bucket, key) + assert len(remaining) == 0, f"Source not deleted: {key}" + + def test_archive_present_source_gone( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + ) -> None: + """Archive destinations exist AND sources are gone after replaced_or_suppressed archive.""" + from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies + + s3 = minio_s3_client + files = { + f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"genomic", + f"{ASSEMBLY_DIR_A}_protein.faa.gz": b"protein", + } + seed_lakehouse(s3, test_bucket, ACCESSION_A, files, PATH_PREFIX, ASSEMBLY_DIR_A) + + removed_manifest = _write_manifest(tmp_path, [ACCESSION_A], "removed_manifest.txt") + _archive_assemblies( + str(removed_manifest), + lakehouse_bucket=test_bucket, + ncbi_release="2024-01", + archive_reason="replaced_or_suppressed", + lakehouse_key_prefix=PATH_PREFIX, + delete_source=True, + ) + + rel = build_accession_path(ASSEMBLY_DIR_A) + # Archive keys present + archive_keys = list_all_keys(s3, test_bucket, f"{PATH_PREFIX}archive/2024-01/replaced_or_suppressed/") + assert len(archive_keys) == len(files), f"Expected {len(files)} archive keys, got: {archive_keys}" + # Source keys absent + source_keys = list_all_keys(s3, test_bucket, f"{PATH_PREFIX}{rel}") + assert len(source_keys) == 0, f"Source objects remain: {source_keys}" + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestPartialArchiveResume: + """Corner case: a prior archive run was interrupted mid-way. + + Re-running must complete cleanly without errors, leave all archive keys + present with current content, and (when delete_source=True) remove all + source keys regardless of which files were processed in the prior run. + """ + + def test_partial_updated_archive_resumes( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + ) -> None: + """Re-running after a partial updated archive overwrites stale copies and archives missing files. + + Scenario: 3 files, file_a was archived in a prior run (stale content), + file_b and file_c were not. Re-run should overwrite file_a with current + content and archive file_b, file_c. + """ + from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies + + s3 = minio_s3_client + rel = build_accession_path(ASSEMBLY_DIR_A) + + file_a = f"{ASSEMBLY_DIR_A}_genomic.fna.gz" + file_b = f"{ASSEMBLY_DIR_A}_protein.faa.gz" + file_c = f"{ASSEMBLY_DIR_A}_rna.fna.gz" + + current_content = {file_a: b"current-genomic", file_b: b"current-protein", file_c: b"current-rna"} + seed_lakehouse(s3, test_bucket, ACCESSION_A, current_content, PATH_PREFIX, ASSEMBLY_DIR_A) + + # Pre-seed a stale archive copy for file_a (simulating prior partial run) + archive_prefix = f"{PATH_PREFIX}archive/2024-01/updated/{rel}" + s3.put_object(Bucket=test_bucket, Key=f"{archive_prefix}{file_a}", Body=b"stale-genomic") + + updated_manifest = _write_manifest(tmp_path, [ACCESSION_A], "updated_manifest.txt") + archived = _archive_assemblies( + str(updated_manifest), + lakehouse_bucket=test_bucket, + ncbi_release="2024-01", + archive_reason="updated", + lakehouse_key_prefix=PATH_PREFIX, + delete_source=False, + ) + + # All 3 files counted + assert archived == 3 # noqa: PLR2004 + # file_a overwritten with current content + obj_a = s3.get_object(Bucket=test_bucket, Key=f"{archive_prefix}{file_a}") + assert obj_a["Body"].read() == b"current-genomic", "file_a archive should be overwritten" + # file_b and file_c now archived + for fname in (file_b, file_c): + resp = s3.head_object(Bucket=test_bucket, Key=f"{archive_prefix}{fname}") + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + # Sources untouched (delete_source=False) + source_keys = list_all_keys(s3, test_bucket, f"{PATH_PREFIX}{rel}") + assert len(source_keys) == len(current_content) + + def test_partial_replaced_archive_resumes_and_deletes( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + ) -> None: + """Re-running replaced_or_suppressed archive after partial run completes and deletes all sources. + + Scenario: file_a was copied+deleted in prior run (no longer at source), + file_b was copied but NOT deleted (still at source), file_c was untouched. + Re-run processes file_b and file_c, deletes both. Result: no sources remain. + """ + from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies + + s3 = minio_s3_client + rel = build_accession_path(ASSEMBLY_DIR_A) + + file_a = f"{ASSEMBLY_DIR_A}_genomic.fna.gz" + file_b = f"{ASSEMBLY_DIR_A}_protein.faa.gz" + file_c = f"{ASSEMBLY_DIR_A}_rna.fna.gz" + archive_prefix = f"{PATH_PREFIX}archive/2024-01/replaced_or_suppressed/{rel}" + + # Only file_b and file_c remain at source (file_a already gone) + s3.put_object( + Bucket=test_bucket, + Key=f"{PATH_PREFIX}{rel}{file_b}", + Body=b"protein", + Metadata={"md5": hashlib.md5(b"protein").hexdigest()}, # noqa: S324 + ) + s3.put_object( + Bucket=test_bucket, + Key=f"{PATH_PREFIX}{rel}{file_c}", + Body=b"rna", + Metadata={"md5": hashlib.md5(b"rna").hexdigest()}, # noqa: S324 + ) + # file_a already at archive destination + s3.put_object(Bucket=test_bucket, Key=f"{archive_prefix}{file_a}", Body=b"genomic") + + removed_manifest = _write_manifest(tmp_path, [ACCESSION_A], "removed_manifest.txt") + archived = _archive_assemblies( + str(removed_manifest), + lakehouse_bucket=test_bucket, + ncbi_release="2024-01", + archive_reason="replaced_or_suppressed", + lakehouse_key_prefix=PATH_PREFIX, + delete_source=True, + ) + + # 2 newly archived (file_b and file_c) + assert archived == 2 # noqa: PLR2004 + # file_b and file_c archive keys exist + for fname in (file_b, file_c): + resp = s3.head_object(Bucket=test_bucket, Key=f"{archive_prefix}{fname}") + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + # No source keys remain (file_b and file_c were deleted) + source_keys = list_all_keys(s3, test_bucket, f"{PATH_PREFIX}{rel}") + assert len(source_keys) == 0, f"Source objects remain: {source_keys}" + # file_a archive key is still intact + resp_a = s3.head_object(Bucket=test_bucket, Key=f"{archive_prefix}{file_a}") + assert resp_a["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + + def test_full_rerun_after_complete_archive_is_idempotent( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + ) -> None: + """Running archive again when all files already exist at archive paths is safe (no errors).""" + from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies + + s3 = minio_s3_client + files = { + f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"genomic", + f"{ASSEMBLY_DIR_A}_protein.faa.gz": b"protein", + } + seed_lakehouse(s3, test_bucket, ACCESSION_A, files, PATH_PREFIX, ASSEMBLY_DIR_A) + + updated_manifest = _write_manifest(tmp_path, [ACCESSION_A], "updated_manifest.txt") + + # First run — archives all files + archived_1 = _archive_assemblies( + str(updated_manifest), + lakehouse_bucket=test_bucket, + ncbi_release="2024-01", + archive_reason="updated", + lakehouse_key_prefix=PATH_PREFIX, + delete_source=False, + ) + + # Second run — same manifest, same source files still present + archived_2 = _archive_assemblies( + str(updated_manifest), + lakehouse_bucket=test_bucket, + ncbi_release="2024-01", + archive_reason="updated", + lakehouse_key_prefix=PATH_PREFIX, + delete_source=False, + ) + + assert archived_1 == len(files) + assert archived_2 == len(files) + # Archive keys still present with correct content + rel = build_accession_path(ASSEMBLY_DIR_A) + for fname, expected_body in files.items(): + key = f"{PATH_PREFIX}archive/2024-01/updated/{rel}{fname}" + obj = s3.get_object(Bucket=test_bucket, Key=key) + assert obj["Body"].read() == expected_body + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestArchiveMultiAccessionManifest: + """Multiple accessions in a single manifest are all archived.""" + + def test_two_accessions_both_archived( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + ) -> None: + """Both accessions are archived with correct keys when listed in one manifest.""" + from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies + + s3 = minio_s3_client + + files_a = {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"genomic-A"} + files_b = {f"{ASSEMBLY_DIR_B}_genomic.fna.gz": b"genomic-B"} + seed_lakehouse(s3, test_bucket, ACCESSION_A, files_a, PATH_PREFIX, ASSEMBLY_DIR_A) + seed_lakehouse(s3, test_bucket, ACCESSION_B, files_b, PATH_PREFIX, ASSEMBLY_DIR_B) + + manifest = _write_manifest(tmp_path, [ACCESSION_A, ACCESSION_B], "removed_manifest.txt") + + archived = _archive_assemblies( + str(manifest), + lakehouse_bucket=test_bucket, + ncbi_release="2024-01", + archive_reason="replaced_or_suppressed", + lakehouse_key_prefix=PATH_PREFIX, + delete_source=True, + ) + + assert archived == 2 # noqa: PLR2004 + rel_a = build_accession_path(ASSEMBLY_DIR_A) + rel_b = build_accession_path(ASSEMBLY_DIR_B) + key_a = f"{PATH_PREFIX}archive/2024-01/replaced_or_suppressed/{rel_a}{ASSEMBLY_DIR_A}_genomic.fna.gz" + key_b = f"{PATH_PREFIX}archive/2024-01/replaced_or_suppressed/{rel_b}{ASSEMBLY_DIR_B}_genomic.fna.gz" + assert s3.head_object(Bucket=test_bucket, Key=key_a)["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + assert s3.head_object(Bucket=test_bucket, Key=key_b)["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + # Sources deleted + assert len(list_all_keys(s3, test_bucket, f"{PATH_PREFIX}{rel_a}")) == 0 + assert len(list_all_keys(s3, test_bucket, f"{PATH_PREFIX}{rel_b}")) == 0 + + def test_three_accessions_correct_archive_reason_segment( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + ) -> None: + """Archive keys for all three accessions include the archive_reason segment.""" + from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies + + s3 = minio_s3_client + accessions_and_dirs = [ + (ACCESSION_A, ASSEMBLY_DIR_A), + (ACCESSION_B, ASSEMBLY_DIR_B), + (ACCESSION_C, ASSEMBLY_DIR_C), + ] + for accession, assembly_dir in accessions_and_dirs: + seed_lakehouse( + s3, + test_bucket, + accession, + {f"{assembly_dir}_genomic.fna.gz": b"data"}, + PATH_PREFIX, + assembly_dir, + ) + + manifest = _write_manifest(tmp_path, [acc for acc, _ in accessions_and_dirs], "removed_manifest.txt") + _archive_assemblies( + str(manifest), + lakehouse_bucket=test_bucket, + ncbi_release="2024-03", + archive_reason="replaced_or_suppressed", + lakehouse_key_prefix=PATH_PREFIX, + delete_source=False, + ) + + all_archive_keys = list_all_keys(s3, test_bucket, f"{PATH_PREFIX}archive/2024-03/") + assert len(all_archive_keys) == 3 # noqa: PLR2004 + for key in all_archive_keys: + assert "/replaced_or_suppressed/" in key, f"Archive key missing reason segment: {key}" + assert "/2024-03/" in key, f"Archive key missing release segment: {key}" + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestArchiveDryRunParallel: + """Dry-run with many files leaves everything unchanged.""" + + def test_dry_run_no_copies_no_deletes( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + ) -> None: + """Dry-run with multiple files per accession creates no archive keys and keeps sources.""" + from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies + + s3 = minio_s3_client + many_files = { + f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"genomic", + f"{ASSEMBLY_DIR_A}_protein.faa.gz": b"protein", + f"{ASSEMBLY_DIR_A}_rna.fna.gz": b"rna", + } + source_keys = seed_lakehouse(s3, test_bucket, ACCESSION_A, many_files, PATH_PREFIX, ASSEMBLY_DIR_A) + + removed_manifest = _write_manifest(tmp_path, [ACCESSION_A], "removed_manifest.txt") + archived = _archive_assemblies( + str(removed_manifest), + lakehouse_bucket=test_bucket, + ncbi_release="2024-01", + archive_reason="replaced_or_suppressed", + lakehouse_key_prefix=PATH_PREFIX, + delete_source=True, + dry_run=True, + ) + + assert archived == len(many_files) + # No archive keys + archive_keys = list_all_keys(s3, test_bucket, f"{PATH_PREFIX}archive/") + assert len(archive_keys) == 0, f"Dry-run created archive keys: {archive_keys}" + # All sources still present + for key in source_keys: + remaining = list_all_keys(s3, test_bucket, key) + assert len(remaining) == 1, f"Source missing after dry-run: {key}" diff --git a/tests/ncbi_ftp/test_promote.py b/tests/ncbi_ftp/test_promote.py index 3c7bc05e..976d06e0 100644 --- a/tests/ncbi_ftp/test_promote.py +++ b/tests/ncbi_ftp/test_promote.py @@ -231,3 +231,324 @@ def test_archive_assemblies_unknown_release_fallback( archive_key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/unknown/unknown/raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" assert mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_key).get("KeyCount", 0) == 1 + + +# ── Concurrent / multi-file archive (new behaviour) ───────────────────── + + +@pytest.mark.s3 +def test_archive_assemblies_multi_file_all_copied( + mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path +) -> None: + """All files for an accession are copied concurrently — none missed.""" + accession = "GCF_000001215.4" + asm_dir = f"{accession}_Release_6" + base = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/" + file_names = [ + f"{accession}_genomic.fna.gz", + f"{accession}_protein.faa.gz", + f"{accession}_rna.fna.gz", + f"{accession}_assembly_report.txt", + f"{accession}_assembly_stats.txt", + ] + for fname in file_names: + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{base}{fname}", Body=fname.encode()) + + manifest = tmp_path / "updated.txt" + manifest.write_text(f"{accession}\n") + + archived = _archive_assemblies( + str(manifest), + lakehouse_bucket=TEST_BUCKET, + ncbi_release="2024-01", + archive_reason="updated", + delete_source=False, + ) + + assert archived == len(file_names) + archive_base = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/updated/raw_data/GCF/000/001/215/{asm_dir}/" + for fname in file_names: + resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{archive_base}{fname}") + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + + +@pytest.mark.s3 +def test_archive_assemblies_multi_file_content_preserved( + mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path +) -> None: + """Archive copies preserve byte-for-byte content of each file.""" + accession = "GCF_000001215.4" + asm_dir = f"{accession}_Release_6" + base = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/" + files = { + f"{accession}_genomic.fna.gz": b"\x1f\x8bGENOMIC", + f"{accession}_protein.faa.gz": b"\x1f\x8bPROTEIN", + } + for fname, body in files.items(): + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{base}{fname}", Body=body) + + manifest = tmp_path / "updated.txt" + manifest.write_text(f"{accession}\n") + + _archive_assemblies( + str(manifest), + lakehouse_bucket=TEST_BUCKET, + ncbi_release="2024-01", + archive_reason="updated", + delete_source=False, + ) + + archive_base = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/updated/raw_data/GCF/000/001/215/{asm_dir}/" + for fname, original_body in files.items(): + obj = mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=f"{archive_base}{fname}") + assert obj["Body"].read() == original_body, f"Content mismatch for {fname}" + + +@pytest.mark.s3 +def test_archive_assemblies_multi_file_delete_all( + mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path +) -> None: + """Batch delete removes ALL source files when delete_source=True.""" + accession = "GCF_000005845.2" + asm_dir = f"{accession}_ASM584v2" + base = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/005/845/{asm_dir}/" + file_names = [ + f"{accession}_genomic.fna.gz", + f"{accession}_protein.faa.gz", + f"{accession}_assembly_report.txt", + ] + for fname in file_names: + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{base}{fname}", Body=b"data") + + manifest = tmp_path / "removed.txt" + manifest.write_text(f"{accession}\n") + + archived = _archive_assemblies( + str(manifest), + lakehouse_bucket=TEST_BUCKET, + ncbi_release="2024-03", + archive_reason="replaced_or_suppressed", + delete_source=True, + ) + + assert archived == len(file_names) + # All sources deleted + for fname in file_names: + result = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=f"{base}{fname}") + assert result.get("KeyCount", 0) == 0, f"Source not deleted: {fname}" + # All archives present + archive_base = ( + f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-03/replaced_or_suppressed/raw_data/GCF/000/005/845/{asm_dir}/" + ) + for fname in file_names: + resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{archive_base}{fname}") + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + + +# ── Partial-archive idempotency ────────────────────────────────────────── + + +@pytest.mark.s3 +def test_archive_assemblies_partial_already_archived_overwritten( + mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path +) -> None: + """Re-running archive after a partial run overwrites the already-archived files. + + Simulates a partial failure: file_a was archived, file_b was not. + The second run should archive both file_a (overwrite) and file_b. + """ + accession = "GCF_000001215.4" + asm_dir = f"{accession}_Release_6" + base = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/" + archive_base = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/updated/raw_data/GCF/000/001/215/{asm_dir}/" + + file_a = f"{accession}_genomic.fna.gz" + file_b = f"{accession}_protein.faa.gz" + + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{base}{file_a}", Body=b"new-genomic") + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{base}{file_b}", Body=b"new-protein") + # Simulate partial prior run: file_a already archived with stale content + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{archive_base}{file_a}", Body=b"stale-genomic") + + manifest = tmp_path / "updated.txt" + manifest.write_text(f"{accession}\n") + + archived = _archive_assemblies( + str(manifest), + lakehouse_bucket=TEST_BUCKET, + ncbi_release="2024-01", + archive_reason="updated", + delete_source=False, + ) + + assert archived == 2 # noqa: PLR2004 + # file_a should now have the current content (overwritten) + obj_a = mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=f"{archive_base}{file_a}") + assert obj_a["Body"].read() == b"new-genomic", "Re-run should overwrite stale archive" + # file_b should now be archived + obj_b = mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=f"{archive_base}{file_b}") + assert obj_b["Body"].read() == b"new-protein" + + +@pytest.mark.s3 +def test_archive_assemblies_partial_delete_resumes( + mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path +) -> None: + """Re-running replaced_or_suppressed archive after partial copy+delete is safe. + + Simulates: file_a was copied+deleted, file_b was copied but NOT deleted, + file_c was not touched. The re-run finds only file_b and file_c present + (file_a is gone), archives both, and deletes both. + """ + accession = "GCF_000005845.2" + asm_dir = f"{accession}_ASM584v2" + base = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/005/845/{asm_dir}/" + archive_base = ( + f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-03/replaced_or_suppressed/raw_data/GCF/000/005/845/{asm_dir}/" + ) + + file_b = f"{accession}_protein.faa.gz" + file_c = f"{accession}_assembly_report.txt" + + # file_a already gone (deleted in first partial run) + # file_b present at source (not yet deleted from first partial run) + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{base}{file_b}", Body=b"protein") + # file_c present at source (not touched at all) + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{base}{file_c}", Body=b"report") + # file_a already at archive destination + mock_s3_client_no_checksum.put_object( + Bucket=TEST_BUCKET, Key=f"{archive_base}{accession}_genomic.fna.gz", Body=b"genomic" + ) + + manifest = tmp_path / "removed.txt" + manifest.write_text(f"{accession}\n") + + archived = _archive_assemblies( + str(manifest), + lakehouse_bucket=TEST_BUCKET, + ncbi_release="2024-03", + archive_reason="replaced_or_suppressed", + delete_source=True, + ) + + # Only the 2 remaining source files were archived + assert archived == 2 # noqa: PLR2004 + # Both now gone from source + for fname in (file_b, file_c): + result = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=f"{base}{fname}") + assert result.get("KeyCount", 0) == 0, f"Expected {fname} deleted" + # file_a archive still intact (not touched by re-run) + resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{archive_base}{accession}_genomic.fna.gz") + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + + +@pytest.mark.s3 +def test_archive_assemblies_idempotent_updated_reruns_cleanly( + mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path +) -> None: + """Running updated archive twice on the same data produces the same result.""" + accession = "GCF_000001215.4" + asm_dir = f"{accession}_Release_6" + base = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/" + file_names = [f"{accession}_genomic.fna.gz", f"{accession}_protein.faa.gz"] + for fname in file_names: + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{base}{fname}", Body=b"content") + + manifest = tmp_path / "updated.txt" + manifest.write_text(f"{accession}\n") + + archived_1 = _archive_assemblies( + str(manifest), lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated" + ) + archived_2 = _archive_assemblies( + str(manifest), lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated" + ) + + assert archived_1 == len(file_names) + assert archived_2 == len(file_names) + # Sources still present after both runs (delete_source=False) + for fname in file_names: + resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{base}{fname}") + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + + +@pytest.mark.s3 +def test_archive_assemblies_multi_accession_manifest( + mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path +) -> None: + """Multiple accessions in a single manifest are all archived.""" + accessions = [ + ("GCF_000001215.4", "GCF_000001215.4_Release_6", "GCF/000/001/215"), + ("GCF_000005845.2", "GCF_000005845.2_ASM584v2", "GCF/000/005/845"), + ] + for accession, asm_dir, path in accessions: + key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/{path}/{asm_dir}/{accession}_genomic.fna.gz" + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"data") + + manifest = tmp_path / "updated.txt" + manifest.write_text("\n".join(acc for acc, _, _ in accessions) + "\n") + + archived = _archive_assemblies( + str(manifest), lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated" + ) + + assert archived == len(accessions) + for accession, asm_dir, path in accessions: + archive_key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/updated/raw_data/{path}/{asm_dir}/{accession}_genomic.fna.gz" + result = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_key) + assert result.get("KeyCount", 0) == 1, f"Archive missing for {accession}" + + +@pytest.mark.s3 +def test_archive_assemblies_dry_run_multi_file( + mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path +) -> None: + """dry_run with multiple files per accession makes no copies and no deletes.""" + accession = "GCF_000005845.2" + asm_dir = f"{accession}_ASM584v2" + base = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/005/845/{asm_dir}/" + file_names = [f"{accession}_genomic.fna.gz", f"{accession}_protein.faa.gz", f"{accession}_rna.fna.gz"] + for fname in file_names: + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{base}{fname}", Body=b"data") + + manifest = tmp_path / "removed.txt" + manifest.write_text(f"{accession}\n") + + archived = _archive_assemblies( + str(manifest), + lakehouse_bucket=TEST_BUCKET, + ncbi_release="2024-01", + archive_reason="replaced_or_suppressed", + delete_source=True, + dry_run=True, + ) + + # Reported count matches + assert archived == len(file_names) + # No actual archive keys created + archive_prefix = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/" + result = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_prefix) + assert result.get("KeyCount", 0) == 0 + # Sources untouched + for fname in file_names: + resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{base}{fname}") + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + + +@pytest.mark.s3 +def test_archive_assemblies_invalid_accession_skipped( + mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path +) -> None: + """Malformed accession lines are skipped; valid ones still archived.""" + accession = "GCF_000001215.4" + asm_dir = f"{accession}_Release_6" + key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"data") + + manifest = tmp_path / "mixed.txt" + manifest.write_text("NOT_AN_ACCESSION\n\n \n" + f"{accession}\n") + + archived = _archive_assemblies( + str(manifest), lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated" + ) + assert archived == 1 diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index 36dd9205..8c6c2f9b 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -22,6 +22,7 @@ copy_directory, copy_object, delete_object, + delete_objects, download_file, get_s3_client, head_object, @@ -1017,3 +1018,34 @@ def test_delete_object_no_such_bucket() -> None: assert object_exists(s3_path) is False with pytest.raises(Exception, match="The specified bucket does not exist"): delete_object(s3_path) + + +# delete_objects +@pytest.mark.s3 +def test_delete_objects_removes_all(mock_s3_client: Any) -> None: + """delete_objects removes every listed key in a single call.""" + keys = ["bulk/a.txt", "bulk/b.txt", "bulk/c.txt"] + for k in keys: + mock_s3_client.put_object(Bucket=CDM_LAKE_BUCKET, Key=k, Body=b"data") + + errors = delete_objects(CDM_LAKE_BUCKET, keys) + + assert errors == [] + for k in keys: + assert object_exists(f"{CDM_LAKE_BUCKET}/{k}") is False + + +@pytest.mark.s3 +def test_delete_objects_empty_list_is_noop(mock_s3_client: Any) -> None: + """delete_objects with an empty list makes no API call and returns no errors.""" + mock_s3_client.put_object(Bucket=CDM_LAKE_BUCKET, Key="keep/me.txt", Body=b"safe") + errors = delete_objects(CDM_LAKE_BUCKET, []) + assert errors == [] + assert object_exists(f"{CDM_LAKE_BUCKET}/keep/me.txt") is True + + +@pytest.mark.s3 +def test_delete_objects_nonexistent_keys_no_error(mock_s3_client: Any) -> None: + """Deleting keys that don't exist returns no errors (S3 delete is idempotent).""" + errors = delete_objects(CDM_LAKE_BUCKET, ["ghost/a.txt", "ghost/b.txt"]) + assert errors == [] From 409626d423ddd9d76237c5fc924ad2130d7f07ea Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 30 Apr 2026 12:47:44 -0700 Subject: [PATCH 076/128] optimize promotion step Co-authored-by: Copilot --- src/cdm_data_loaders/ncbi_ftp/promote.py | 140 +++---- tests/integration/test_promote_e2e.py | 370 ++++++++++++++++++ tests/ncbi_ftp/test_promote.py | 453 +++++++++++++++++++++++ 3 files changed, 896 insertions(+), 67 deletions(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index 52a71612..0c50118d 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -27,7 +27,6 @@ from cdm_data_loaders.utils.cdm_logger import get_cdm_logger from cdm_data_loaders.utils.s3 import ( copy_object, - delete_object, delete_objects, get_s3_client, object_exists, @@ -179,6 +178,50 @@ def _promote_data_files( # noqa: PLR0913, PLR0915 if acc_match and adir_match: assembly_files[(adir_match.group(1), acc_match.group(1))].append(staged_key) + def _promote_one(staged_key: str) -> tuple[DescriptorResource, str]: + """Download one staged file, re-upload to Lakehouse with MD5 metadata. + + :return: ``(resource_dict, staged_key)`` on success; raises on failure. + """ + rel_path = staged_key[len(normalized_staging_prefix) :] + final_key = lakehouse_key_prefix + rel_path + final_key_path = PurePosixPath(final_key) + + with tempfile.NamedTemporaryFile(delete=False) as tmp: + tmp_path = tmp.name + try: + s3.download_file(Bucket=staging_bucket, Key=staged_key, Filename=tmp_path) + + metadata: dict[str, str] = {} + md5_key = staged_key + ".md5" + if md5_key in sidecars: + md5_obj = s3.get_object(Bucket=staging_bucket, Key=md5_key) + metadata["md5"] = md5_obj["Body"].read().decode().strip() + + upload_succeeded = upload_file( + tmp_path, + f"{lakehouse_bucket}/{final_key_path.parent}", + tags=metadata, + object_name=final_key_path.name, + show_progress=False, + ) + if not upload_succeeded: + msg = f"upload_file returned False for {staged_key}" + raise RuntimeError(msg) + + fname = final_key_path.name + ext = fname.rsplit(".", 1)[-1] if "." in fname else "" + resource: DescriptorResource = { + "name": fname.lower(), + "path": final_key, + "format": ext, + "bytes": Path(tmp_path).stat().st_size, + "hash": metadata.get("md5"), + } + return resource, staged_key + finally: + Path(tmp_path).unlink() + total_files = sum(len(v) for v in assembly_files.values()) _dry_run_log_count = 0 with tqdm.tqdm(total=total_files, unit="file", desc="Promoting") as pbar: @@ -187,12 +230,10 @@ def _promote_data_files( # noqa: PLR0913, PLR0915 resources: list[DescriptorResource] = [] promoted_keys: list[str] = [] - for staged_key in files: - rel_path = staged_key[len(normalized_staging_prefix) :] - final_key = lakehouse_key_prefix + rel_path - final_key_path = PurePosixPath(final_key) - - if dry_run: + if dry_run: + for staged_key in files: + rel_path = staged_key[len(normalized_staging_prefix) :] + final_key = lakehouse_key_prefix + rel_path if _dry_run_log_count < 10: logger.info("[dry-run] would promote: %s -> %s", staged_key, final_key) else: @@ -200,56 +241,24 @@ def _promote_data_files( # noqa: PLR0913, PLR0915 _dry_run_log_count += 1 promoted += 1 pbar.update(1) - continue - - file_promoted = False - try: - with tempfile.NamedTemporaryFile(delete=False) as tmp: - tmp_path = tmp.name + continue + + # Download and re-upload all files for this assembly concurrently + n_workers = min(32, len(files)) + with ThreadPoolExecutor(max_workers=n_workers) as executor: + futures = {executor.submit(_promote_one, key): key for key in files} + for future in as_completed(futures): + staged_key = futures[future] try: - s3.download_file(Bucket=staging_bucket, Key=staged_key, Filename=tmp_path) - - # Read MD5 from sidecar - metadata: dict[str, str] = {} - md5_key = staged_key + ".md5" - if md5_key in sidecars: - md5_obj = s3.get_object(Bucket=staging_bucket, Key=md5_key) - metadata["md5"] = md5_obj["Body"].read().decode().strip() - - upload_succeeded = upload_file( - tmp_path, - f"{lakehouse_bucket}/{final_key_path.parent}", - tags=metadata, - object_name=final_key_path.name, - show_progress=False, - ) - if not upload_succeeded: - logger.error("Failed to upload promoted file %s to %s", staged_key, final_key) - else: - promoted += 1 - promoted_keys.append(staged_key) - promoted_accessions.add(acc) - file_promoted = True - - fname = final_key_path.name - ext = fname.rsplit(".", 1)[-1] if "." in fname else "" - resource: DescriptorResource = { - "name": fname.lower(), - "path": final_key, - "format": ext, - "bytes": Path(tmp_path).stat().st_size, - "hash": metadata.get("md5"), - } - resources.append(resource) - - finally: - Path(tmp_path).unlink() - except Exception: - logger.exception("Failed to promote %s", staged_key) - - if not file_promoted: - assembly_failed += 1 - pbar.update(1) + resource, _ = future.result() + resources.append(resource) + promoted_keys.append(staged_key) + promoted += 1 + promoted_accessions.add(acc) + except Exception: + logger.exception("Failed to promote %s", staged_key) + assembly_failed += 1 + pbar.update(1) failed += assembly_failed @@ -266,18 +275,15 @@ def _promote_data_files( # noqa: PLR0913, PLR0915 except Exception: logger.exception("Failed to write descriptor for %s", adir) - for staged_key in promoted_keys: - try: - delete_object(f"{staging_bucket}/{staged_key}") - except Exception: - logger.warning("Failed to delete staged file %s", staged_key) + # Batch-delete all staged data files and their sidecars in one API call + keys_to_delete = list(promoted_keys) + for key in promoted_keys: for sidecar_ext in (".md5", ".crc64nvme"): - sidecar_key = staged_key + sidecar_ext - if sidecar_key in sidecars: - try: - delete_object(f"{staging_bucket}/{sidecar_key}") - except Exception: - logger.warning("Failed to delete staged sidecar %s", sidecar_key) + if key + sidecar_ext in sidecars: + keys_to_delete.append(key + sidecar_ext) + del_errors = delete_objects(staging_bucket, keys_to_delete) + for err in del_errors: + logger.warning("Failed to delete staged file %s: %s", err.get("Key"), err.get("Message")) return promoted, failed, descriptors_written, promoted_accessions diff --git a/tests/integration/test_promote_e2e.py b/tests/integration/test_promote_e2e.py index b805f03c..d412ea27 100644 --- a/tests/integration/test_promote_e2e.py +++ b/tests/integration/test_promote_e2e.py @@ -944,3 +944,373 @@ def test_dry_run_no_copies_no_deletes( for key in source_keys: remaining = list_all_keys(s3, test_bucket, key) assert len(remaining) == 1, f"Source missing after dry-run: {key}" + + +# ── Concurrent promotion tests ──────────────────────────────────────────── + + +def _stage_many( + s3: object, + bucket: str, + assembly_dir: str, + files: dict[str, bytes], + *, + with_md5: bool = True, +) -> None: + """Stage *files* with optional .md5 sidecars under the standard staging prefix.""" + rel = build_accession_path(assembly_dir) + base = f"{STAGING_PREFIX}{rel}" + for fname, content in files.items(): + key = f"{base}{fname}" + s3.put_object(Bucket=bucket, Key=key, Body=content) + if with_md5: + s3.put_object(Bucket=bucket, Key=f"{key}.md5", Body=_md5(content).encode()) + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestPromoteMultiFileConcurrent: + """Verify concurrent promotion lands all files with correct content and MD5.""" + + def test_six_files_all_promoted_with_correct_content( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + ) -> None: + """Every staged file arrives at the correct final key with byte-identical content.""" + s3 = minio_s3_client + many_files = { + f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"GENOMIC", + f"{ASSEMBLY_DIR_A}_protein.faa.gz": b"PROTEIN", + f"{ASSEMBLY_DIR_A}_rna.fna.gz": b"RNA", + f"{ASSEMBLY_DIR_A}_assembly_report.txt": b"REPORT", + f"{ASSEMBLY_DIR_A}_assembly_stats.txt": b"STATS", + f"{ASSEMBLY_DIR_A}_cds_from_genomic.fna.gz": b"CDS", + } + _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, many_files) + + report = promote_from_s3( + staging_key_prefix=STAGING_PREFIX, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, + lakehouse_key_prefix=PATH_PREFIX, + ) + + assert report["promoted"] == len(many_files) + assert report["failed"] == 0 + + rel = build_accession_path(ASSEMBLY_DIR_A) + for fname, expected_body in many_files.items(): + key = f"{PATH_PREFIX}{rel}{fname}" + obj = s3.get_object(Bucket=test_bucket, Key=key) + assert obj["Body"].read() == expected_body, f"Content mismatch: {fname}" + + def test_md5_metadata_correct_per_file( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + ) -> None: + """Each promoted file carries MD5 metadata matching its own content, not another file's.""" + s3 = minio_s3_client + files = { + f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"GENOMIC_UNIQUE", + f"{ASSEMBLY_DIR_A}_protein.faa.gz": b"PROTEIN_UNIQUE", + f"{ASSEMBLY_DIR_A}_rna.fna.gz": b"RNA_UNIQUE", + } + _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, files, with_md5=True) + + promote_from_s3( + staging_key_prefix=STAGING_PREFIX, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, + lakehouse_key_prefix=PATH_PREFIX, + ) + + rel = build_accession_path(ASSEMBLY_DIR_A) + for fname, content in files.items(): + key = f"{PATH_PREFIX}{rel}{fname}" + meta = get_object_metadata(s3, test_bucket, key) + assert meta.get("md5") == _md5(content), f"Wrong MD5 metadata on {fname}" + + def test_file_without_sidecar_has_no_md5_metadata( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + ) -> None: + """A file staged without a .md5 sidecar is promoted but has no md5 metadata key.""" + s3 = minio_s3_client + fname = f"{ASSEMBLY_DIR_A}_genomic.fna.gz" + _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, {fname: FAKE_GENOMIC}, with_md5=False) + + promote_from_s3( + staging_key_prefix=STAGING_PREFIX, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, + lakehouse_key_prefix=PATH_PREFIX, + ) + + rel = build_accession_path(ASSEMBLY_DIR_A) + meta = get_object_metadata(s3, test_bucket, f"{PATH_PREFIX}{rel}{fname}") + assert "md5" not in meta, f"Expected no md5 metadata, got: {meta}" + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestPromoteStagingCleanup: + """After a fully successful promote, all staged files and sidecars are deleted.""" + + def test_staged_data_files_deleted( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + ) -> None: + """Data files are removed from staging after a successful assembly promote.""" + s3 = minio_s3_client + files = { + f"{ASSEMBLY_DIR_A}_genomic.fna.gz": FAKE_GENOMIC, + f"{ASSEMBLY_DIR_A}_protein.faa.gz": FAKE_PROTEIN, + } + _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, files, with_md5=False) + + promote_from_s3( + staging_key_prefix=STAGING_PREFIX, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, + lakehouse_key_prefix=PATH_PREFIX, + ) + + remaining_staging = list_all_keys(s3, staging_test_bucket, STAGING_PREFIX) + assert len(remaining_staging) == 0, f"Staging not cleaned: {remaining_staging}" + + def test_md5_sidecars_deleted(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: + """Both data files and .md5 sidecars are removed from staging after promote.""" + s3 = minio_s3_client + files = { + f"{ASSEMBLY_DIR_A}_genomic.fna.gz": FAKE_GENOMIC, + f"{ASSEMBLY_DIR_A}_protein.faa.gz": FAKE_PROTEIN, + } + _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, files, with_md5=True) + + # Verify sidecars exist before promote + rel = build_accession_path(ASSEMBLY_DIR_A) + before_keys = list_all_keys(s3, staging_test_bucket, STAGING_PREFIX) + assert any(k.endswith(".md5") for k in before_keys), "Test setup: expected .md5 sidecars" + + promote_from_s3( + staging_key_prefix=STAGING_PREFIX, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, + lakehouse_key_prefix=PATH_PREFIX, + ) + + after_keys = list_all_keys(s3, staging_test_bucket, STAGING_PREFIX) + assert len(after_keys) == 0, f"Staging not fully cleaned (including sidecars): {after_keys}" + + def test_two_assemblies_staging_both_cleaned( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + ) -> None: + """Staging for both assemblies is fully cleaned when both assemblies succeed.""" + s3 = minio_s3_client + _stage_many( + s3, + staging_test_bucket, + ASSEMBLY_DIR_A, + {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": FAKE_GENOMIC}, + with_md5=True, + ) + _stage_many( + s3, + staging_test_bucket, + ASSEMBLY_DIR_B, + {f"{ASSEMBLY_DIR_B}_genomic.fna.gz": FAKE_GENOMIC}, + with_md5=True, + ) + + report = promote_from_s3( + staging_key_prefix=STAGING_PREFIX, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, + lakehouse_key_prefix=PATH_PREFIX, + ) + + assert report["promoted"] == 2 # noqa: PLR2004 + assert report["failed"] == 0 + remaining = list_all_keys(s3, staging_test_bucket, STAGING_PREFIX) + assert len(remaining) == 0, f"Staging not fully cleaned after two-assembly promote: {remaining}" + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestPromoteTwoAssembliesBothLand: + """Both assemblies staged together are both promoted to correct Lakehouse paths.""" + + def test_both_assemblies_at_correct_final_paths( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + ) -> None: + """Each assembly's files appear at distinct, correctly-routed final Lakehouse paths.""" + s3 = minio_s3_client + files_a = {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"genomic-A"} + files_b = {f"{ASSEMBLY_DIR_B}_genomic.fna.gz": b"genomic-B"} + _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, files_a) + _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_B, files_b) + + report = promote_from_s3( + staging_key_prefix=STAGING_PREFIX, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, + lakehouse_key_prefix=PATH_PREFIX, + ) + + assert report["promoted"] == 2 # noqa: PLR2004 + assert report["failed"] == 0 + + rel_a = build_accession_path(ASSEMBLY_DIR_A) + rel_b = build_accession_path(ASSEMBLY_DIR_B) + obj_a = s3.get_object(Bucket=test_bucket, Key=f"{PATH_PREFIX}{rel_a}{ASSEMBLY_DIR_A}_genomic.fna.gz") + obj_b = s3.get_object(Bucket=test_bucket, Key=f"{PATH_PREFIX}{rel_b}{ASSEMBLY_DIR_B}_genomic.fna.gz") + assert obj_a["Body"].read() == b"genomic-A" + assert obj_b["Body"].read() == b"genomic-B" + + def test_final_path_keys_do_not_overlap( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + ) -> None: + """Files for assembly A and assembly B land at distinct paths — no key collision.""" + s3 = minio_s3_client + _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"a"}) + _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_B, {f"{ASSEMBLY_DIR_B}_genomic.fna.gz": b"b"}) + + promote_from_s3( + staging_key_prefix=STAGING_PREFIX, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, + lakehouse_key_prefix=PATH_PREFIX, + ) + + rel_a = build_accession_path(ASSEMBLY_DIR_A) + rel_b = build_accession_path(ASSEMBLY_DIR_B) + keys_a = list_all_keys(s3, test_bucket, f"{PATH_PREFIX}{rel_a}") + keys_b = list_all_keys(s3, test_bucket, f"{PATH_PREFIX}{rel_b}") + assert len(keys_a) == 1 + assert len(keys_b) == 1 + assert keys_a[0] != keys_b[0] + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestPromoteDryRunMultiFile: + """dry_run leaves staging untouched and writes nothing to the Lakehouse.""" + + def test_dry_run_many_files_staging_untouched( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + ) -> None: + """All staged files (data + .md5) survive a dry-run promote unchanged.""" + s3 = minio_s3_client + many_files = { + f"{ASSEMBLY_DIR_A}_genomic.fna.gz": FAKE_GENOMIC, + f"{ASSEMBLY_DIR_A}_protein.faa.gz": FAKE_PROTEIN, + f"{ASSEMBLY_DIR_A}_rna.fna.gz": b"RNA", + } + _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, many_files, with_md5=True) + staging_before = list_all_keys(s3, staging_test_bucket, STAGING_PREFIX) + + report = promote_from_s3( + staging_key_prefix=STAGING_PREFIX, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, + lakehouse_key_prefix=PATH_PREFIX, + dry_run=True, + ) + + assert report["promoted"] == len(many_files) + assert report["dry_run"] is True + + # Staging unchanged + staging_after = list_all_keys(s3, staging_test_bucket, STAGING_PREFIX) + assert staging_after == staging_before, "Dry-run should not alter staging" + + # Nothing at final path + final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + assert len(final_keys) == 0, f"Dry-run created Lakehouse objects: {final_keys}" + + def test_dry_run_two_assemblies_nothing_written( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + ) -> None: + """Dry-run with two staged assemblies creates no Lakehouse objects.""" + s3 = minio_s3_client + _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": FAKE_GENOMIC}) + _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_B, {f"{ASSEMBLY_DIR_B}_genomic.fna.gz": FAKE_GENOMIC}) + + promote_from_s3( + staging_key_prefix=STAGING_PREFIX, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, + lakehouse_key_prefix=PATH_PREFIX, + dry_run=True, + ) + + final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + assert len(final_keys) == 0, f"Dry-run created objects: {final_keys}" + + +@pytest.mark.integration +@pytest.mark.slow_test +class TestPromoteSecondRunOnEmptyStaging: + """After staging is cleaned, a second promote run promotes 0 files without error.""" + + def test_second_run_promoted_zero( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + ) -> None: + """Re-running promote on already-cleaned staging succeeds with promoted=0.""" + s3 = minio_s3_client + _stage_many( + s3, + staging_test_bucket, + ASSEMBLY_DIR_A, + {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": FAKE_GENOMIC}, + with_md5=True, + ) + + report1 = promote_from_s3( + staging_key_prefix=STAGING_PREFIX, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, + lakehouse_key_prefix=PATH_PREFIX, + ) + report2 = promote_from_s3( + staging_key_prefix=STAGING_PREFIX, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, + lakehouse_key_prefix=PATH_PREFIX, + ) + + assert report1["promoted"] == 1 + assert report2["promoted"] == 0 + assert report2["failed"] == 0 + + # Final key still present after second run + rel = build_accession_path(ASSEMBLY_DIR_A) + final_keys = list_all_keys(s3, test_bucket, f"{PATH_PREFIX}{rel}") + assert len(final_keys) == 1 + + def test_lakehouse_unchanged_on_second_run( + self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + ) -> None: + """Lakehouse contents are identical before and after a second (no-op) promote run.""" + s3 = minio_s3_client + _stage_many( + s3, + staging_test_bucket, + ASSEMBLY_DIR_A, + {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": FAKE_GENOMIC, f"{ASSEMBLY_DIR_A}_protein.faa.gz": FAKE_PROTEIN}, + with_md5=True, + ) + + promote_from_s3( + staging_key_prefix=STAGING_PREFIX, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, + lakehouse_key_prefix=PATH_PREFIX, + ) + keys_after_first = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + + promote_from_s3( + staging_key_prefix=STAGING_PREFIX, + staging_bucket=staging_test_bucket, + lakehouse_bucket=test_bucket, + lakehouse_key_prefix=PATH_PREFIX, + ) + keys_after_second = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + + assert keys_after_first == keys_after_second diff --git a/tests/ncbi_ftp/test_promote.py b/tests/ncbi_ftp/test_promote.py index 976d06e0..ab1184df 100644 --- a/tests/ncbi_ftp/test_promote.py +++ b/tests/ncbi_ftp/test_promote.py @@ -1,10 +1,13 @@ """Tests for ncbi_ftp.promote module — S3 promote, archive, manifest trimming.""" +import hashlib from pathlib import Path +from unittest.mock import patch import botocore.client import pytest +import cdm_data_loaders.ncbi_ftp.promote as promote_mod from cdm_data_loaders.ncbi_ftp.promote import ( DEFAULT_LAKEHOUSE_KEY_PREFIX, _archive_assemblies, @@ -14,6 +17,51 @@ from tests.ncbi_ftp.conftest import TEST_BUCKET +# ── Promotion test constants ───────────────────────────────────────────── + +_STAGE_PREFIX = "staging/run1/" + +# Assembly 1 +_ACC1 = "GCF_000001215.4" +_DIR1 = "GCF_000001215.4_Release_6" +_STG1 = f"{_STAGE_PREFIX}raw_data/GCF/000/001/215/{_DIR1}/" +_LKH1 = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{_DIR1}/" + +# Assembly 2 +_ACC2 = "GCF_000005845.2" +_DIR2 = "GCF_000005845.2_ASM584v2" +_STG2 = f"{_STAGE_PREFIX}raw_data/GCF/000/005/845/{_DIR2}/" +_LKH2 = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/005/845/{_DIR2}/" + + +def _stage( + s3: botocore.client.BaseClient, + staging_base: str, + files: dict[str, bytes], + *, + with_md5: bool = True, + with_crc64: bool = False, +) -> list[str]: + """Stage files at *staging_base*, optionally adding .md5 / .crc64nvme sidecars. + + Returns list of all staged keys (data files only, not sidecars). + """ + keys = [] + for fname, content in files.items(): + key = f"{staging_base}{fname}" + s3.put_object(Bucket=TEST_BUCKET, Key=key, Body=content) + keys.append(key) + if with_md5: + s3.put_object( + Bucket=TEST_BUCKET, + Key=f"{key}.md5", + Body=hashlib.md5(content).hexdigest().encode(), # noqa: S324 + ) + if with_crc64: + s3.put_object(Bucket=TEST_BUCKET, Key=f"{key}.crc64nvme", Body=b"fake-crc") + return keys + + def _stage_files(s3_client: botocore.client.BaseClient, prefix: str) -> None: """Upload sample staged files to mock S3.""" for key in [ @@ -552,3 +600,408 @@ def test_archive_assemblies_invalid_accession_skipped( str(manifest), lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated" ) assert archived == 1 + + +# ── Concurrent / multi-file promotion (new behaviour) ──────────────────── + + +@pytest.mark.s3 +def test_promote_multi_file_all_land_at_final_path( + mock_s3_client_no_checksum: botocore.client.BaseClient, +) -> None: + """All files for an assembly are promoted concurrently — none missed.""" + file_names = [ + f"{_ACC1}_genomic.fna.gz", + f"{_ACC1}_protein.faa.gz", + f"{_ACC1}_rna.fna.gz", + f"{_ACC1}_assembly_report.txt", + f"{_ACC1}_assembly_stats.txt", + ] + _stage(mock_s3_client_no_checksum, _STG1, {f: f.encode() for f in file_names}) + + report = promote_from_s3( + staging_key_prefix=_STAGE_PREFIX, + staging_bucket=TEST_BUCKET, + lakehouse_bucket=TEST_BUCKET, + ) + + assert report["promoted"] == len(file_names) + assert report["failed"] == 0 + for fname in file_names: + resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{_LKH1}{fname}") + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + + +@pytest.mark.s3 +def test_promote_multi_file_content_preserved( + mock_s3_client_no_checksum: botocore.client.BaseClient, +) -> None: + """Content at the final key is byte-identical to the staged content.""" + files = { + f"{_ACC1}_genomic.fna.gz": b"\x1f\x8bGENOMIC", + f"{_ACC1}_protein.faa.gz": b"\x1f\x8bPROTEIN", + f"{_ACC1}_rna.fna.gz": b"\x1f\x8bRNA", + } + _stage(mock_s3_client_no_checksum, _STG1, files) + + promote_from_s3( + staging_key_prefix=_STAGE_PREFIX, + staging_bucket=TEST_BUCKET, + lakehouse_bucket=TEST_BUCKET, + ) + + for fname, expected in files.items(): + obj = mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=f"{_LKH1}{fname}") + assert obj["Body"].read() == expected, f"Content mismatch for {fname}" + + +@pytest.mark.s3 +def test_promote_md5_metadata_set_from_sidecar( + mock_s3_client_no_checksum: botocore.client.BaseClient, +) -> None: + """MD5 metadata on the promoted object matches the .md5 sidecar value.""" + content = b"\x1f\x8bGENOMIC" + fname = f"{_ACC1}_genomic.fna.gz" + _stage(mock_s3_client_no_checksum, _STG1, {fname: content}, with_md5=True) + expected_md5 = hashlib.md5(content).hexdigest() # noqa: S324 + + promote_from_s3( + staging_key_prefix=_STAGE_PREFIX, + staging_bucket=TEST_BUCKET, + lakehouse_bucket=TEST_BUCKET, + ) + + resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{_LKH1}{fname}") + assert resp["Metadata"].get("md5") == expected_md5 + + +@pytest.mark.s3 +def test_promote_no_sidecar_no_md5_metadata( + mock_s3_client_no_checksum: botocore.client.BaseClient, +) -> None: + """A file staged without a .md5 sidecar is promoted but carries no md5 metadata.""" + fname = f"{_ACC1}_genomic.fna.gz" + _stage(mock_s3_client_no_checksum, _STG1, {fname: b"data"}, with_md5=False) + + promote_from_s3( + staging_key_prefix=_STAGE_PREFIX, + staging_bucket=TEST_BUCKET, + lakehouse_bucket=TEST_BUCKET, + ) + + resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{_LKH1}{fname}") + assert resp["Metadata"].get("md5") is None + + +@pytest.mark.s3 +def test_promote_staging_data_files_deleted_after_promote( + mock_s3_client_no_checksum: botocore.client.BaseClient, +) -> None: + """Staged data files are deleted from staging after a fully successful assembly promote.""" + files = { + f"{_ACC1}_genomic.fna.gz": b"genomic", + f"{_ACC1}_protein.faa.gz": b"protein", + } + staged_keys = _stage(mock_s3_client_no_checksum, _STG1, files) + + promote_from_s3( + staging_key_prefix=_STAGE_PREFIX, + staging_bucket=TEST_BUCKET, + lakehouse_bucket=TEST_BUCKET, + ) + + for key in staged_keys: + result = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=key) + assert result.get("KeyCount", 0) == 0, f"Staged data file not deleted: {key}" + + +@pytest.mark.s3 +def test_promote_md5_sidecars_deleted_after_promote( + mock_s3_client_no_checksum: botocore.client.BaseClient, +) -> None: + """Staged .md5 sidecar files are deleted from staging after a successful promote.""" + files = { + f"{_ACC1}_genomic.fna.gz": b"genomic", + f"{_ACC1}_protein.faa.gz": b"protein", + } + staged_keys = _stage(mock_s3_client_no_checksum, _STG1, files, with_md5=True) + + promote_from_s3( + staging_key_prefix=_STAGE_PREFIX, + staging_bucket=TEST_BUCKET, + lakehouse_bucket=TEST_BUCKET, + ) + + for key in staged_keys: + for sidecar_key in (f"{key}.md5", f"{key}.crc64nvme"): + result = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=sidecar_key) + assert result.get("KeyCount", 0) == 0, f"Sidecar not deleted: {sidecar_key}" + + +@pytest.mark.s3 +def test_promote_crc64nvme_sidecars_deleted_after_promote( + mock_s3_client_no_checksum: botocore.client.BaseClient, +) -> None: + """Staged .crc64nvme sidecar files are also batch-deleted after a successful promote.""" + fname = f"{_ACC1}_genomic.fna.gz" + _stage(mock_s3_client_no_checksum, _STG1, {fname: b"data"}, with_md5=True, with_crc64=True) + + promote_from_s3( + staging_key_prefix=_STAGE_PREFIX, + staging_bucket=TEST_BUCKET, + lakehouse_bucket=TEST_BUCKET, + ) + + staged_key = f"{_STG1}{fname}" + for sidecar_key in (f"{staged_key}.md5", f"{staged_key}.crc64nvme"): + result = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=sidecar_key) + assert result.get("KeyCount", 0) == 0, f"Sidecar not deleted: {sidecar_key}" + + +@pytest.mark.s3 +def test_promote_partial_failure_staging_not_cleaned( + mock_s3_client_no_checksum: botocore.client.BaseClient, +) -> None: + """When one file in an assembly fails, NO staged files for that assembly are deleted. + + Preserving staging on partial failure lets an operator re-run without + re-staging and without losing the partially-promoted state. + """ + file_ok = f"{_ACC1}_genomic.fna.gz" + file_fail = f"{_ACC1}_protein.faa.gz" + _stage(mock_s3_client_no_checksum, _STG1, {file_ok: b"ok", file_fail: b"fail"}) + staged_ok = f"{_STG1}{file_ok}" + staged_fail = f"{_STG1}{file_fail}" + + # Make download_file raise for exactly the failing key + original_download = mock_s3_client_no_checksum.download_file + + def _download_one_fail(Bucket: str, Key: str, Filename: str, **kw: object) -> None: # noqa: N803 + if Key == staged_fail: + msg = "simulated download failure" + raise RuntimeError(msg) + return original_download(Bucket=Bucket, Key=Key, Filename=Filename, **kw) + + mock_s3_client_no_checksum.download_file = _download_one_fail + + report = promote_from_s3( + staging_key_prefix=_STAGE_PREFIX, + staging_bucket=TEST_BUCKET, + lakehouse_bucket=TEST_BUCKET, + ) + + assert report["failed"] == 1 + # Staging files must still be present (cleanup skipped due to failure) + for key in (staged_ok, staged_fail): + resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=key) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200, ( + f"Expected staged file to survive partial failure: {key}" + ) # noqa: PLR2004 + + +@pytest.mark.s3 +def test_promote_partial_failure_failed_count( + mock_s3_client_no_checksum: botocore.client.BaseClient, +) -> None: + """report[\"failed\"] reflects the number of files that could not be promoted.""" + file_names = [f"{_ACC1}_genomic.fna.gz", f"{_ACC1}_protein.faa.gz", f"{_ACC1}_rna.fna.gz"] + _stage(mock_s3_client_no_checksum, _STG1, {f: b"data" for f in file_names}) + + failing_key = f"{_STG1}{file_names[1]}" + original_download = mock_s3_client_no_checksum.download_file + + def _download_middle_fail(Bucket: str, Key: str, Filename: str, **kw: object) -> None: # noqa: N803 + if Key == failing_key: + msg = "simulated failure" + raise RuntimeError(msg) + return original_download(Bucket=Bucket, Key=Key, Filename=Filename, **kw) + + mock_s3_client_no_checksum.download_file = _download_middle_fail + + report = promote_from_s3( + staging_key_prefix=_STAGE_PREFIX, + staging_bucket=TEST_BUCKET, + lakehouse_bucket=TEST_BUCKET, + ) + + assert report["failed"] == 1 + assert report["promoted"] == 2 # noqa: PLR2004 + + +@pytest.mark.s3 +def test_promote_two_assemblies_independent_cleanup( + mock_s3_client_no_checksum: botocore.client.BaseClient, +) -> None: + """A fully successful assembly cleans up its staging even when another assembly partially fails. + + Assembly 1 fully succeeds → staging cleared. + Assembly 2 has one failing file → staging NOT cleared. + """ + # Assembly 1: two files, both succeed + _stage( + mock_s3_client_no_checksum, + _STG1, + {f"{_ACC1}_genomic.fna.gz": b"g1", f"{_ACC1}_protein.faa.gz": b"p1"}, + ) + # Assembly 2: two files, one will fail + _stage( + mock_s3_client_no_checksum, + _STG2, + {f"{_ACC2}_genomic.fna.gz": b"g2", f"{_ACC2}_protein.faa.gz": b"p2"}, + ) + failing_key = f"{_STG2}{_ACC2}_protein.faa.gz" + original_download = mock_s3_client_no_checksum.download_file + + def _patched(Bucket: str, Key: str, Filename: str, **kw: object) -> None: # noqa: N803 + if Key == failing_key: + msg = "simulated failure" + raise RuntimeError(msg) + return original_download(Bucket=Bucket, Key=Key, Filename=Filename, **kw) + + mock_s3_client_no_checksum.download_file = _patched + + report = promote_from_s3( + staging_key_prefix=_STAGE_PREFIX, + staging_bucket=TEST_BUCKET, + lakehouse_bucket=TEST_BUCKET, + ) + + assert report["failed"] == 1 + + # Assembly 1 staging must be gone + for fname in (f"{_ACC1}_genomic.fna.gz", f"{_ACC1}_protein.faa.gz"): + result = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=f"{_STG1}{fname}") + assert result.get("KeyCount", 0) == 0, f"Assembly 1 staging should be cleaned: {fname}" + + # Assembly 2 staging must remain (partial failure) + for fname in (f"{_ACC2}_genomic.fna.gz", f"{_ACC2}_protein.faa.gz"): + resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{_STG2}{fname}") + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200, ( + f"Assembly 2 staging must survive partial failure: {fname}" + ) # noqa: PLR2004 + + +@pytest.mark.s3 +def test_promote_multi_assembly_all_succeed_all_cleaned( + mock_s3_client_no_checksum: botocore.client.BaseClient, +) -> None: + """Two assemblies both fully succeed → all staged files removed for both.""" + _stage(mock_s3_client_no_checksum, _STG1, {f"{_ACC1}_genomic.fna.gz": b"g1"}) + _stage(mock_s3_client_no_checksum, _STG2, {f"{_ACC2}_genomic.fna.gz": b"g2"}) + + report = promote_from_s3( + staging_key_prefix=_STAGE_PREFIX, + staging_bucket=TEST_BUCKET, + lakehouse_bucket=TEST_BUCKET, + ) + + assert report["promoted"] == 2 # noqa: PLR2004 + assert report["failed"] == 0 + + for stg, fname, lkh in ( + (_STG1, f"{_ACC1}_genomic.fna.gz", _LKH1), + (_STG2, f"{_ACC2}_genomic.fna.gz", _LKH2), + ): + result = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=f"{stg}{fname}") + assert result.get("KeyCount", 0) == 0, f"Staging not cleaned: {fname}" + resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{lkh}{fname}") + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + + +@pytest.mark.s3 +def test_promote_dry_run_multi_file_no_writes_no_cleanup( + mock_s3_client_no_checksum: botocore.client.BaseClient, +) -> None: + """dry_run with multiple files writes nothing to final path and does not delete staging.""" + file_names = [f"{_ACC1}_genomic.fna.gz", f"{_ACC1}_protein.faa.gz", f"{_ACC1}_rna.fna.gz"] + staged_keys = _stage(mock_s3_client_no_checksum, _STG1, {f: f.encode() for f in file_names}) + + report = promote_from_s3( + staging_key_prefix=_STAGE_PREFIX, + staging_bucket=TEST_BUCKET, + lakehouse_bucket=TEST_BUCKET, + dry_run=True, + ) + + assert report["promoted"] == len(file_names) + assert report["dry_run"] is True + + # Final path must be empty + result = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=_LKH1) + assert result.get("KeyCount", 0) == 0 + + # Staging keys must survive + for key in staged_keys: + resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=key) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200, f"Staging deleted during dry-run: {key}" # noqa: PLR2004 + + +@pytest.mark.s3 +def test_promote_skips_non_raw_data_paths( + mock_s3_client_no_checksum: botocore.client.BaseClient, +) -> None: + """Files outside raw_data/ (e.g. download_report.json) are silently skipped.""" + # Stage a real data file alongside non-promotable files + _stage(mock_s3_client_no_checksum, _STG1, {f"{_ACC1}_genomic.fna.gz": b"data"}) + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{_STAGE_PREFIX}download_report.json", Body=b"{}") + mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{_STAGE_PREFIX}logs/run.log", Body=b"logs") + + report = promote_from_s3( + staging_key_prefix=_STAGE_PREFIX, + staging_bucket=TEST_BUCKET, + lakehouse_bucket=TEST_BUCKET, + ) + + assert report["promoted"] == 1 # only the .fna.gz + assert report["failed"] == 0 + + +@pytest.mark.s3 +def test_promote_idempotent_second_run_on_empty_staging( + mock_s3_client_no_checksum: botocore.client.BaseClient, +) -> None: + """Second promote run after staging has been cleaned promotes 0 files without error.""" + _stage(mock_s3_client_no_checksum, _STG1, {f"{_ACC1}_genomic.fna.gz": b"data"}) + + report1 = promote_from_s3( + staging_key_prefix=_STAGE_PREFIX, + staging_bucket=TEST_BUCKET, + lakehouse_bucket=TEST_BUCKET, + ) + report2 = promote_from_s3( + staging_key_prefix=_STAGE_PREFIX, + staging_bucket=TEST_BUCKET, + lakehouse_bucket=TEST_BUCKET, + ) + + assert report1["promoted"] == 1 + assert report2["promoted"] == 0 + assert report2["failed"] == 0 + + # Final key still present after second run + resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{_LKH1}{_ACC1}_genomic.fna.gz") + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + + +@pytest.mark.s3 +def test_promote_multi_file_md5_per_file( + mock_s3_client_no_checksum: botocore.client.BaseClient, +) -> None: + """Each promoted file carries the MD5 matching its own content, not another file's.""" + files = { + f"{_ACC1}_genomic.fna.gz": b"GENOMIC_UNIQUE", + f"{_ACC1}_protein.faa.gz": b"PROTEIN_UNIQUE", + f"{_ACC1}_rna.fna.gz": b"RNA_UNIQUE", + } + _stage(mock_s3_client_no_checksum, _STG1, files, with_md5=True) + + promote_from_s3( + staging_key_prefix=_STAGE_PREFIX, + staging_bucket=TEST_BUCKET, + lakehouse_bucket=TEST_BUCKET, + ) + + for fname, content in files.items(): + expected_md5 = hashlib.md5(content).hexdigest() # noqa: S324 + resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{_LKH1}{fname}") + assert resp["Metadata"].get("md5") == expected_md5, f"Wrong MD5 on {fname}" From fdb702646dc72e0367898a433caef4ee9af2fdb5 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 30 Apr 2026 12:50:14 -0700 Subject: [PATCH 077/128] Potential fix for pull request finding 'Unused local variable' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- tests/integration/test_promote_e2e.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_promote_e2e.py b/tests/integration/test_promote_e2e.py index d412ea27..20c72022 100644 --- a/tests/integration/test_promote_e2e.py +++ b/tests/integration/test_promote_e2e.py @@ -1084,7 +1084,6 @@ def test_md5_sidecars_deleted(self, minio_s3_client: object, test_bucket: str, s _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, files, with_md5=True) # Verify sidecars exist before promote - rel = build_accession_path(ASSEMBLY_DIR_A) before_keys = list_all_keys(s3, staging_test_bucket, STAGING_PREFIX) assert any(k.endswith(".md5") for k in before_keys), "Test setup: expected .md5 sidecars" From 4d690a8af1c3a0b030e2d45e0fad633521bfa646 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 30 Apr 2026 12:50:24 -0700 Subject: [PATCH 078/128] Potential fix for pull request finding 'Unused import' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- tests/ncbi_ftp/test_promote.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/ncbi_ftp/test_promote.py b/tests/ncbi_ftp/test_promote.py index ab1184df..bb04d080 100644 --- a/tests/ncbi_ftp/test_promote.py +++ b/tests/ncbi_ftp/test_promote.py @@ -2,7 +2,6 @@ import hashlib from pathlib import Path -from unittest.mock import patch import botocore.client import pytest From e305d74affa8837c85d87602498abd946ff1a67e Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 30 Apr 2026 12:50:34 -0700 Subject: [PATCH 079/128] Potential fix for pull request finding 'Unused import' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- tests/ncbi_ftp/test_promote.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/ncbi_ftp/test_promote.py b/tests/ncbi_ftp/test_promote.py index bb04d080..ae0ff887 100644 --- a/tests/ncbi_ftp/test_promote.py +++ b/tests/ncbi_ftp/test_promote.py @@ -6,7 +6,6 @@ import botocore.client import pytest -import cdm_data_loaders.ncbi_ftp.promote as promote_mod from cdm_data_loaders.ncbi_ftp.promote import ( DEFAULT_LAKEHOUSE_KEY_PREFIX, _archive_assemblies, From c01dbdca283d748454b0678809ddd11982b1673a Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 30 Apr 2026 14:51:20 -0700 Subject: [PATCH 080/128] capture missing return values Co-authored-by: Copilot --- src/cdm_data_loaders/ncbi_ftp/metadata.py | 2 +- src/cdm_data_loaders/ncbi_ftp/promote.py | 9 +++++++-- src/cdm_data_loaders/pipelines/ncbi_ftp_download.py | 12 ++++++++---- tests/integration/test_full_pipeline.py | 10 +++++----- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/metadata.py b/src/cdm_data_loaders/ncbi_ftp/metadata.py index e616c9b5..b5b6175e 100644 --- a/src/cdm_data_loaders/ncbi_ftp/metadata.py +++ b/src/cdm_data_loaders/ncbi_ftp/metadata.py @@ -246,7 +246,7 @@ def archive_descriptor( # noqa: PLR0913 return False raise - copy_object( + _ = copy_object( f"{bucket}/{source_key}", f"{bucket}/{archive_key}", ) diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index 0c50118d..4fbe6707 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -270,7 +270,10 @@ def _promote_one(staged_key: str) -> tuple[DescriptorResource, str]: logger.debug("Descriptor already exists, skipping: %s", descriptor_key) else: descriptor = create_descriptor(adir, acc, resources) - upload_descriptor(descriptor, adir, lakehouse_bucket, lakehouse_key_prefix, dry_run=False) + descriptor_key = upload_descriptor( + descriptor, adir, lakehouse_bucket, lakehouse_key_prefix, dry_run=False + ) + logger.debug("Uploaded descriptor: %s", descriptor_key) descriptors_written += 1 except Exception: logger.exception("Failed to write descriptor for %s", adir) @@ -402,7 +405,7 @@ def _archive_assemblies( # noqa: PLR0913 # Archive the frictionless descriptor alongside raw data if assembly_dir: try: - archive_descriptor( + archived_desc = archive_descriptor( assembly_dir, lakehouse_bucket, lakehouse_key_prefix, @@ -410,6 +413,8 @@ def _archive_assemblies( # noqa: PLR0913 archive_reason=archive_reason, dry_run=dry_run, ) + if not archived_desc: + logger.debug("No descriptor found to archive for %s", assembly_dir) except Exception: logger.exception("Failed to archive descriptor for %s", assembly_dir) diff --git a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py index 062575f6..9eff6af3 100644 --- a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py +++ b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py @@ -104,9 +104,11 @@ def _upload_assembly_dir( if f.is_file(): relative = f.relative_to(tmp_root) dest_prefix = f"{bucket}/{staging_key_prefix.rstrip('/')}/{relative.parent}" - upload_file(f, dest_prefix, show_progress=False) + if upload_file(f, dest_prefix, show_progress=False): + count += 1 + else: + logger.warning("Failed to upload %s to %s", f, dest_prefix) f.unlink() - count += 1 shutil.rmtree(assembly_dir, ignore_errors=True) return count @@ -360,8 +362,10 @@ def _attempt() -> dict[str, Any]: report_path = tmp / "download_report.json" with report_path.open("w") as f: json.dump(report, f, indent=2) - upload_file(report_path, f"{bucket}/{staging_key_prefix.rstrip('/')}", show_progress=False) - staged_objects += 1 + if upload_file(report_path, f"{bucket}/{staging_key_prefix.rstrip('/')}", show_progress=False): + staged_objects += 1 + else: + logger.warning("Failed to upload download report to s3://%s/%s", bucket, staging_key_prefix) logger.info("Staged %d objects to s3://%s/%s", staged_objects, bucket, staging_key_prefix) logger.info( diff --git a/tests/integration/test_full_pipeline.py b/tests/integration/test_full_pipeline.py index 0c948764..63776ae5 100644 --- a/tests/integration/test_full_pipeline.py +++ b/tests/integration/test_full_pipeline.py @@ -61,7 +61,7 @@ def test_full_pipeline_small_batch( assert len(diff.new) > 0, f"No new assemblies in prefix {STABLE_PREFIX}" manifest_path = tmp_path / "transfer_manifest.txt" - write_transfer_manifest(diff, filtered, manifest_path) + _ = write_transfer_manifest(diff, filtered, manifest_path) # ── Phase 2: Download one assembly from real FTP ──────────────── output_dir = tmp_path / "output" @@ -129,7 +129,7 @@ def test_full_pipeline_incremental( assert len(diff1.new) > 0, f"No new assemblies in prefix {STABLE_PREFIX}" manifest1 = tmp_path / "transfer_manifest_1.txt" - write_transfer_manifest(diff1, filtered, manifest1) + _ = write_transfer_manifest(diff1, filtered, manifest1) output1 = tmp_path / "output1" output1.mkdir() @@ -179,13 +179,13 @@ def test_full_pipeline_incremental( assert downloaded_acc in diff2.updated, f"Expected {downloaded_acc} in updated list" manifest2 = tmp_path / "transfer_manifest_2.txt" - write_transfer_manifest(diff2, filtered, manifest2) + _ = write_transfer_manifest(diff2, filtered, manifest2) updated_manifest = tmp_path / "updated_manifest.txt" - write_updated_manifest(diff2, updated_manifest) + _ = write_updated_manifest(diff2, updated_manifest) removed_manifest = tmp_path / "removed_manifest.txt" - write_removed_manifest(diff2, removed_manifest) + _ = write_removed_manifest(diff2, removed_manifest) # Phase 2 — re-download the updated assembly output2 = tmp_path / "output2" From 2b79b46edceece7de51bd5cde68a823c5c56b0c6 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Fri, 1 May 2026 08:57:29 -0700 Subject: [PATCH 081/128] Add AI Covenant checklist item to PR template --- .github/pull_request_template.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index f4d3c2a5..92f036dc 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -11,6 +11,7 @@ # Dev Checklist: - [ ] My code follows the guidelines at https://sites.google.com/lbl.gov/trussresources/home?authuser=0 +- [ ] My submission follows the [AI Covenant](/AI_COVENANT.md) principles - [ ] I have performed a self-review of my own code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation From eb8bb283dff409fc5a7ae82f6bb27e629d4f2028 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 May 2026 17:46:04 +0000 Subject: [PATCH 082/128] Bump urllib3 from 2.6.3 to 2.7.0 in the uv group across 1 directory Bumps the uv group with 1 update in the / directory: [urllib3](https://github.com/urllib3/urllib3). Updates `urllib3` from 2.6.3 to 2.7.0 - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/2.6.3...2.7.0) --- updated-dependencies: - dependency-name: urllib3 dependency-version: 2.7.0 dependency-type: indirect dependency-group: uv ... Signed-off-by: dependabot[bot] --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 5e485d34..98b01e9c 100644 --- a/uv.lock +++ b/uv.lock @@ -3508,11 +3508,11 @@ wheels = [ [[package]] name = "urllib3" -version = "2.6.3" +version = "2.7.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, ] [[package]] From 9695cf77f619e4f30516e2c580422ebdd7c528ef Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 26 May 2026 13:52:19 -0700 Subject: [PATCH 083/128] Updating python deps --- pyproject.toml | 21 +- uv.lock | 2112 +++++++++++++++++++++++++++++------------------- 2 files changed, 1313 insertions(+), 820 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c1642088..7e7c54b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,17 +9,18 @@ authors = [ ] dependencies = [ - "bioregistry>=0.13.31", + "bioregistry>=0.13.57", "boto3[crt]>=1.42.55", - "click>=8.3.1", + "click>=8.3.2", "defusedxml>=0.7.1", "delta-spark>=4.1.0", - "dlt[deltalake,duckdb,filesystem,parquet]>=1.22.2", - "frictionless[aws]>=5.18.1", + "dlt[deltalake,duckdb,filesystem,iceberg,parquet]>=1.27.0", + "frictionless[aws]>=5.19.0", "frozendict>=2.4.7", + "ipykernel>=7.2.0", "lxml>=6.1.0", - "pydantic>=2.12.5", - "pydantic-settings>=2.12.0", + "pydantic>=2.13.4", + "pydantic-settings>=2.14.1", "tqdm>=4.67.3", ] @@ -31,14 +32,14 @@ uniref = "cdm_data_loaders.pipelines.uniref:cli" [dependency-groups] dev = [ - "berdl-notebook-utils", + "berdl-notebook-utils>=0.0.1", "moto[s3]>=5.1.22", "pytest>=9.0.2", "pytest-asyncio>=1.3.0", "pytest-cov>=7.1.0", "pytest-env>=1.6.0", "pytest-recording>=0.13.4", - "ruff>=0.15.4", + "ruff>=0.15.11", ] models = [ "genson>=1.3.0", @@ -46,7 +47,7 @@ models = [ ] xml = [ "xmlschema>=4.3.1", - "xsdata[cli,lxml]>=26.1", + "xsdata[cli,lxml]>=26.2", ] [project.optional-dependencies] @@ -179,7 +180,7 @@ max-complexity = 15 convention = "google" [build-system] -requires = ["uv_build>=0.9.9,<0.20.0"] +requires = ["uv_build>=0.11.8,<0.20.0"] build-backend = "uv_build" [tool.pytest] diff --git a/uv.lock b/uv.lock index 5e485d34..a7ba1296 100644 --- a/uv.lock +++ b/uv.lock @@ -32,11 +32,11 @@ wheels = [ [[package]] name = "aiohappyeyeballs" -version = "2.6.1" +version = "2.6.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/26/30/f84a107a9c4331c14b2b586036f40965c128aa4fee4dda5d3d51cb14ad54/aiohappyeyeballs-2.6.1.tar.gz", hash = "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558", size = 22760, upload-time = "2025-03-12T01:42:48.764Z" } +sdist = { url = "https://files.pythonhosted.org/packages/33/c6/61a2d7b7572279226bb2e7f61d7a19ca7c90da0329c93fa0d560cbf288d8/aiohappyeyeballs-2.6.2.tar.gz", hash = "sha256:e202810ee718bd01fc6ef49e8ea53d023d5cb6b581076d7925aa499fa55dbe64", size = 22591, upload-time = "2026-05-20T15:12:24.631Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl", hash = "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8", size = 15265, upload-time = "2025-03-12T01:42:47.083Z" }, + { url = "https://files.pythonhosted.org/packages/5f/fc/a7bf5b6e4e617b45f90f2d9d2a68519c249c81dd4fc2658c7a2a61c4f4b7/aiohappyeyeballs-2.6.2-py3-none-any.whl", hash = "sha256:4708045e2d7a6c6bdf8aafa8ed39649eaf926a4543b54560659129e3365953c4", size = 15062, upload-time = "2026-05-20T15:12:23.328Z" }, ] [[package]] @@ -148,7 +148,7 @@ wheels = [ [[package]] name = "anthropic" -version = "0.96.0" +version = "0.104.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -160,9 +160,9 @@ dependencies = [ { name = "sniffio" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b9/7e/672f533dee813028d2c699bfd2a7f52c9118d7353680d9aa44b9e23f717f/anthropic-0.96.0.tar.gz", hash = "sha256:9de947b737f39452f68aa520f1c2239d44119c9b73b0fb6d4e6ca80f00279ee6", size = 658210, upload-time = "2026-04-16T14:28:02.846Z" } +sdist = { url = "https://files.pythonhosted.org/packages/22/c7/7a655b948916f777354648ce979f68b94d5b8dbdb5f61fed1f37fad9378c/anthropic-0.104.1.tar.gz", hash = "sha256:17362b6c45f527afcc9b0fdf62011ffd359726ab2ebcb1978ea0cc41bd8d8d40", size = 850081, upload-time = "2026-05-22T15:36:57.432Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/48/5a/72f33204064b6e87601a71a6baf8d855769f8a0c1eaae8d06a1094872371/anthropic-0.96.0-py3-none-any.whl", hash = "sha256:9a6e335a354602a521cd9e777e92bfd46ba6e115bf9bbfe6135311e8fb2015b2", size = 635930, upload-time = "2026-04-16T14:28:01.436Z" }, + { url = "https://files.pythonhosted.org/packages/b8/12/d9ab42790494d7c428391a46cd28492395566a6a8ccb138d681978594455/anthropic-0.104.1-py3-none-any.whl", hash = "sha256:35c8cb456f5a4405aafe1f10f03f6fcc54fa51fa8ec01d655cc4b437d120e9b7", size = 832996, upload-time = "2026-05-22T15:36:59.519Z" }, ] [[package]] @@ -177,6 +177,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, ] +[[package]] +name = "appnope" +version = "0.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/35/5d/752690df9ef5b76e169e68d6a129fa6d08a7100ca7f754c89495db3c6019/appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee", size = 4170, upload-time = "2024-02-06T09:43:11.258Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/29/5ecc3a15d5a33e31b26c11426c45c501e439cb865d0bff96315d86443b78/appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c", size = 4321, upload-time = "2024-02-06T09:43:09.663Z" }, +] + [[package]] name = "argon2-cffi" version = "25.1.0" @@ -310,9 +319,10 @@ wheels = [ [[package]] name = "berdl-notebook-utils" version = "0.0.1" -source = { git = "https://github.com/BERDataLakehouse/spark_notebook.git?subdirectory=notebook_utils#4128863f3cf0eff4aa941e9dd7d932712587ee18" } +source = { git = "https://github.com/BERDataLakehouse/spark_notebook.git?subdirectory=notebook_utils#dfc45ceea60dab41e5a9fee2eee0b29ffb81bdaa" } dependencies = [ { name = "attrs" }, + { name = "cacheout" }, { name = "cdm-spark-manager-client" }, { name = "cdm-task-service-client" }, { name = "datalake-mcp-server-client" }, @@ -323,7 +333,10 @@ dependencies = [ { name = "itables" }, { name = "langchain" }, { name = "langchain-anthropic" }, + { name = "langchain-classic" }, { name = "langchain-community" }, + { name = "langchain-mcp-tools" }, + { name = "langchain-ollama" }, { name = "langchain-openai" }, { name = "minio" }, { name = "minio-manager-service-client" }, @@ -336,7 +349,7 @@ dependencies = [ [[package]] name = "bioregistry" -version = "0.13.40" +version = "0.13.57" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -348,9 +361,9 @@ dependencies = [ { name = "sssom-pydantic" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/56/48/1b0d2628c3ead36efc02b94517c86573e7a30a05b3e70430f321b1b4b8e0/bioregistry-0.13.40.tar.gz", hash = "sha256:ef8305b9a29d17e00a345e0b4e22294c52c08a15cf25156af9d66ab44015e66d", size = 6000260, upload-time = "2026-04-15T11:00:43.195Z" } +sdist = { url = "https://files.pythonhosted.org/packages/eb/b4/1843ebbe998ec88b19b2128e4e1ea51976d12949066e5b9260ca1dbe5ea5/bioregistry-0.13.57.tar.gz", hash = "sha256:03d504adb861b858fbfcfe3a6e83057fa10a6f5992e46958f971b53a3fe812ea", size = 6048550, upload-time = "2026-05-23T04:33:55.516Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b2/a6/4e45c6863714678188a061cc94585f2ffb1330658945953def9fb698634d/bioregistry-0.13.40-py3-none-any.whl", hash = "sha256:3841e53d6297dfb32ec8a7ee4f6d74fe87ac83cfbd9b9e2553e442599c4e8e73", size = 6087315, upload-time = "2026-04-15T11:00:38.229Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b5/20c3ccde3f3432851f8470d4f95271a018ac73e05e563cfbce63d0527f40/bioregistry-0.13.57-py3-none-any.whl", hash = "sha256:59a0341b666fd960e7508792e5cffb128de3677202c1dbdf17c12051fbe145ef", size = 6139134, upload-time = "2026-05-23T04:33:52.303Z" }, ] [[package]] @@ -391,6 +404,15 @@ crt = [ { name = "awscrt" }, ] +[[package]] +name = "cacheout" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d1/60/ed4c4b27b2131a0b2cc461789be2cf06866644ca462cb34a5d8fca114c15/cacheout-0.16.0.tar.gz", hash = "sha256:ee264897cbaa089ae5f406da11952697d99fa7f3583cfab69fe8a00ff8e1952d", size = 42050, upload-time = "2023-12-22T17:44:33.29Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/14/a89bb55107b8a9b586c8878f47d0b7750c3688c209f05f915e70de74880d/cacheout-0.16.0-py3-none-any.whl", hash = "sha256:1a52d9aa8b1e9720d8453b061348f15795578231f9ec4ad376fec49e717d0ed8", size = 21837, upload-time = "2023-12-22T17:44:31.556Z" }, +] + [[package]] name = "cdm-data-loaders" version = "0.1.8" @@ -404,6 +426,7 @@ dependencies = [ { name = "dlt", extra = ["deltalake", "duckdb", "filesystem", "parquet"] }, { name = "frictionless", extra = ["aws"] }, { name = "frozendict" }, + { name = "ipykernel" }, { name = "lxml" }, { name = "pydantic" }, { name = "pydantic-settings" }, @@ -432,17 +455,18 @@ xml = [ [package.metadata] requires-dist = [ - { name = "bioregistry", specifier = ">=0.13.31" }, + { name = "bioregistry", specifier = ">=0.13.57" }, { name = "boto3", extras = ["crt"], specifier = ">=1.42.55" }, - { name = "click", specifier = ">=8.3.1" }, + { name = "click", specifier = ">=8.3.2" }, { name = "defusedxml", specifier = ">=0.7.1" }, { name = "delta-spark", specifier = ">=4.1.0" }, - { name = "dlt", extras = ["deltalake", "duckdb", "filesystem", "parquet"], specifier = ">=1.22.2" }, - { name = "frictionless", extras = ["aws"], specifier = ">=5.18.1" }, + { name = "dlt", extras = ["deltalake", "duckdb", "filesystem", "iceberg", "parquet"], specifier = ">=1.27.0" }, + { name = "frictionless", extras = ["aws"], specifier = ">=5.19.0" }, { name = "frozendict", specifier = ">=2.4.7" }, + { name = "ipykernel", specifier = ">=7.2.0" }, { name = "lxml", specifier = ">=6.1.0" }, - { name = "pydantic", specifier = ">=2.12.5" }, - { name = "pydantic-settings", specifier = ">=2.12.0" }, + { name = "pydantic", specifier = ">=2.13.4" }, + { name = "pydantic-settings", specifier = ">=2.14.1" }, { name = "tqdm", specifier = ">=4.67.3" }, ] @@ -455,7 +479,7 @@ dev = [ { name = "pytest-cov", specifier = ">=7.1.0" }, { name = "pytest-env", specifier = ">=1.6.0" }, { name = "pytest-recording", specifier = ">=0.13.4" }, - { name = "ruff", specifier = ">=0.15.4" }, + { name = "ruff", specifier = ">=0.15.11" }, ] models = [ { name = "genson", specifier = ">=1.3.0" }, @@ -463,7 +487,7 @@ models = [ ] xml = [ { name = "xmlschema", specifier = ">=4.3.1" }, - { name = "xsdata", extras = ["cli", "lxml"], specifier = ">=26.1" }, + { name = "xsdata", extras = ["cli", "lxml"], specifier = ">=26.2" }, ] [[package]] @@ -773,40 +797,44 @@ wheels = [ [[package]] name = "curies" -version = "0.13.3" +version = "0.13.11" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "pystow" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/06/5f/abed32e461a1b8d201da0744250e569cccd401df3c8a1e133868fa7923e8/curies-0.13.3.tar.gz", hash = "sha256:ee48a8def8fe65b162029505e2554221d3ccdc7c5580716abd9d5693539ed9ee", size = 68827, upload-time = "2026-04-08T11:53:42.081Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/68/06/5ca84afe654733e274e81b2d59b39578d9b6787421eae0e9c27a1ef013c9/curies-0.13.3-py3-none-any.whl", hash = "sha256:eefd58bcc7e0e49ab8f7341a751cce1d62a4669ef93de4683f0a370824bd2289", size = 78026, upload-time = "2026-04-08T11:53:40.834Z" }, -] - -[[package]] -name = "dataclasses-json" -version = "0.6.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "marshmallow" }, - { name = "typing-inspect" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/64/a4/f71d9cf3a5ac257c993b5ca3f93df5f7fb395c725e7f1e6479d2514173c3/dataclasses_json-0.6.7.tar.gz", hash = "sha256:b6b3e528266ea45b9535223bc53ca645f5208833c29229e847b3f26a1cc55fc0", size = 32227, upload-time = "2024-06-09T16:20:19.103Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/95/eb5da50c3d9ba371b37a3fe2c8a496d30c2e0934ceb7b7ea5786f568351c/curies-0.13.11.tar.gz", hash = "sha256:b9cb5fab9d2cade296811343aa2af722298350faca8273e805036c8d0080f0cf", size = 72192, upload-time = "2026-05-07T15:49:24.955Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686, upload-time = "2024-06-09T16:20:16.715Z" }, + { url = "https://files.pythonhosted.org/packages/c5/82/30edbe41d5d00641fecde5470a029a6dd378357bb4ae67d8b8717f1a9293/curies-0.13.11-py3-none-any.whl", hash = "sha256:dae85622a6730a83b2f0854530e7a35bcd2b931a648f181ceeddbe19da25f300", size = 81882, upload-time = "2026-05-07T15:49:26.938Z" }, ] [[package]] name = "datalake-mcp-server-client" version = "0.0.1" -source = { git = "https://github.com/BERDataLakehouse/datalake-mcp-server-client.git?rev=v0.0.9#321ecadba9c1d635215ebfa6f86c475325298457" } +source = { git = "https://github.com/BERDataLakehouse/datalake-mcp-server-client.git?rev=v0.0.10#0b13bbdc0c4e62719a8650d30aed1d1578efe8dc" } dependencies = [ { name = "httpx" }, { name = "pydantic" }, ] +[[package]] +name = "debugpy" +version = "1.8.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/b7/cd8080344452e4874aae67c40d8940e2b4d47b01601a8fd9f44786c757c7/debugpy-1.8.20.tar.gz", hash = "sha256:55bc8701714969f1ab89a6d5f2f3d40c36f91b2cbe2f65d98bf8196f6a6a2c33", size = 1645207, upload-time = "2026-01-29T23:03:28.199Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/e2/fc500524cc6f104a9d049abc85a0a8b3f0d14c0a39b9c140511c61e5b40b/debugpy-1.8.20-cp313-cp313-macosx_15_0_universal2.whl", hash = "sha256:5dff4bb27027821fdfcc9e8f87309a28988231165147c31730128b1c983e282a", size = 2539560, upload-time = "2026-01-29T23:03:48.738Z" }, + { url = "https://files.pythonhosted.org/packages/90/83/fb33dcea789ed6018f8da20c5a9bc9d82adc65c0c990faed43f7c955da46/debugpy-1.8.20-cp313-cp313-manylinux_2_34_x86_64.whl", hash = "sha256:84562982dd7cf5ebebfdea667ca20a064e096099997b175fe204e86817f64eaf", size = 4293272, upload-time = "2026-01-29T23:03:50.169Z" }, + { url = "https://files.pythonhosted.org/packages/a6/25/b1e4a01bfb824d79a6af24b99ef291e24189080c93576dfd9b1a2815cd0f/debugpy-1.8.20-cp313-cp313-win32.whl", hash = "sha256:da11dea6447b2cadbf8ce2bec59ecea87cc18d2c574980f643f2d2dfe4862393", size = 5331208, upload-time = "2026-01-29T23:03:51.547Z" }, + { url = "https://files.pythonhosted.org/packages/13/f7/a0b368ce54ffff9e9028c098bd2d28cfc5b54f9f6c186929083d4c60ba58/debugpy-1.8.20-cp313-cp313-win_amd64.whl", hash = "sha256:eb506e45943cab2efb7c6eafdd65b842f3ae779f020c82221f55aca9de135ed7", size = 5372930, upload-time = "2026-01-29T23:03:53.585Z" }, + { url = "https://files.pythonhosted.org/packages/33/2e/f6cb9a8a13f5058f0a20fe09711a7b726232cd5a78c6a7c05b2ec726cff9/debugpy-1.8.20-cp314-cp314-macosx_15_0_universal2.whl", hash = "sha256:9c74df62fc064cd5e5eaca1353a3ef5a5d50da5eb8058fcef63106f7bebe6173", size = 2538066, upload-time = "2026-01-29T23:03:54.999Z" }, + { url = "https://files.pythonhosted.org/packages/c5/56/6ddca50b53624e1ca3ce1d1e49ff22db46c47ea5fb4c0cc5c9b90a616364/debugpy-1.8.20-cp314-cp314-manylinux_2_34_x86_64.whl", hash = "sha256:077a7447589ee9bc1ff0cdf443566d0ecf540ac8aa7333b775ebcb8ce9f4ecad", size = 4269425, upload-time = "2026-01-29T23:03:56.518Z" }, + { url = "https://files.pythonhosted.org/packages/c5/d9/d64199c14a0d4c476df46c82470a3ce45c8d183a6796cfb5e66533b3663c/debugpy-1.8.20-cp314-cp314-win32.whl", hash = "sha256:352036a99dd35053b37b7803f748efc456076f929c6a895556932eaf2d23b07f", size = 5331407, upload-time = "2026-01-29T23:03:58.481Z" }, + { url = "https://files.pythonhosted.org/packages/e0/d9/1f07395b54413432624d61524dfd98c1a7c7827d2abfdb8829ac92638205/debugpy-1.8.20-cp314-cp314-win_amd64.whl", hash = "sha256:a98eec61135465b062846112e5ecf2eebb855305acc1dfbae43b72903b8ab5be", size = 5372521, upload-time = "2026-01-29T23:03:59.864Z" }, + { url = "https://files.pythonhosted.org/packages/e0/c3/7f67dea8ccf8fdcb9c99033bbe3e90b9e7395415843accb81428c441be2d/debugpy-1.8.20-py2.py3-none-any.whl", hash = "sha256:5be9bed9ae3be00665a06acaa48f8329d2b9632f15fd09f6a9a8c8d9907e54d7", size = 5337658, upload-time = "2026-01-29T23:04:17.404Z" }, +] + [[package]] name = "decorator" version = "5.2.1" @@ -840,20 +868,20 @@ wheels = [ [[package]] name = "deltalake" -version = "1.5.0" +version = "1.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "arro3-core" }, { name = "deprecated" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1c/bf/906ff8f875847bb2d2cf9f612d4de6e775ace366c04ad6356b6666504e6a/deltalake-1.5.0.tar.gz", hash = "sha256:cdea832ebcadd9f6ccedfcf023f244f2830152fd82b2f78b42e701989dd73b2d", size = 5326885, upload-time = "2026-03-12T14:59:22.366Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9c/75/ae5593e1836ea81ab14ab9a58e81e25f351597cb6a66d9e84e9d40a99d21/deltalake-1.5.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:b13c693989f50b3ec6e6a7ebeb3ca4ef7cb3f340b8fe8e1a0e0767319c5f0bf5", size = 37946411, upload-time = "2026-03-12T15:06:43.069Z" }, - { url = "https://files.pythonhosted.org/packages/ff/b6/2c983a79593b5fdda60fc49b4f15be360b102212561bcf7a6bf05e12ed61/deltalake-1.5.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:db388bd519c327953e6ccd688f0cf132c9186362b54d0323d0d5ffeb00cfcde1", size = 34817619, upload-time = "2026-03-12T15:25:22.443Z" }, - { url = "https://files.pythonhosted.org/packages/14/6a/e0d363f25e422a185d3b771da4b7eecb230a77c37260f13ddc0c31dafef1/deltalake-1.5.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2fe5d6fe4eb20781ae593659f77a382079503c06f3525691c8fee2815de2322", size = 38744214, upload-time = "2026-03-12T14:59:19.793Z" }, - { url = "https://files.pythonhosted.org/packages/c8/4c/fc68c0c053f3acc53264e84e1447f70d4a06a7489df78161a0d0fc786c47/deltalake-1.5.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7baa94c7f8234c0840627e8f2f5e3f88a02ff011a2991b8e034c187ffafcb3a0", size = 37338903, upload-time = "2026-03-12T14:47:41.005Z" }, - { url = "https://files.pythonhosted.org/packages/a9/20/82929cf32aab56ad8f8350279b4c42cd14e8d0db97826d5bea1d246b9262/deltalake-1.5.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cfc7b124dc22e885c0af413c9a3f1c4a5fd52ec78bce6fd957a78a90c7943e1b", size = 38742962, upload-time = "2026-03-12T14:58:07.976Z" }, - { url = "https://files.pythonhosted.org/packages/a2/84/6dd4fb8d0fee8e2533a80afd8b9c57dc442138152e57a41c7b8f986b8a64/deltalake-1.5.0-cp310-abi3-win_amd64.whl", hash = "sha256:2ad8f11a64c0477be57d310aa9b470a7c3c3ba2a4e4e86ad92c7ca3554c539f2", size = 41044010, upload-time = "2026-03-12T15:25:13.975Z" }, + { url = "https://files.pythonhosted.org/packages/fe/e1/b26c473480347ba82ffe4abde5452d4aa36313cc267e08d9a8ae9c7083fd/deltalake-1.6.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e980c834c3657d0476a0e98763a7f6c6934717175d3ef86208dd974eceb35b15", size = 40940249, upload-time = "2026-05-18T14:25:43.718Z" }, + { url = "https://files.pythonhosted.org/packages/e4/22/ae4148a1d4a0b3cfaea1eed627610a96b79f3d25443237a26af1acb43a67/deltalake-1.6.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:d15cdc95a816dc1744fb208da54bb8dd57548ac4635c1b955727d57123a2e599", size = 37520755, upload-time = "2026-05-18T14:30:38.908Z" }, + { url = "https://files.pythonhosted.org/packages/0f/5d/a9fa27de555540fc11e83bd9e3de5c356f3e9e1019694f9ce32340b9e09f/deltalake-1.6.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd22cf2301c8a3d06819813726ba3ecf68ac16ba12a7c4ac4b1c9b7177461da7", size = 41804855, upload-time = "2026-05-18T13:56:40.982Z" }, + { url = "https://files.pythonhosted.org/packages/f1/bc/e09d13df887783018580d2383765cbd3aaf10c45ab428dd38e9c91053b75/deltalake-1.6.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6780a3702f501c902fa7deecd7501a4f0e52d69321835bf6f87d72b13679bfd4", size = 40545361, upload-time = "2026-05-18T17:52:57.876Z" }, + { url = "https://files.pythonhosted.org/packages/4f/08/30720d394ecc394465c74ca2342cc51f9fe6f1867985b3df5cec3b6e8c63/deltalake-1.6.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a2da86d4964cfd1db988158248cdf46acaea2d5ac7cc0eccee04bff565265153", size = 40538718, upload-time = "2026-05-18T13:46:27.813Z" }, + { url = "https://files.pythonhosted.org/packages/68/d0/9e0c87894641ce9b921e4d3f576ccedce68907cc55ba5004fc4291260289/deltalake-1.6.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5fc975a93a043619175b86fbdf0951bf15ec4ac6e499d981dfc8c9024748d831", size = 41806788, upload-time = "2026-05-18T13:57:09.396Z" }, + { url = "https://files.pythonhosted.org/packages/0a/5b/a1e6d8d35e5b2f8460e45e9171af50c461283c7390c3977f4b7f623193b1/deltalake-1.6.0-cp310-abi3-win_amd64.whl", hash = "sha256:cdc15e2ad80376363ad4d13f475b00c23e4192b5e65e8e421b8aa7cf9df533fa", size = 44143517, upload-time = "2026-05-18T14:23:12.694Z" }, ] [[package]] @@ -879,7 +907,7 @@ wheels = [ [[package]] name = "dlt" -version = "1.25.0" +version = "1.27.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -908,9 +936,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "tzdata" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ae/6c/a9142c33f631f94b9ae233b76c9e59bff15a367597955369e63e6098584b/dlt-1.25.0.tar.gz", hash = "sha256:86ecc24f506c2e08d150d174390a1302cb01ba576c522e2ef3d9708d5789105c", size = 982603, upload-time = "2026-04-15T07:28:40.47Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bb/80/4cc5a1af42e2b9aa3b4a7a8285f1f28fd889a2270bf7c76f812295bbec5b/dlt-1.27.0.tar.gz", hash = "sha256:cec18a34386150ac1620b2f1b5aa160f08e171ec0609af5da573792277f5218c", size = 1069777, upload-time = "2026-05-18T21:07:19.941Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6f/9b/8c1bc26014729e1f49c17c2988458985b93914f54de5bd52a0d547655a96/dlt-1.25.0-py3-none-any.whl", hash = "sha256:a32efddb871eacded32f10ae6b539d85956fda04d1c6f824d8e29a63c8c64095", size = 1238592, upload-time = "2026-04-15T07:28:42.769Z" }, + { url = "https://files.pythonhosted.org/packages/76/23/26e6dcc639bd74025e6b54bb5af140a0a7582047e6b848d072cb947b271a/dlt-1.27.0-py3-none-any.whl", hash = "sha256:69ac35c56f9b319ab3ae5378df2bc199bf746073df287fdd17994f44efd27ad0", size = 1347146, upload-time = "2026-05-18T21:07:23.631Z" }, ] [package.optional-dependencies] @@ -949,24 +977,24 @@ wheels = [ [[package]] name = "duckdb" -version = "1.5.2" +version = "1.5.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0c/66/744b4931b799a42f8cb9bc7a6f169e7b8e51195b62b246db407fd90bf15f/duckdb-1.5.2.tar.gz", hash = "sha256:638da0d5102b6cb6f7d47f83d0600708ac1d3cb46c5e9aaabc845f9ba4d69246", size = 18017166, upload-time = "2026-04-13T11:30:09.065Z" } +sdist = { url = "https://files.pythonhosted.org/packages/69/00/d579dcb2a536b6ea3a2563cdad6844f77d81a9b2d4b22a858097f2468acf/duckdb-1.5.3.tar.gz", hash = "sha256:df39428eb130faa35ae96fd35245bdeae6ecf43936250b116b5fead568eb9f16", size = 18026640, upload-time = "2026-05-20T11:55:31.901Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/98/f2/e3d742808f138d374be4bb516fade3d1f33749b813650810ab7885cdc363/duckdb-1.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:4420b3f47027a7849d0e1815532007f377fa95ee5810b47ea717d35525c12f79", size = 30064879, upload-time = "2026-04-13T11:29:30.763Z" }, - { url = "https://files.pythonhosted.org/packages/72/0d/f3dc1cf97e1267ca15e4307d456f96ce583961f0703fd75e62b2ad8d64fa/duckdb-1.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:bb42e6ed543902e14eae647850da24103a89f0bc2587dec5601b1c1f213bd2ed", size = 15969327, upload-time = "2026-04-13T11:29:33.481Z" }, - { url = "https://files.pythonhosted.org/packages/b1/e0/d5418def53ae4e05a63075705ff44ed5af5a1a5932627eb2b600c5df1c93/duckdb-1.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:98c0535cd6d901f61a5ea3c2e26a1fd28482953d794deb183daf568e3aa5dda6", size = 14225107, upload-time = "2026-04-13T11:29:35.882Z" }, - { url = "https://files.pythonhosted.org/packages/16/a7/15aaa59dbecc35e9711980fcdbf525b32a52470b32d18ef678193a146213/duckdb-1.5.2-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:486c862bf7f163c0110b6d85b3e5c031d224a671cca468f12ebb1d3a348f6b39", size = 19313433, upload-time = "2026-04-13T11:29:38.367Z" }, - { url = "https://files.pythonhosted.org/packages/bd/21/d903cc63a5140c822b7b62b373a87dc557e60c29b321dfb435061c5e67cf/duckdb-1.5.2-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:70631c847ca918ee710ec874241b00cf9d2e5be90762cbb2a0389f17823c08f7", size = 21429837, upload-time = "2026-04-13T11:29:41.135Z" }, - { url = "https://files.pythonhosted.org/packages/e3/0a/b770d1f60c70597302130d6247f418549b7094251a02348fbaf1c7e147ae/duckdb-1.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:52a21823f3fbb52f0f0e5425e20b07391ad882464b955879499b5ff0b45a376b", size = 13107699, upload-time = "2026-04-13T11:29:43.905Z" }, - { url = "https://files.pythonhosted.org/packages/d9/cf/e200fe431d700962d1a908d2ce89f53ccee1cc8db260174ae663ba09686b/duckdb-1.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:411ad438bd4140f189a10e7f515781335962c5d18bd07837dc6d202e3985253d", size = 13927646, upload-time = "2026-04-13T11:29:46.598Z" }, - { url = "https://files.pythonhosted.org/packages/83/a1/f6286c67726cc1ea60a6e3c0d9fbc66527dde24ae089a51bbe298b13ca78/duckdb-1.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:6b0fe75c148000f060aa1a27b293cacc0ea08cc1cad724fbf2143d56070a3785", size = 30078598, upload-time = "2026-04-13T11:29:49.828Z" }, - { url = "https://files.pythonhosted.org/packages/de/6a/59febb02f21a4a5c6b0b0099ef7c965fdd5e61e4904cf813809bb792e35f/duckdb-1.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:35579b8e3a064b5eaf15b0eafc558056a13f79a0a62e34cc4baf57119daecfec", size = 15975120, upload-time = "2026-04-13T11:29:52.631Z" }, - { url = "https://files.pythonhosted.org/packages/09/70/ce750854d37bb5a45cccbb2c3cb04df4af56aea8fc30a2499bb643b4a9c0/duckdb-1.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ea58ff5b0880593a280cf5511734b17711b32ee1f58b47d726e8600848358160", size = 14227762, upload-time = "2026-04-13T11:29:55.564Z" }, - { url = "https://files.pythonhosted.org/packages/28/dc/ad45ac3c0b6c4687dc649e8f6cf01af1c8b0443932a39b2abb4ebcb3babd/duckdb-1.5.2-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef461bca07313412dc09961c4a4757a851f56b95ac01c58fac6007632b7b94f2", size = 19315668, upload-time = "2026-04-13T11:29:58.427Z" }, - { url = "https://files.pythonhosted.org/packages/cc/b1/1464f468d2e5813f5808de95df9d3113a645a5bfa2ffcaecbc542ddae272/duckdb-1.5.2-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be37680ddb380015cb37318e378c53511c45c4f0d8fac5599d22b7d092b9217a", size = 21434056, upload-time = "2026-04-13T11:30:01.238Z" }, - { url = "https://files.pythonhosted.org/packages/ce/32/6673607e024722473fa7aafdd29c0e3dd231dd528f6cd8b5797fbeeb229d/duckdb-1.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:0b291786014df1133f8f18b9df4d004484613146e858d71a21791e0fcca16cf4", size = 13633667, upload-time = "2026-04-13T11:30:04.05Z" }, - { url = "https://files.pythonhosted.org/packages/7a/e3/9d34173ec068631faea3ea6e73050700729363e7e33306a9a3218e5cdc61/duckdb-1.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:c9f3e0b71b8a50fccfb42794899285d9d318ce2503782b9dd54868e5ecd0ad31", size = 14402513, upload-time = "2026-04-13T11:30:06.609Z" }, + { url = "https://files.pythonhosted.org/packages/cc/9c/a528eb09d8be51954c485864bd06753e616939a080cbc3dd4417e8c94a57/duckdb-1.5.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e75a6122c12579a99848517f6f00a4e342aebda3590c30fe9b5cc5f39d5e6afc", size = 32626254, upload-time = "2026-05-20T11:54:53.65Z" }, + { url = "https://files.pythonhosted.org/packages/ec/3c/1534c0a6db347c05eb7d0f6ecfb7aefbe74cbff398e4892a8fd1903a20e8/duckdb-1.5.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fd3963c1cb9d9567777f4a898a9dbe388a2fe9724681801b1e7d6d93eecf1b76", size = 17300917, upload-time = "2026-05-20T11:54:56.628Z" }, + { url = "https://files.pythonhosted.org/packages/23/fa/beafb91e6e152d2161c4a9cbc472334c87607eb61ad7104b5a7fa8d8d7b1/duckdb-1.5.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3d5db8c0b55e072cf437948ebb5d7e23d7b9d03d905fa5f9145583e65aa447f7", size = 15449411, upload-time = "2026-05-20T11:54:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/50/0a/49b6fe04e2fcd63729eb607dadd44818dde77342a4f5ce086c6c92f1dd4d/duckdb-1.5.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ce80aed7a538422129a57eaca9141e3afb51f8bf562b1908b1576c9725b5b22", size = 19333120, upload-time = "2026-05-20T11:55:01.727Z" }, + { url = "https://files.pythonhosted.org/packages/63/4c/0907c3f76adb9dd90e67610b31e0304a35814e65c4c41a354a262c09b885/duckdb-1.5.3-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:787df63824f07bf18022dbc3b8ca4b2bfab0ebe616464f55c6e8cd0f59ea762e", size = 21453266, upload-time = "2026-05-20T11:55:04.5Z" }, + { url = "https://files.pythonhosted.org/packages/6d/9c/d2f23a7803ddbbd9413f7572ecf66a15120ed5ced7ce5c73e698c1406b76/duckdb-1.5.3-cp313-cp313-win_amd64.whl", hash = "sha256:bb5bb5dcdd09d62ee60f0ddbbef918e71cce304ffe28428b1131949d39ffaabf", size = 13118640, upload-time = "2026-05-20T11:55:07.389Z" }, + { url = "https://files.pythonhosted.org/packages/27/d5/7ba2316415bcdab6edd765bbbe35c2ca8a3800f2fe695cd70e3cdb997f09/duckdb-1.5.3-cp313-cp313-win_arm64.whl", hash = "sha256:2fa17ecdd5d3db122836cb71bb93601c2106a3be883c17dffddc02fbf3fa7888", size = 13926409, upload-time = "2026-05-20T11:55:10.166Z" }, + { url = "https://files.pythonhosted.org/packages/a5/c2/d4b6f8a5e4d3bc25773be6da76a99d9661ebbf3552c007c460d2dd59dbf8/duckdb-1.5.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:4bfa9a4dadf71e83e2c4eaca2f9421c82a54defecc1b0b4c0be95e2389dec4fe", size = 32636685, upload-time = "2026-05-20T11:55:13.158Z" }, + { url = "https://files.pythonhosted.org/packages/42/58/e835c8298979d29db7a62cb5acc29e9b57aeaca7cdde2fcd3ac980f5cb18/duckdb-1.5.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:aea7baf67ad7e1829ac76f67d7dcbd7fb1f57c3eb179d55ac30952df4709ae30", size = 17308134, upload-time = "2026-05-20T11:55:16.194Z" }, + { url = "https://files.pythonhosted.org/packages/c9/46/617b51363f5613418c8b224b3cce16b58e6dde80904566bec232579c1d4e/duckdb-1.5.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0b0b4f088a65d77e1217ce5d7eff889e63fedc44281200d899ff47c84d8ff836", size = 15449891, upload-time = "2026-05-20T11:55:18.687Z" }, + { url = "https://files.pythonhosted.org/packages/b3/72/354146656e8d9ba3853d3a5ee80a481b8c5f70edfc3d5ae80a8c4479c967/duckdb-1.5.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe8d0c1f6a120aa03fa6e0d03897c71a1842e6cf7afd31d181348391f7108fe1", size = 19338499, upload-time = "2026-05-20T11:55:21.34Z" }, + { url = "https://files.pythonhosted.org/packages/56/8f/65fc623b51448f2bfba1a9ec6ab3debb4664c0876c0113a5e782600b53ac/duckdb-1.5.3-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0405eae18ec6e8210a471c97dbfe87a7e4d605274b7fe572a1f276e92158f13", size = 21455828, upload-time = "2026-05-20T11:55:23.847Z" }, + { url = "https://files.pythonhosted.org/packages/2b/db/d0274cbe9f5fe219f77c0bdf900ac77103569e83c102a4225ce04cbc607d/duckdb-1.5.3-cp314-cp314-win_amd64.whl", hash = "sha256:33ae08b3e818d7613d8936744b67718c2062c2f530376895bfd89efb51b81538", size = 13640011, upload-time = "2026-05-20T11:55:26.276Z" }, + { url = "https://files.pythonhosted.org/packages/07/5d/8f1899b8bef291caf953992fcd6c24df9f29387a35645e58c2504a5ca473/duckdb-1.5.3-cp314-cp314-win_arm64.whl", hash = "sha256:746433e49bbc667b4df283153415fbe37e9083e0eff6c3cd6e54de7536869cd4", size = 14411554, upload-time = "2026-05-20T11:55:29.037Z" }, ] [[package]] @@ -1119,11 +1147,11 @@ wheels = [ [[package]] name = "fsspec" -version = "2026.3.0" +version = "2026.4.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e1/cf/b50ddf667c15276a9ab15a70ef5f257564de271957933ffea49d2cdbcdfb/fsspec-2026.3.0.tar.gz", hash = "sha256:1ee6a0e28677557f8c2f994e3eea77db6392b4de9cd1f5d7a9e87a0ae9d01b41", size = 313547, upload-time = "2026-03-27T19:11:14.892Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d5/8d/1c51c094345df128ca4a990d633fe1a0ff28726c9e6b3c41ba65087bba1d/fsspec-2026.4.0.tar.gz", hash = "sha256:301d8ac70ae90ef3ad05dcf94d6c3754a097f9b5fe4667d2787aa359ec7df7e4", size = 312760, upload-time = "2026-04-29T20:42:38.635Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d5/1f/5f4a3cd9e4440e9d9bc78ad0a91a1c8d46b4d429d5239ebe6793c9fe5c41/fsspec-2026.3.0-py3-none-any.whl", hash = "sha256:d2ceafaad1b3457968ed14efa28798162f1638dbb5d2a6868a2db002a5ee39a4", size = 202595, upload-time = "2026-03-27T19:11:13.595Z" }, + { url = "https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl", hash = "sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2", size = 203402, upload-time = "2026-04-29T20:42:36.842Z" }, ] [[package]] @@ -1149,14 +1177,14 @@ wheels = [ [[package]] name = "gitpython" -version = "3.1.46" +version = "3.1.50" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "gitdb" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/df/b5/59d16470a1f0dfe8c793f9ef56fd3826093fc52b3bd96d6b9d6c26c7e27b/gitpython-3.1.46.tar.gz", hash = "sha256:400124c7d0ef4ea03f7310ac2fbf7151e09ff97f2a3288d64a440c584a29c37f", size = 215371, upload-time = "2026-01-01T15:37:32.073Z" } +sdist = { url = "https://files.pythonhosted.org/packages/33/f6/354ae6491228b5eb40e10d89c4d13c651fe1cf7556e35ebdded50cff57ce/gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc", size = 219798, upload-time = "2026-05-06T04:01:26.571Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/09/e21df6aef1e1ffc0c816f0522ddc3f6dcded766c3261813131c78a704470/gitpython-3.1.46-py3-none-any.whl", hash = "sha256:79812ed143d9d25b6d176a10bb511de0f9c67b1fa641d82097b0ab90398a2058", size = 208620, upload-time = "2026-01-01T15:37:30.574Z" }, + { url = "https://files.pythonhosted.org/packages/20/7a/1c6e3562dfd8950adbb11ffbc65d21e7c89d01a6e4f137fa981056de25c5/gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9", size = 212507, upload-time = "2026-05-06T04:01:23.799Z" }, ] [[package]] @@ -1170,45 +1198,61 @@ wheels = [ [[package]] name = "googleapis-common-protos" -version = "1.74.0" +version = "1.75.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/20/18/a746c8344152d368a5aac738d4c857012f2c5d1fd2eac7e17b647a7861bd/googleapis_common_protos-1.74.0.tar.gz", hash = "sha256:57971e4eeeba6aad1163c1f0fc88543f965bb49129b8bb55b2b7b26ecab084f1", size = 151254, upload-time = "2026-04-02T21:23:26.679Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/c8/f439cffde755cffa462bfbb156278fa6f9d09119719af9814b858fd4f81f/googleapis_common_protos-1.75.0.tar.gz", hash = "sha256:53a062ff3c32552fbd62c11fe23768b78e4ddf0494d5e5fd97d3f4689c75fbbd", size = 151035, upload-time = "2026-05-07T08:04:49.423Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/b0/be5d3329badb9230b765de6eea66b73abd5944bdeb5afb3562ddcd80ae84/googleapis_common_protos-1.74.0-py3-none-any.whl", hash = "sha256:702216f78610bb510e3f12ac3cafd281b7ac45cc5d86e90ad87e4d301a3426b5", size = 300743, upload-time = "2026-04-02T21:22:49.108Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c8/e2645aa8ed02fd4c7a2f59d68783b65b1f3cbdfe39a6308e156509d1fee8/googleapis_common_protos-1.75.0-py3-none-any.whl", hash = "sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed", size = 300631, upload-time = "2026-05-07T08:03:30.345Z" }, ] [[package]] name = "greenlet" -version = "3.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/86/94/a5935717b307d7c71fe877b52b884c6af707d2d2090db118a03fbd799369/greenlet-3.4.0.tar.gz", hash = "sha256:f50a96b64dafd6169e595a5c56c9146ef80333e67d4476a65a9c55f400fc22ff", size = 195913, upload-time = "2026-04-08T17:08:00.863Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7a/75/7e9cd1126a1e1f0cd67b0eda02e5221b28488d352684704a78ed505bd719/greenlet-3.4.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:43748988b097f9c6f09364f260741aa73c80747f63389824435c7a50bfdfd5c1", size = 285856, upload-time = "2026-04-08T15:52:45.82Z" }, - { url = "https://files.pythonhosted.org/packages/9d/c4/3e2df392e5cb199527c4d9dbcaa75c14edcc394b45040f0189f649631e3c/greenlet-3.4.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5566e4e2cd7a880e8c27618e3eab20f3494452d12fd5129edef7b2f7aa9a36d1", size = 610208, upload-time = "2026-04-08T16:24:39.674Z" }, - { url = "https://files.pythonhosted.org/packages/da/af/750cdfda1d1bd30a6c28080245be8d0346e669a98fdbae7f4102aa95fff3/greenlet-3.4.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1054c5a3c78e2ab599d452f23f7adafef55062a783a8e241d24f3b633ba6ff82", size = 621269, upload-time = "2026-04-08T16:30:59.767Z" }, - { url = "https://files.pythonhosted.org/packages/54/78/0cbc693622cd54ebe25207efbb3a0eb07c2639cb8594f6e3aaaa0bb077a8/greenlet-3.4.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f82cb6cddc27dd81c96b1506f4aa7def15070c3b2a67d4e46fd19016aacce6cf", size = 617549, upload-time = "2026-04-08T15:56:34.893Z" }, - { url = "https://files.pythonhosted.org/packages/ba/c0/8966767de01343c1ff47e8b855dc78e7d1a8ed2b7b9c83576a57e289f81d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:227a46251ecba4ff46ae742bc5ce95c91d5aceb4b02f885487aff269c127a729", size = 1575310, upload-time = "2026-04-08T16:26:21.671Z" }, - { url = "https://files.pythonhosted.org/packages/b8/38/bcdc71ba05e9a5fda87f63ffc2abcd1f15693b659346df994a48c968003d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5b99e87be7eba788dd5b75ba1cde5639edffdec5f91fe0d734a249535ec3408c", size = 1640435, upload-time = "2026-04-08T15:57:32.572Z" }, - { url = "https://files.pythonhosted.org/packages/a1/c2/19b664b7173b9e4ef5f77e8cef9f14c20ec7fce7920dc1ccd7afd955d093/greenlet-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:849f8bc17acd6295fcb5de8e46d55cc0e52381c56eaf50a2afd258e97bc65940", size = 238760, upload-time = "2026-04-08T17:04:03.878Z" }, - { url = "https://files.pythonhosted.org/packages/9b/96/795619651d39c7fbd809a522f881aa6f0ead504cc8201c3a5b789dfaef99/greenlet-3.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:9390ad88b652b1903814eaabd629ca184db15e0eeb6fe8a390bbf8b9106ae15a", size = 235498, upload-time = "2026-04-08T17:05:00.584Z" }, - { url = "https://files.pythonhosted.org/packages/78/02/bde66806e8f169cf90b14d02c500c44cdbe02c8e224c9c67bafd1b8cadd1/greenlet-3.4.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:10a07aca6babdd18c16a3f4f8880acfffc2b88dfe431ad6aa5f5740759d7d75e", size = 286291, upload-time = "2026-04-08T17:09:34.307Z" }, - { url = "https://files.pythonhosted.org/packages/05/1f/39da1c336a87d47c58352fb8a78541ce63d63ae57c5b9dae1fe02801bbc2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:076e21040b3a917d3ce4ad68fb5c3c6b32f1405616c4a57aa83120979649bd3d", size = 656749, upload-time = "2026-04-08T16:24:41.721Z" }, - { url = "https://files.pythonhosted.org/packages/d3/6c/90ee29a4ee27af7aa2e2ec408799eeb69ee3fcc5abcecac6ddd07a5cd0f2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e82689eea4a237e530bb5cb41b180ef81fa2160e1f89422a67be7d90da67f615", size = 669084, upload-time = "2026-04-08T16:31:01.372Z" }, - { url = "https://files.pythonhosted.org/packages/07/49/d4cad6e5381a50947bb973d2f6cf6592621451b09368b8c20d9b8af49c5b/greenlet-3.4.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df3b0b2289ec686d3c821a5fee44259c05cfe824dd5e6e12c8e5f5df23085cf", size = 665621, upload-time = "2026-04-08T15:56:35.995Z" }, - { url = "https://files.pythonhosted.org/packages/37/31/d1edd54f424761b5d47718822f506b435b6aab2f3f93b465441143ea5119/greenlet-3.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8bff29d586ea415688f4cec96a591fcc3bf762d046a796cdadc1fdb6e7f2d5bf", size = 1622259, upload-time = "2026-04-08T16:26:23.201Z" }, - { url = "https://files.pythonhosted.org/packages/b0/c6/6d3f9cdcb21c4e12a79cb332579f1c6aa1af78eb68059c5a957c7812d95e/greenlet-3.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8a569c2fb840c53c13a2b8967c63621fafbd1a0e015b9c82f408c33d626a2fda", size = 1686916, upload-time = "2026-04-08T15:57:34.282Z" }, - { url = "https://files.pythonhosted.org/packages/63/45/c1ca4a1ad975de4727e52d3ffe641ae23e1d7a8ffaa8ff7a0477e1827b92/greenlet-3.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:207ba5b97ea8b0b60eb43ffcacf26969dd83726095161d676aac03ff913ee50d", size = 239821, upload-time = "2026-04-08T17:03:48.423Z" }, - { url = "https://files.pythonhosted.org/packages/71/c4/6f621023364d7e85a4769c014c8982f98053246d142420e0328980933ceb/greenlet-3.4.0-cp314-cp314-win_arm64.whl", hash = "sha256:f8296d4e2b92af34ebde81085a01690f26a51eb9ac09a0fcadb331eb36dbc802", size = 236932, upload-time = "2026-04-08T17:04:33.551Z" }, - { url = "https://files.pythonhosted.org/packages/d4/8f/18d72b629783f5e8d045a76f5325c1e938e659a9e4da79c7dcd10169a48d/greenlet-3.4.0-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:d70012e51df2dbbccfaf63a40aaf9b40c8bed37c3e3a38751c926301ce538ece", size = 294681, upload-time = "2026-04-08T15:52:35.778Z" }, - { url = "https://files.pythonhosted.org/packages/9e/ad/5fa86ec46769c4153820d58a04062285b3b9e10ba3d461ee257b68dcbf53/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a58bec0751f43068cd40cff31bb3ca02ad6000b3a51ca81367af4eb5abc480c8", size = 658899, upload-time = "2026-04-08T16:24:43.32Z" }, - { url = "https://files.pythonhosted.org/packages/43/f0/4e8174ca0e87ae748c409f055a1ba161038c43cc0a5a6f1433a26ac2e5bf/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:05fa0803561028f4b2e3b490ee41216a842eaee11aed004cc343a996d9523aa2", size = 665284, upload-time = "2026-04-08T16:31:02.833Z" }, - { url = "https://files.pythonhosted.org/packages/19/da/991cf7cd33662e2df92a1274b7eb4d61769294d38a1bba8a45f31364845e/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e60d38719cb80b3ab5e85f9f1aed4960acfde09868af6762ccb27b260d68f4ed", size = 661861, upload-time = "2026-04-08T15:56:37.269Z" }, - { url = "https://files.pythonhosted.org/packages/36/c5/6c2c708e14db3d9caea4b459d8464f58c32047451142fe2cfd90e7458f41/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7f50c804733b43eded05ae694691c9aa68bca7d0a867d67d4a3f514742a2d53f", size = 1622182, upload-time = "2026-04-08T16:26:24.777Z" }, - { url = "https://files.pythonhosted.org/packages/7a/4c/50c5fed19378e11a29fabab1f6be39ea95358f4a0a07e115a51ca93385d8/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2d4f0635dc4aa638cda4b2f5a07ae9a2cff9280327b581a3fcb6f317b4fbc38a", size = 1685050, upload-time = "2026-04-08T15:57:36.453Z" }, - { url = "https://files.pythonhosted.org/packages/db/72/85ae954d734703ab48e622c59d4ce35d77ce840c265814af9c078cacc7aa/greenlet-3.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:1a4a48f24681300c640f143ba7c404270e1ebbbcf34331d7104a4ff40f8ea705", size = 245554, upload-time = "2026-04-08T17:03:50.044Z" }, +version = "3.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/6e/802acd792aebb2256fbbee8cacf2727faaeb6f240ac11008f09eae4414bc/greenlet-3.5.1.tar.gz", hash = "sha256:5a56aeb7d5d9cc4b3a735efb5095bd4b4f6f0e4f93e5ca876d0e2315137b7829", size = 197356, upload-time = "2026-05-20T15:05:03.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/69/7f7e5372d998b81001899b1c0823c957aa413ba0f2662e65821611cc31e4/greenlet-3.5.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:51518ff74664078fc51bffcc6fc529b0df5ae58da192691cee765d45ce944a2b", size = 285060, upload-time = "2026-05-20T13:08:51.899Z" }, + { url = "https://files.pythonhosted.org/packages/b1/bf/387f9b6b865fd2ae0d0be09e0004827295a01b71be76ed350dd1e28a91a4/greenlet-3.5.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ffdb3c0bb002c99cd8f298957e046c3dbf6006b5b7cdf11a4e19194624a0a0a", size = 604370, upload-time = "2026-05-20T14:00:07.492Z" }, + { url = "https://files.pythonhosted.org/packages/32/f5/169ce3d4e4c67291bd18f8cbe0299c9f3e45102c7f1fb3c14780c93e4532/greenlet-3.5.1-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7715a5a2c3378ba602c3a440558261e13a820bb53a82693aacd7b7f6d964e283", size = 616987, upload-time = "2026-05-20T14:05:44.237Z" }, + { url = "https://files.pythonhosted.org/packages/ee/e5/7f2e41d5273be07e77560d61ea4e56485b4d6c316d2a84518c62d1364061/greenlet-3.5.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc71ff466927a201b08305acac451ebe1aedfcea002f62f1f2f2ac2ac1e6a135", size = 613911, upload-time = "2026-05-20T13:14:27.539Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a4/fbdc67579b73615a1f91615e814303cc71e06128f7baaba87be79b8fb90c/greenlet-3.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cd443683db272ebaaca03af98c0b063ab30db70ea8a31a1559f35e3f7b744ccd", size = 1570689, upload-time = "2026-05-20T14:02:27.225Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b4/77abbe35078be39718a46cd49caf16bceb35662f97a34101dca28aa98e47/greenlet-3.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:089fff7a6ce8d9316d1f65ebc00273a56be258c1725b32b94de90a3a979557e1", size = 1635602, upload-time = "2026-05-20T13:14:36.344Z" }, + { url = "https://files.pythonhosted.org/packages/37/f7/129f27ca700845b8ee8ca88ce7f43435a1239c2eddb7677fc938822762cf/greenlet-3.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:110a1ca7b49b014b097f6078272c3f4ed31af45b254de5228b79adba879f6af9", size = 238683, upload-time = "2026-05-20T13:11:50.57Z" }, + { url = "https://files.pythonhosted.org/packages/6d/5c/a485a36e87df8d8fd0632ee01511244f5156a20ed3746cc6599340326395/greenlet-3.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f16ba1efc0715b680a18b8123d90dad887c6112ae3555b4b5c32c149540c6b4e", size = 235499, upload-time = "2026-05-20T13:12:42.028Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cb/c62454606daf5640369c94d8a9dd540599b1bfc090e2d2180cb77f4038d2/greenlet-3.5.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d8ab31c9de8651a2facdd5c5bb0011f2380dd1a7af78ce2adf4b56095294fc07", size = 285579, upload-time = "2026-05-20T13:08:56.396Z" }, + { url = "https://files.pythonhosted.org/packages/ec/71/c4270398c2eba968a6071af1dfbdcaeee6ec1c24bc8b435b8cc452700da6/greenlet-3.5.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e300185139abc337ade480c327183adf42a875ac7181bfe66d7d4efea31fbea", size = 651106, upload-time = "2026-05-20T14:00:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/1a/ab/71e34b78a44ec271fb5f550c17bc46d301ddc5953890d935f270b0dcdb5a/greenlet-3.5.1-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7ffdb990dcaa0234cf9845aead5df2e3c3a8b6507d409274dd87e0d5ab05ffc2", size = 663478, upload-time = "2026-05-20T14:05:45.88Z" }, + { url = "https://files.pythonhosted.org/packages/77/96/4efd6fa5c62c85426a0c19077a586258ebc3a2a146ff2493e4312a697a22/greenlet-3.5.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2f82b3597e9d83b63408affed0b48fd0f54935edac4302237b9a837be0dae33c", size = 660800, upload-time = "2026-05-20T13:14:29.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/e0/6c71401a25cac7000261304e866a2f2cc04dc74810d40e2f118aa4799495/greenlet-3.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c0141e37414c10164e702b8fb1473304221ad98f71600850c6ef7ff4880feba0", size = 1617518, upload-time = "2026-05-20T14:02:28.662Z" }, + { url = "https://files.pythonhosted.org/packages/41/26/c5c06643e8c0af9e7bf18e16cb51d0ab7625155f0392e1c9015d66d556cd/greenlet-3.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:50ae25a67bea74ea41fb14b960bc532df73eb713417b2d61892dced82fe8d3bc", size = 1681593, upload-time = "2026-05-20T13:14:39.417Z" }, + { url = "https://files.pythonhosted.org/packages/8a/bd/e11a108317485075e68af9d23039619b86b28130c3b50d227d42edece64b/greenlet-3.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:8a17c42330e261299766b75ac1ea32caa437a9453c8f65d16a13140db378ecd3", size = 239800, upload-time = "2026-05-20T13:09:30.128Z" }, + { url = "https://files.pythonhosted.org/packages/47/f8/8e8e8417b7bf28639a5a56356ef934d0375e1d0c70a57e04d7701e870ffe/greenlet-3.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:7b5f5fae05b8ac6d176a61b60c394a8cbdc2b5b91b81793066e68745cf165e54", size = 236862, upload-time = "2026-05-20T13:09:10.498Z" }, + { url = "https://files.pythonhosted.org/packages/90/12/41bf27fde4d3605d3773ae57751eda182b8be2f5398011c041173b1d9534/greenlet-3.5.1-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:ea8da1e900d758d078810d4255d8c6aa572181896a31ec79d779eb79c3adc9ad", size = 293637, upload-time = "2026-05-20T13:12:35.529Z" }, + { url = "https://files.pythonhosted.org/packages/44/44/ba14b23e9757707050c2f397d305bbcae62e5d7cad122f8b6baec5ae4a1f/greenlet-3.5.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a19570c52a21420dcbc94e661994bc325c0b5b11304540fed514586da5dc8f2e", size = 650840, upload-time = "2026-05-20T14:00:11.079Z" }, + { url = "https://files.pythonhosted.org/packages/a8/37/5ddc2b686a6844f91abecef43411842426da2e1573f60b49ecf2547f4ae1/greenlet-3.5.1-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3d955c89b75eeca4723d7cc14135f393cd47c32e2a6cb4a8e4c6e760a26b0986", size = 656416, upload-time = "2026-05-20T14:05:47.118Z" }, + { url = "https://files.pythonhosted.org/packages/e1/f0/d17510297c35a2992712f0bf84de3779749999f7d3d63aa1f09db7c62dbe/greenlet-3.5.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2daaaebd1a5aa88c49045b6baf9310b3263796bd88db713edf37cf53e7bb4e", size = 654397, upload-time = "2026-05-20T13:14:30.696Z" }, + { url = "https://files.pythonhosted.org/packages/37/eb/147387705bb89092645b012586e7273cb5ed3c90ef7eaf3a69173eaf0209/greenlet-3.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3bfbd69cc349e43bf3a8ae1c85548ff0718efc887615c2db16c3833d7b0b072d", size = 1614469, upload-time = "2026-05-20T14:02:30.192Z" }, + { url = "https://files.pythonhosted.org/packages/a6/4e/37ee0da7732b7aa9896f17e15579a9df34b9fcb9dd494f0adfa749af6623/greenlet-3.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4378720dd888136c27215a0214d32a4d37c3852765d45bc37aad0623423cfd78", size = 1675115, upload-time = "2026-05-20T13:14:40.972Z" }, + { url = "https://files.pythonhosted.org/packages/57/f3/97dfcf4a6eb5077f8a672234216fb5923eb89f2cab7081cb10b2cf75b605/greenlet-3.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:45718441607f9325d948db98cbc691276059316d0358c188c246da4e1d4d23d2", size = 245246, upload-time = "2026-05-20T13:12:22.646Z" }, + { url = "https://files.pythonhosted.org/packages/5d/73/d7f72e34b582f694f4a9b248162db7b09cc458a259ba8f0c0bfa1a34ea7d/greenlet-3.5.1-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:2baee5ca02031757ffe8cc3d69f0cc0aec7065ce362622da74f32d3bcab1c541", size = 285575, upload-time = "2026-05-20T13:12:07.043Z" }, + { url = "https://files.pythonhosted.org/packages/df/59/fa9c6e87dc8ad27a95dabe2f29f372b733d05a8a67470f6c901ed9975655/greenlet-3.5.1-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b1ec3274918a81d3ea778b9e75b56b72b33f300edb6cf7f3a7fe1dae56683de", size = 656428, upload-time = "2026-05-20T14:00:12.556Z" }, + { url = "https://files.pythonhosted.org/packages/f6/f9/e753408871eaa61dfe35e619cfc67512b036fde99893685d50eea9e07146/greenlet-3.5.1-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:111e2390ffffc47d5840b01711dd7fac07d4c09283d0283e7f3264b14e284c64", size = 667064, upload-time = "2026-05-20T14:05:48.662Z" }, + { url = "https://files.pythonhosted.org/packages/96/27/5565b5b40389f1c7753003a07e21892fda8660926787036d5bc0308b8113/greenlet-3.5.1-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e630136e905fe5ff43e86945ae41220b6d1470956a39220e708110ac48d01ea5", size = 665697, upload-time = "2026-05-20T13:14:32.943Z" }, + { url = "https://files.pythonhosted.org/packages/cf/82/e7de4178c0c2d1c9a5a3be3cc0b33e46a85b3ee4a77c071bf7ad8600e079/greenlet-3.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:975eac34b44a7077ca4d421348455b94f0f518246a7f14bc6d2fdcfe5b584368", size = 1621256, upload-time = "2026-05-20T14:02:31.91Z" }, + { url = "https://files.pythonhosted.org/packages/00/10/f2dddcf7dacac17dfc68691809589adad06135eb28930429cf58a6467a2f/greenlet-3.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:9ab3c3a0b2ae6198e67c898dad5215a49f9ae0d0081b3c3ec59f333e39eeca26", size = 1685956, upload-time = "2026-05-20T13:14:42.55Z" }, + { url = "https://files.pythonhosted.org/packages/22/17/4a232b32133230ada52f70e9d7f5b65b0caef8772f01849bd8d149e7e4ca/greenlet-3.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:cbfc69be86e10dcfef5b1e6269d1d6926552aa89ee39e1de3353360c1b6989ab", size = 239802, upload-time = "2026-05-20T13:13:15.481Z" }, + { url = "https://files.pythonhosted.org/packages/c2/ae/4e623a7e6d4d2a5f4cb8e4c82de4169fc637942caae68d6e676b8a128ac5/greenlet-3.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:92fd6d44ac5e5a887c8a5dc4a8ba0ba908527c31c12f78c6bc7dcfe8aab279f6", size = 236853, upload-time = "2026-05-20T13:15:37.301Z" }, + { url = "https://files.pythonhosted.org/packages/7a/57/816d9cff29119da3505b3d6a5e14a8af89006ac36f47f891ff293ee05af1/greenlet-3.5.1-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:a6fdf2433a5441ef9a95464f7c3e674775da1c8c1177fff311cee1acad4626ed", size = 293877, upload-time = "2026-05-20T13:10:19.078Z" }, + { url = "https://files.pythonhosted.org/packages/23/a1/59b0a7c7d140ff1a75626680b9a9899b79a9176cab298b394968fb023295/greenlet-3.5.1-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7546556f0d649f99f6a361098a55f761181bb2ea12ff150bb16d26092ad88244", size = 655333, upload-time = "2026-05-20T14:00:14.758Z" }, + { url = "https://files.pythonhosted.org/packages/72/1b/5efe127597625042218939d01855109f352779050768b670b52edcc16a6c/greenlet-3.5.1-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d5ee3ea898009fa898f85f9982255d35278c477bebe185beca249cab42d4526c", size = 659443, upload-time = "2026-05-20T14:05:50.159Z" }, + { url = "https://files.pythonhosted.org/packages/6c/6d/c404246ea4d22d097a7426d0efb5b781bd7eb67715f09e79001bd552ab18/greenlet-3.5.1-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5c81f74d204d3edd136ebfd50dce53acbb776995d721a0fe801626cfc93b8cd", size = 658356, upload-time = "2026-05-20T13:14:35.091Z" }, + { url = "https://files.pythonhosted.org/packages/51/02/f8ee37fb6d2219329f350af241c27fcf12df57e723d11f6fc6d3bacdadaa/greenlet-3.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:2c18ef16bf6d4dd410e4dd52996888ea1497be26892fe5bbc73580aba4287b8e", size = 1619216, upload-time = "2026-05-20T14:02:33.403Z" }, + { url = "https://files.pythonhosted.org/packages/93/c5/3dc9475ace2c7a3680da12372cddd7f1ac874eb410a1ac48d3e9dab83782/greenlet-3.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:17d86354f0ae6b61bf9be5148d0dd34e06c3cb7c602c671f79f29ac3b150e659", size = 1678427, upload-time = "2026-05-20T13:14:43.71Z" }, + { url = "https://files.pythonhosted.org/packages/df/4e/750c15c317a41ffb36f0bf40b933e3d744a7dede61889f74443ea69690cf/greenlet-3.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:e7516cf6ae6b8a582c2770a0caed47b8a48373ed732c33d69a72913ae6ac923e", size = 245225, upload-time = "2026-05-20T13:13:59.366Z" }, + { url = "https://files.pythonhosted.org/packages/4f/fd/d3baea2eeb7b617efd47e87ca06e2ec2c6118d303aa9e918e0ce16eadc10/greenlet-3.5.1-cp315-cp315t-win_arm64.whl", hash = "sha256:5028648bf2253ec4745add746129d3904121fa7fe871a76bed23c5720573ce0a", size = 239590, upload-time = "2026-05-20T13:13:37.382Z" }, ] [[package]] @@ -1363,6 +1407,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "ipykernel" +version = "7.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "appnope", marker = "sys_platform == 'darwin'" }, + { name = "comm" }, + { name = "debugpy" }, + { name = "ipython" }, + { name = "jupyter-client" }, + { name = "jupyter-core" }, + { name = "matplotlib-inline" }, + { name = "nest-asyncio" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyzmq" }, + { name = "tornado" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ca/8d/b68b728e2d06b9e0051019640a40a9eb7a88fcd82c2e1b5ce70bef5ff044/ipykernel-7.2.0.tar.gz", hash = "sha256:18ed160b6dee2cbb16e5f3575858bc19d8f1fe6046a9a680c708494ce31d909e", size = 176046, upload-time = "2026-02-06T16:43:27.403Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/b9/e73d5d9f405cba7706c539aa8b311b49d4c2f3d698d9c12f815231169c71/ipykernel-7.2.0-py3-none-any.whl", hash = "sha256:3bbd4420d2b3cc105cbdf3756bfc04500b1e52f090a90716851f3916c62e1661", size = 118788, upload-time = "2026-02-06T16:43:25.149Z" }, +] + [[package]] name = "ipython" version = "9.12.0" @@ -1423,11 +1491,11 @@ wheels = [ [[package]] name = "itables" -version = "2.7.3" +version = "2.8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/85/43/c438b31429d4e8cefc76c7cba80a1c948d2e18d74589fb771ea719474bbf/itables-2.7.3.tar.gz", hash = "sha256:7a9680da20aa6495868265d397f1a8cc03d8fdbc592760d3ddb10b4d6081ff73", size = 2428026, upload-time = "2026-03-22T20:05:14.373Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/ef/d4ad45b67f741a70293628aa49adc78ed5665976556dbd0f4d04850636a7/itables-2.8.0.tar.gz", hash = "sha256:b8325fbdbd061004f21ea62425944e5e71a64c2a64849964e0a4b000cde70362", size = 1465957, upload-time = "2026-05-17T09:44:25.457Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/79/a5/60a55bd4cb9488d21adcdce88625e1de9bc14ea454677160c162132838cb/itables-2.7.3-py3-none-any.whl", hash = "sha256:b24ebd6a4ab3edab200f41c56e20a12b12b95ec15d3697aeb9bee71d92c008a9", size = 2462653, upload-time = "2026-03-22T20:05:11.819Z" }, + { url = "https://files.pythonhosted.org/packages/49/bb/3c497ab4a60d921a25340e5e3f646c2e55145454a9312b515f093c16d534/itables-2.8.0-py3-none-any.whl", hash = "sha256:9965c2edc457490d2709d28bd5eabf8f4ac43b44b0e08693fc9bb466a7738926", size = 1489733, upload-time = "2026-05-17T09:44:22.849Z" }, ] [[package]] @@ -1456,56 +1524,56 @@ wheels = [ [[package]] name = "jiter" -version = "0.14.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6e/c1/0cddc6eb17d4c53a99840953f95dd3accdc5cfc7a337b0e9b26476276be9/jiter-0.14.0.tar.gz", hash = "sha256:e8a39e66dac7153cf3f964a12aad515afa8d74938ec5cc0018adcdae5367c79e", size = 165725, upload-time = "2026-04-10T14:28:42.01Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/97/2a/09f70020898507a89279659a1afe3364d57fc1b2c89949081975d135f6f5/jiter-0.14.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:af72f204cf4d44258e5b4c1745130ac45ddab0e71a06333b01de660ab4187a94", size = 315502, upload-time = "2026-04-10T14:26:47.697Z" }, - { url = "https://files.pythonhosted.org/packages/d6/be/080c96a45cd74f9fce5db4fd68510b88087fb37ffe2541ff73c12db92535/jiter-0.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4b77da71f6e819be5fbcec11a453fde5b1d0267ef6ed487e2a392fd8e14e4e3a", size = 314870, upload-time = "2026-04-10T14:26:49.149Z" }, - { url = "https://files.pythonhosted.org/packages/7d/5e/2d0fee155826a968a832cc32438de5e2a193292c8721ca70d0b53e58245b/jiter-0.14.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f4ea612fe8b84b8b04e51d0e78029ecf3466348e25973f953de6e6a59aa4c1", size = 343406, upload-time = "2026-04-10T14:26:50.762Z" }, - { url = "https://files.pythonhosted.org/packages/70/af/bf9ee0d3a4f8dc0d679fc1337f874fe60cdbf841ebbb304b374e1c9aaceb/jiter-0.14.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:62fe2451f8fcc0240261e6a4df18ecbcd58327857e61e625b2393ea3b468aac9", size = 369415, upload-time = "2026-04-10T14:26:52.188Z" }, - { url = "https://files.pythonhosted.org/packages/0f/83/8e8561eadba31f4d3948a5b712fb0447ec71c3560b57a855449e7b8ddc98/jiter-0.14.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6112f26f5afc75bcb475787d29da3aa92f9d09c7858f632f4be6ffe607be82e9", size = 461456, upload-time = "2026-04-10T14:26:53.611Z" }, - { url = "https://files.pythonhosted.org/packages/f6/c9/c5299e826a5fe6108d172b344033f61c69b1bb979dd8d9ddd4278a160971/jiter-0.14.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:215a6cb8fb7dc702aa35d475cc00ddc7f970e5c0b1417fb4b4ac5d82fa2a29db", size = 378488, upload-time = "2026-04-10T14:26:55.211Z" }, - { url = "https://files.pythonhosted.org/packages/5d/37/c16d9d15c0a471b8644b1abe3c82668092a707d9bedcf076f24ff2e380cd/jiter-0.14.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc4ab96a30fb3cb2c7e0cd33f7616c8860da5f5674438988a54ac717caccdbaa", size = 353242, upload-time = "2026-04-10T14:26:56.705Z" }, - { url = "https://files.pythonhosted.org/packages/58/ea/8050cb0dc654e728e1bfacbc0c640772f2181af5dedd13ae70145743a439/jiter-0.14.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:3a99c1387b1f2928f799a9de899193484d66206a50e98233b6b088a7f0c1edb2", size = 356823, upload-time = "2026-04-10T14:26:58.281Z" }, - { url = "https://files.pythonhosted.org/packages/b0/3b/cf71506d270e5f84d97326bf220e47aed9b95e9a4a060758fb07772170ab/jiter-0.14.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ab18d11074485438695f8d34a1b6da61db9754248f96d51341956607a8f39985", size = 392564, upload-time = "2026-04-10T14:27:00.018Z" }, - { url = "https://files.pythonhosted.org/packages/b0/cc/8c6c74a3efb5bd671bfd14f51e8a73375464ca914b1551bc3b40e26ac2c9/jiter-0.14.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:801028dcfc26ac0895e4964cbc0fd62c73be9fd4a7d7b1aaf6e5790033a719b7", size = 520322, upload-time = "2026-04-10T14:27:01.664Z" }, - { url = "https://files.pythonhosted.org/packages/41/24/68d7b883ec959884ddf00d019b2e0e82ba81b167e1253684fa90519ce33c/jiter-0.14.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ad425b087aafb4a1c7e1e98a279200743b9aaf30c3e0ba723aec93f061bd9bc8", size = 552619, upload-time = "2026-04-10T14:27:03.316Z" }, - { url = "https://files.pythonhosted.org/packages/b6/89/b1a0985223bbf3150ff9e8f46f98fc9360c1de94f48abe271bbe1b465682/jiter-0.14.0-cp313-cp313-win32.whl", hash = "sha256:882bcb9b334318e233950b8be366fe5f92c86b66a7e449e76975dfd6d776a01f", size = 205699, upload-time = "2026-04-10T14:27:04.662Z" }, - { url = "https://files.pythonhosted.org/packages/4c/19/3f339a5a7f14a11730e67f6be34f9d5105751d547b615ef593fa122a5ded/jiter-0.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:9b8c571a5dba09b98bd3462b5a53f27209a5cbbe85670391692ede71974e979f", size = 201323, upload-time = "2026-04-10T14:27:06.139Z" }, - { url = "https://files.pythonhosted.org/packages/50/56/752dd89c84be0e022a8ea3720bcfa0a8431db79a962578544812ce061739/jiter-0.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:34f19dcc35cb1abe7c369b3756babf8c7f04595c0807a848df8f26ef8298ef92", size = 191099, upload-time = "2026-04-10T14:27:07.564Z" }, - { url = "https://files.pythonhosted.org/packages/91/28/292916f354f25a1fe8cf2c918d1415c699a4a659ae00be0430e1c5d9ffea/jiter-0.14.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e89bcd7d426a75bb4952c696b267075790d854a07aad4c9894551a82c5b574ab", size = 320880, upload-time = "2026-04-10T14:27:09.326Z" }, - { url = "https://files.pythonhosted.org/packages/ad/c7/b002a7d8b8957ac3d469bd59c18ef4b1595a5216ae0de639a287b9816023/jiter-0.14.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b25beaa0d4447ea8c7ae0c18c688905d34840d7d0b937f2f7bdd52162c98a40", size = 346563, upload-time = "2026-04-10T14:27:11.287Z" }, - { url = "https://files.pythonhosted.org/packages/f9/3b/f8d07580d8706021d255a6356b8fab13ee4c869412995550ce6ed4ddf97d/jiter-0.14.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:651a8758dd413c51e3b7f6557cdc6921faf70b14106f45f969f091f5cda990ea", size = 357928, upload-time = "2026-04-10T14:27:12.729Z" }, - { url = "https://files.pythonhosted.org/packages/47/5b/ac1a974da29e35507230383110ffec59998b290a8732585d04e19a9eb5ba/jiter-0.14.0-cp313-cp313t-win_amd64.whl", hash = "sha256:e1a7eead856a5038a8d291f1447176ab0b525c77a279a058121b5fccee257f6f", size = 203519, upload-time = "2026-04-10T14:27:14.125Z" }, - { url = "https://files.pythonhosted.org/packages/96/6d/9fc8433d667d2454271378a79747d8c76c10b51b482b454e6190e511f244/jiter-0.14.0-cp313-cp313t-win_arm64.whl", hash = "sha256:2e692633a12cda97e352fdcd1c4acc971b1c28707e1e33aeef782b0cbf051975", size = 190113, upload-time = "2026-04-10T14:27:16.638Z" }, - { url = "https://files.pythonhosted.org/packages/4f/1e/354ed92461b165bd581f9ef5150971a572c873ec3b68a916d5aa91da3cc2/jiter-0.14.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:6f396837fc7577871ca8c12edaf239ed9ccef3bbe39904ae9b8b63ce0a48b140", size = 315277, upload-time = "2026-04-10T14:27:18.109Z" }, - { url = "https://files.pythonhosted.org/packages/a6/95/8c7c7028aa8636ac21b7a55faef3e34215e6ed0cbf5ae58258427f621aa3/jiter-0.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a4d50ea3d8ba4176f79754333bd35f1bbcd28e91adc13eb9b7ca91bc52a6cef9", size = 315923, upload-time = "2026-04-10T14:27:19.603Z" }, - { url = "https://files.pythonhosted.org/packages/47/40/e2a852a44c4a089f2681a16611b7ce113224a80fd8504c46d78491b47220/jiter-0.14.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce17f8a050447d1b4153bda4fb7d26e6a9e74eb4f4a41913f30934c5075bf615", size = 344943, upload-time = "2026-04-10T14:27:21.262Z" }, - { url = "https://files.pythonhosted.org/packages/fc/1f/670f92adee1e9895eac41e8a4d623b6da68c4d46249d8b556b60b63f949e/jiter-0.14.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f4f1c4b125e1652aefbc2e2c1617b60a160ab789d180e3d423c41439e5f32850", size = 369725, upload-time = "2026-04-10T14:27:22.766Z" }, - { url = "https://files.pythonhosted.org/packages/01/2f/541c9ba567d05de1c4874a0f8f8c5e3fd78e2b874266623da9a775cf46e0/jiter-0.14.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be808176a6a3a14321d18c603f2d40741858a7c4fc982f83232842689fe86dd9", size = 461210, upload-time = "2026-04-10T14:27:24.315Z" }, - { url = "https://files.pythonhosted.org/packages/ce/a9/c31cbec09627e0d5de7aeaec7690dba03e090caa808fefd8133137cf45bc/jiter-0.14.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:26679d58ba816f88c3849306dd58cb863a90a1cf352cdd4ef67e30ccf8a77994", size = 380002, upload-time = "2026-04-10T14:27:26.155Z" }, - { url = "https://files.pythonhosted.org/packages/50/02/3c05c1666c41904a2f607475a73e7a4763d1cbde2d18229c4f85b22dc253/jiter-0.14.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80381f5a19af8fa9aef743f080e34f6b25ebd89656475f8cf0470ec6157052aa", size = 354678, upload-time = "2026-04-10T14:27:27.701Z" }, - { url = "https://files.pythonhosted.org/packages/7d/97/e15b33545c2b13518f560d695f974b9891b311641bdcf178d63177e8801e/jiter-0.14.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:004df5fdb8ecbd6d99f3227df18ba1a259254c4359736a2e6f036c944e02d7c5", size = 358920, upload-time = "2026-04-10T14:27:29.256Z" }, - { url = "https://files.pythonhosted.org/packages/ad/d2/8b1461def6b96ba44530df20d07ef7a1c7da22f3f9bf1727e2d611077bf1/jiter-0.14.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cff5708f7ed0fa098f2b53446c6fa74c48469118e5cd7497b4f1cd569ab06928", size = 394512, upload-time = "2026-04-10T14:27:31.344Z" }, - { url = "https://files.pythonhosted.org/packages/e3/88/837566dd6ed6e452e8d3205355afd484ce44b2533edfa4ed73a298ea893e/jiter-0.14.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:2492e5f06c36a976d25c7cc347a60e26d5470178d44cde1b9b75e60b4e519f28", size = 521120, upload-time = "2026-04-10T14:27:33.299Z" }, - { url = "https://files.pythonhosted.org/packages/89/6b/b00b45c4d1b4c031777fe161d620b755b5b02cdade1e316dcb46e4471d63/jiter-0.14.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:7609cfbe3a03d37bfdbf5052012d5a879e72b83168a363deae7b3a26564d57de", size = 553668, upload-time = "2026-04-10T14:27:34.868Z" }, - { url = "https://files.pythonhosted.org/packages/ad/d8/6fe5b42011d19397433d345716eac16728ac241862a2aac9c91923c7509a/jiter-0.14.0-cp314-cp314-win32.whl", hash = "sha256:7282342d32e357543565286b6450378c3cd402eea333fc1ebe146f1fabb306fc", size = 207001, upload-time = "2026-04-10T14:27:36.455Z" }, - { url = "https://files.pythonhosted.org/packages/e5/43/5c2e08da1efad5e410f0eaaabeadd954812612c33fbbd8fd5328b489139d/jiter-0.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:bd77945f38866a448e73b0b7637366afa814d4617790ecd88a18ca74377e6c02", size = 202187, upload-time = "2026-04-10T14:27:38Z" }, - { url = "https://files.pythonhosted.org/packages/aa/1f/6e39ac0b4cdfa23e606af5b245df5f9adaa76f35e0c5096790da430ca506/jiter-0.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:f2d4c61da0821ee42e0cdf5489da60a6d074306313a377c2b35af464955a3611", size = 192257, upload-time = "2026-04-10T14:27:39.504Z" }, - { url = "https://files.pythonhosted.org/packages/05/57/7dbc0ffbbb5176a27e3518716608aa464aee2e2887dc938f0b900a120449/jiter-0.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1bf7ff85517dd2f20a5750081d2b75083c1b269cf75afc7511bdf1f9548beb3b", size = 323441, upload-time = "2026-04-10T14:27:41.039Z" }, - { url = "https://files.pythonhosted.org/packages/83/6e/7b3314398d8983f06b557aa21b670511ec72d3b79a68ee5e4d9bff972286/jiter-0.14.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c8ef8791c3e78d6c6b157c6d360fbb5c715bebb8113bc6a9303c5caff012754a", size = 348109, upload-time = "2026-04-10T14:27:42.552Z" }, - { url = "https://files.pythonhosted.org/packages/ae/4f/8dc674bcd7db6dba566de73c08c763c337058baff1dbeb34567045b27cdc/jiter-0.14.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e74663b8b10da1fe0f4e4703fd7980d24ad17174b6bb35d8498d6e3ebce2ae6a", size = 368328, upload-time = "2026-04-10T14:27:44.574Z" }, - { url = "https://files.pythonhosted.org/packages/3b/5f/188e09a1f20906f98bbdec44ed820e19f4e8eb8aff88b9d1a5a497587ff3/jiter-0.14.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1aca29ba52913f78362ec9c2da62f22cdc4c3083313403f90c15460979b84d9b", size = 463301, upload-time = "2026-04-10T14:27:46.717Z" }, - { url = "https://files.pythonhosted.org/packages/ac/f0/19046ef965ed8f349e8554775bb12ff4352f443fbe12b95d31f575891256/jiter-0.14.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8b39b7d87a952b79949af5fef44d2544e58c21a28da7f1bae3ef166455c61746", size = 378891, upload-time = "2026-04-10T14:27:48.32Z" }, - { url = "https://files.pythonhosted.org/packages/c4/c3/da43bd8431ee175695777ee78cf0e93eacbb47393ff493f18c45231b427d/jiter-0.14.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78d918a68b26e9fab068c2b5453577ef04943ab2807b9a6275df2a812599a310", size = 360749, upload-time = "2026-04-10T14:27:49.88Z" }, - { url = "https://files.pythonhosted.org/packages/72/26/e054771be889707c6161dbdec9c23d33a9ec70945395d70f07cfea1e9a6f/jiter-0.14.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:b08997c35aee1201c1a5361466a8fb9162d03ae7bf6568df70b6c859f1e654a4", size = 358526, upload-time = "2026-04-10T14:27:51.504Z" }, - { url = "https://files.pythonhosted.org/packages/c3/0f/7bea65ea2a6d91f2bf989ff11a18136644392bf2b0497a1fa50934c30a9c/jiter-0.14.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:260bf7ca20704d58d41f669e5e9fe7fe2fa72901a6b324e79056f5d52e9c9be2", size = 393926, upload-time = "2026-04-10T14:27:53.368Z" }, - { url = "https://files.pythonhosted.org/packages/3c/a1/b1ff7d70deef61ac0b7c6c2f12d2ace950cdeecb4fdc94500a0926802857/jiter-0.14.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:37826e3df29e60f30a382f9294348d0238ef127f4b5d7f5f8da78b5b9e050560", size = 521052, upload-time = "2026-04-10T14:27:55.058Z" }, - { url = "https://files.pythonhosted.org/packages/0b/7b/3b0649983cbaf15eda26a414b5b1982e910c67bd6f7b1b490f3cfc76896a/jiter-0.14.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:645be49c46f2900937ba0eaf871ad5183c96858c0af74b6becc7f4e367e36e06", size = 553716, upload-time = "2026-04-10T14:27:57.269Z" }, - { url = "https://files.pythonhosted.org/packages/97/f8/33d78c83bd93ae0c0af05293a6660f88a1977caef39a6d72a84afab94ce0/jiter-0.14.0-cp314-cp314t-win32.whl", hash = "sha256:2f7877ed45118de283786178eceaf877110abacd04fde31efff3940ae9672674", size = 207957, upload-time = "2026-04-10T14:27:59.285Z" }, - { url = "https://files.pythonhosted.org/packages/d6/ac/2b760516c03e2227826d1f7025d89bf6bf6357a28fe75c2a2800873c50bf/jiter-0.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:14c0cb10337c49f5eafe8e7364daca5e29a020ea03580b8f8e6c597fed4e1588", size = 204690, upload-time = "2026-04-10T14:28:00.962Z" }, - { url = "https://files.pythonhosted.org/packages/dc/2e/a44c20c58aeed0355f2d326969a181696aeb551a25195f47563908a815be/jiter-0.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:5419d4aa2024961da9fe12a9cfe7484996735dca99e8e090b5c88595ef1951ff", size = 191338, upload-time = "2026-04-10T14:28:02.853Z" }, +version = "0.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/b5/55f06bb281d92fb3cc86d14e1def2bd908bb77693183e7cb1f5a3c388b0c/jiter-0.15.0.tar.gz", hash = "sha256:4251acc80e2b7c9b7b8823456ea0fceeb0734dac2df7636d3c711b38476b5a76", size = 166640, upload-time = "2026-05-19T10:09:48.361Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/f4/f708c900ecee41b2025ef8413d5351e5649eb2125c506f6720cc69b06f5c/jiter-0.15.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1c11465f97e2abf45a014b83b730222f8f1c5335e802c7055a67d50de6f1f4e3", size = 307829, upload-time = "2026-05-19T10:07:59.704Z" }, + { url = "https://files.pythonhosted.org/packages/86/59/db537c0949e83668c38481d426b9f2fd5ab758c4ee53a811dd0a510626a0/jiter-0.15.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d1e7b1776f0797956c509e123d0952d10d293a9492dea9f288ab9570ec01d1a5", size = 308445, upload-time = "2026-05-19T10:08:01.184Z" }, + { url = "https://files.pythonhosted.org/packages/37/38/ea0e13b18c30ef951da0d47d39e7fa9edb82a93a62990ffbd7cea9b622d4/jiter-0.15.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:351a341c2105aa430b7047e30f1bf7975f6313b00165d3fc07be2edaf741f279", size = 336181, upload-time = "2026-05-19T10:08:02.688Z" }, + { url = "https://files.pythonhosted.org/packages/58/fc/2303901b16c4ba05865588990a420c0b4156270b44379c20931544a1d962/jiter-0.15.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4ab395feec8d249ec4044e228e98a7033f043426a265df439dc3698823f0a4e4", size = 362985, upload-time = "2026-05-19T10:08:04.394Z" }, + { url = "https://files.pythonhosted.org/packages/5b/6f/11bace093c52e7d4d26c8e606ccd7ae8c972189622469ec0d9e28161e28b/jiter-0.15.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2a438005b6f22d0273413484d6094d7c2c5d10ec1b3a3bf128e0d1d3ba53258", size = 453292, upload-time = "2026-05-19T10:08:05.967Z" }, + { url = "https://files.pythonhosted.org/packages/22/db/987f2f086ca4d7a6582eb4ccd513f9b26b42d9e4243a087609a3137a8fc7/jiter-0.15.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f18f85e4218d1b40f000f42a92239a7a61a902cd42c65e6c360dbd17dcb20894", size = 373501, upload-time = "2026-05-19T10:08:07.857Z" }, + { url = "https://files.pythonhosted.org/packages/8f/7c/89fbcabb2739b7a5b8dc959a1b6c5761f6484f5fed3486854b3c789bb1de/jiter-0.15.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1aa62e277fc1cbd80e6deacae6f4d983b41b3d7728e0645c5d741a6149bba45", size = 344683, upload-time = "2026-05-19T10:08:09.431Z" }, + { url = "https://files.pythonhosted.org/packages/30/6f/6cca7692e7dddfec6d8d76c54dc97f2af2a41df4ac0674b999df1f09a5f3/jiter-0.15.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:6550fa135c7deb8ead6af49ed7ff648532ea8334a1447fe34a36315ef79c5c29", size = 350892, upload-time = "2026-05-19T10:08:11.352Z" }, + { url = "https://files.pythonhosted.org/packages/39/14/0338d6190cb8e6d22e677ab1d4eabd4117f67cca70c54cd04b82ff64e068/jiter-0.15.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:066f8f33f18b2419cd8213b2436fa7fbc9c499f315971cfa3ce1f9820c001b1b", size = 388723, upload-time = "2026-05-19T10:08:12.912Z" }, + { url = "https://files.pythonhosted.org/packages/90/31/cc19f4a1bdb6afb09ce6a2f2615aa8d44d994eba0d8e6105ed1af920e736/jiter-0.15.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:75e8a04e91432dde9f1838373cf93d23726c79d3e908d319acf0e796f85592e7", size = 516648, upload-time = "2026-05-19T10:08:14.808Z" }, + { url = "https://files.pythonhosted.org/packages/49/9f/833c541512cd091b63c10c0381973dfe11bc7a503a818c16384417e0c81e/jiter-0.15.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:a97261f1fccb8e50ecd2890a96e46efdc3f57c80a197324c6777827231eca712", size = 547382, upload-time = "2026-05-19T10:08:16.927Z" }, + { url = "https://files.pythonhosted.org/packages/d2/11/e7b70e91f90bc4477e8eee9e8a5f7cf3cb41b4525d6394dc98a714eb8f7f/jiter-0.15.0-cp313-cp313-win32.whl", hash = "sha256:c77496cb10bd7549690fbbab3e5ec05857b83e49276f4a9423a766ddd2afcd4c", size = 205845, upload-time = "2026-05-19T10:08:18.401Z" }, + { url = "https://files.pythonhosted.org/packages/4b/23/5c20d9ad6f02c493e4023e5d2d09e1c1f15fe2753c9102c544aff068a88e/jiter-0.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:b15741f501469009ae0ae90b7147958a664a7dede40aa7ff174a8a4645f546d0", size = 196842, upload-time = "2026-05-19T10:08:20.131Z" }, + { url = "https://files.pythonhosted.org/packages/6b/11/1eb400ef248e8c925fd883fbe325daf5e42cd1b0d308539dd332bd4f7ffc/jiter-0.15.0-cp313-cp313-win_arm64.whl", hash = "sha256:5d6a60072b44c3c2b797a7ddcbcbbf2b34ea3cfd4721580fbfd2a09d9d9b84ba", size = 192212, upload-time = "2026-05-19T10:08:21.807Z" }, + { url = "https://files.pythonhosted.org/packages/8a/60/2fd8d7c79da8acf9b7b277c7616847773779356b92acfc9bb158452174da/jiter-0.15.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:ef1fd24d9413f6209e00d3d5a453e67acfe004a25cc6c8e8484faed4311ab9e8", size = 315065, upload-time = "2026-05-19T10:08:23.218Z" }, + { url = "https://files.pythonhosted.org/packages/46/f4/008fb7d65e8ac2abf00811651a661e025c4ba80bbc6f378450384ddd3aed/jiter-0.15.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:144f8e72cb53dab146347b91cceac01f5481237f2b93b4a339a1ee8f8878b67c", size = 339444, upload-time = "2026-05-19T10:08:24.701Z" }, + { url = "https://files.pythonhosted.org/packages/00/55/90b0c7b9c6896c0f2a591dd36d36b71d22e09674bfef178fa03ba3f81499/jiter-0.15.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:553fcac2ef2cb990877f9fc0833b8b629a3e6a5670b6b5fd58219b41a653ddc4", size = 347779, upload-time = "2026-05-19T10:08:26.408Z" }, + { url = "https://files.pythonhosted.org/packages/51/6b/69666cec5000fd57734c118437394516c749ae8dbeea9fb66d6fef9c4775/jiter-0.15.0-cp313-cp313t-win_amd64.whl", hash = "sha256:774f93f65031856bf14ad9f59bdcab8b8cad501e5ceabd51ba3525f76937a25b", size = 200395, upload-time = "2026-05-19T10:08:28.055Z" }, + { url = "https://files.pythonhosted.org/packages/39/04/a6aa62cd27e8149b0d28df5561f10f6cceaf7935a9ccf3f1c5a05f9a0cd8/jiter-0.15.0-cp313-cp313t-win_arm64.whl", hash = "sha256:f1e1754960f38ec40613a07e5e372df67acb3b890fb383b6fb3de3e49ddbf3c7", size = 190516, upload-time = "2026-05-19T10:08:29.35Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d2/079f350ebf7859d081de30aa890f9e3be68516f754f3ba32366ffff4dcee/jiter-0.15.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:ac0d9ddea4350974be7a221fc25895f251a8fee748c889bdced2141c0fec1a49", size = 308884, upload-time = "2026-05-19T10:08:31.667Z" }, + { url = "https://files.pythonhosted.org/packages/04/4e/a2c30a7f69b48c03b20935d647479106fe932f6e63f75faf53937197e05d/jiter-0.15.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:01a8222cf05ab1128e239421156c207949808acaaea2bdfd33130ae666786e86", size = 310028, upload-time = "2026-05-19T10:08:33.304Z" }, + { url = "https://files.pythonhosted.org/packages/40/90/2e7cdfd3cf8ca967be38c48f5cf474d79f089efaf559a40f15984a77ae69/jiter-0.15.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:182226cbc930c9fab81bc2e41a4da672f89539906dadb05e75670ac07b94f71f", size = 337485, upload-time = "2026-05-19T10:08:35.259Z" }, + { url = "https://files.pythonhosted.org/packages/9b/11/15a1aa28b120b8ee5b4f1fb894c125046225f09847738bd64233d3b84883/jiter-0.15.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:71683c38c825452999b5717fcae07ea708e8c93003e808be4319c1b02e3d176e", size = 364223, upload-time = "2026-05-19T10:08:36.694Z" }, + { url = "https://files.pythonhosted.org/packages/b7/25/f442e8af5f3d0dcf47b39e83a0efd9ee45ea946aa6d04625dc3181eae3b6/jiter-0.15.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:30f2218e6a9e5c18bc10fe6d41ac189c442c88eacf11bad9f28ef95a9bef00e6", size = 456387, upload-time = "2026-05-19T10:08:38.143Z" }, + { url = "https://files.pythonhosted.org/packages/da/f4/37f2d2c9f64f49af7da652ed7532bb5a2372e588e6927c3fdd76f911db65/jiter-0.15.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5157de9f76eb4bc5ea74a1219366a25f945ad305641d74e04f59c54087091aa9", size = 374461, upload-time = "2026-05-19T10:08:39.869Z" }, + { url = "https://files.pythonhosted.org/packages/60/28/edcfbbbf0cb15436f36664a8908a0df47ab9006298d4cd937dc08ea932d6/jiter-0.15.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90c5db5527c221249a876160663ab891ace358c17f7b9c93ec1478b7f0550e5c", size = 345924, upload-time = "2026-05-19T10:08:41.668Z" }, + { url = "https://files.pythonhosted.org/packages/47/13/89fba6398dab7f202b7278c4b4aac122399d2c0183971c4a57a3b7088df5/jiter-0.15.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:3e4540b8e74e4268811ac05db226a6a128ff572e7e0ce3f1163b693cadb184cd", size = 352283, upload-time = "2026-05-19T10:08:43.091Z" }, + { url = "https://files.pythonhosted.org/packages/1b/da/0f6af8cef2c565a1ab44d970f268c43ccaa72707386ea6388e6fe2b6cd26/jiter-0.15.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:62ebd14e47e9aed9df4472afcb2663668ce4d74891cd54f86bf6e44029d6dc89", size = 389985, upload-time = "2026-05-19T10:08:44.915Z" }, + { url = "https://files.pythonhosted.org/packages/a1/ec/b9cb7d6d29e24ee14910266157d2a279d7a8f60ee0df7fa840882976ba64/jiter-0.15.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0be6f5ad41a809f303f416d17cec92a7a725902fb9b4f3de3d19362ac0ef8554", size = 517695, upload-time = "2026-05-19T10:08:46.486Z" }, + { url = "https://files.pythonhosted.org/packages/64/5e/6d1bda880723aae0ad86b4b763f044362448efe31e3e819635d41cb03451/jiter-0.15.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:813dfbb17d65328bf86e5f0905dd277ba2265d3ca20556e86c0c7035b7182e5a", size = 548868, upload-time = "2026-05-19T10:08:48.026Z" }, + { url = "https://files.pythonhosted.org/packages/0c/72/7de501cf38dcacaf35098796f3a50e0f2e338baba18a58946c618544b809/jiter-0.15.0-cp314-cp314-win32.whl", hash = "sha256:50e51156192722a9c58db112837d3f8ef96fb3c5ecc14e95f409134b08b158ec", size = 206380, upload-time = "2026-05-19T10:08:49.738Z" }, + { url = "https://files.pythonhosted.org/packages/1e/a9/e19addf4b0c1bdce52c6da12351e6bc42c340c45e7c09e2158e46d293ccc/jiter-0.15.0-cp314-cp314-win_amd64.whl", hash = "sha256:30ce1a5d16b5641dc935d50ef775af6a0871e3d14ab05d6fc54dff371b78e558", size = 197687, upload-time = "2026-05-19T10:08:51.088Z" }, + { url = "https://files.pythonhosted.org/packages/f2/c9/776b1db01db25fc6c1d58d1979a37b0a9fe787e5f5b1d062d2eaacb77923/jiter-0.15.0-cp314-cp314-win_arm64.whl", hash = "sha256:510c8b3c17a0ed9ac69850c0438dada3c9b82d9c4d589fcb62002a5a9cf3a866", size = 192571, upload-time = "2026-05-19T10:08:52.451Z" }, + { url = "https://files.pythonhosted.org/packages/a0/f6/45bb4670bacf300fd2c7abadbfb3af376e5f1b6ae75fd9bc069891d15870/jiter-0.15.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7553333dd0930c104a5a0db8df72bf7219fe663d731383b576bb6ed6351c984d", size = 317151, upload-time = "2026-05-19T10:08:53.867Z" }, + { url = "https://files.pythonhosted.org/packages/d7/68/ed635ad5acd7b73e454283083bbb7c8205ad10e88b0d9d7d793b09fe8226/jiter-0.15.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2143ab06181d2b029eedcb6af3cebe95f11bbac62441781860f98ee9330a6a6", size = 341243, upload-time = "2026-05-19T10:08:55.383Z" }, + { url = "https://files.pythonhosted.org/packages/5d/db/3ff4176b817b8ea33879e71e13d8bc2b0d481a7ed3fe9e080f333d415c16/jiter-0.15.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6eac374c5c975709b69c10f09afd199df74150172156ad10c8d4fd785b7da995", size = 363629, upload-time = "2026-05-19T10:08:56.928Z" }, + { url = "https://files.pythonhosted.org/packages/ab/24/5f8270e0ba9c883582f96f722f8a0b58015c7ce1f8c6d4571cf394e99b6b/jiter-0.15.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b3b3b775e33d3bfaec9899edc526ae97b0da0bf9d071a46124ba419149a414f8", size = 456198, upload-time = "2026-05-19T10:08:58.618Z" }, + { url = "https://files.pythonhosted.org/packages/45/5b/76fc02b0b5c54c3d18c60653156e2f76fde1816f9b4722db68d6ee2c897e/jiter-0.15.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eda3071db3346334beae1360b46da4606da57bf3528c167b3c38533afaf9f2c5", size = 373710, upload-time = "2026-05-19T10:09:00.151Z" }, + { url = "https://files.pythonhosted.org/packages/c4/52/4310821b0ea9277994d3e1f49fc6a4b34e4800caebacb2c0af81da59a454/jiter-0.15.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6694a173ecabc12eb60efbc0b474464ead1951ff65cd8b1e72100715c64512b", size = 349901, upload-time = "2026-05-19T10:09:01.621Z" }, + { url = "https://files.pythonhosted.org/packages/93/fe/67648c35b3594fba8854ac64cc8a826d8bcd18324bbdb53d77697c60b6ef/jiter-0.15.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:a254e10b593624d230c365b6d616b22ca0ad65e63a16e6631c2b3466022e6ba8", size = 352438, upload-time = "2026-05-19T10:09:03.216Z" }, + { url = "https://files.pythonhosted.org/packages/cb/28/0a1879d07ad6b3e025a2750027363452ced93c2d16d1c9d4b153ffd51c91/jiter-0.15.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d8d2955167274e15d79a7a020afdd9b39c990eb80b2d89fca695d92dcfdd38ec", size = 388152, upload-time = "2026-05-19T10:09:04.741Z" }, + { url = "https://files.pythonhosted.org/packages/c1/78/46c6f6b56ba85c90021f4afd72ed42f691f8f84daacb5fe27277070e3858/jiter-0.15.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:acf4ee4d1fc55917239fe72972fb292dd773055d05eb040d36f4326e02cc2c0e", size = 517707, upload-time = "2026-05-19T10:09:06.231Z" }, + { url = "https://files.pythonhosted.org/packages/ca/cb/720662d4c88fcad606e826fef5424365527ba43ce4868a479aed8f8c507e/jiter-0.15.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:e7196e56f1cd69af1dbb07dff02dcfb260a50b45a82d409d92a06fedb32473b5", size = 548241, upload-time = "2026-05-19T10:09:08.093Z" }, + { url = "https://files.pythonhosted.org/packages/60/e3/935b8034fd143f21125c87d51404a9e0e1449186a494405721ff5d1d695e/jiter-0.15.0-cp314-cp314t-win32.whl", hash = "sha256:7f6163c0f10b055245f814dcc59f4818da60dfe72f3e72ab89fc24b6bd5e9c52", size = 207950, upload-time = "2026-05-19T10:09:09.616Z" }, + { url = "https://files.pythonhosted.org/packages/93/59/984fd9ece895953dad3e0880a650e766f5a2da2c5514f0eafdaaabbeb5f9/jiter-0.15.0-cp314-cp314t-win_amd64.whl", hash = "sha256:980c256edb05b78a111b99c4de3b1d32e31634b867fd1fc2cf726e7b7bba9854", size = 200055, upload-time = "2026-05-19T10:09:11.367Z" }, + { url = "https://files.pythonhosted.org/packages/0e/a4/cf8d779feb133a27a2e3bc833bccb9e13aa332cdf820497ebf72c10ce8c3/jiter-0.15.0-cp314-cp314t-win_arm64.whl", hash = "sha256:66b1880df2d01e206e8339769d1c7c1753bcb653efd6289e203f6f24ebada0c0", size = 191244, upload-time = "2026-05-19T10:09:12.74Z" }, ] [[package]] @@ -1582,6 +1650,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, ] +[[package]] +name = "jsonschema-pydantic" +version = "0.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d4/3e/088097e87f6b24b162cd42b090b1d3891b68e9dac96ceeac8ed5dff94db7/jsonschema_pydantic-0.6.tar.gz", hash = "sha256:6069a8929a333a7c7ea8510e9de50f062e747e655e6ba106da5af1981f995270", size = 6227, upload-time = "2024-02-03T21:50:23.285Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/67/cbb63be985519b51d13499e726db2c960069588d5c2c479859856158b4de/jsonschema_pydantic-0.6-py3-none-any.whl", hash = "sha256:98385ed53ab87598665956b43756746350e2b60411a38381231f9703d36e40eb", size = 4154, upload-time = "2024-02-03T21:50:20.622Z" }, +] + [[package]] name = "jsonschema-specifications" version = "2025.9.1" @@ -1594,6 +1674,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, ] +[[package]] +name = "jupyter-client" +version = "8.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jupyter-core" }, + { name = "python-dateutil" }, + { name = "pyzmq" }, + { name = "tornado" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/05/e4/ba649102a3bc3fbca54e7239fb924fd434c766f855693d86de0b1f2bec81/jupyter_client-8.8.0.tar.gz", hash = "sha256:d556811419a4f2d96c869af34e854e3f059b7cc2d6d01a9cd9c85c267691be3e", size = 348020, upload-time = "2026-01-08T13:55:47.938Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/0b/ceb7694d864abc0a047649aec263878acb9f792e1fec3e676f22dc9015e3/jupyter_client-8.8.0-py3-none-any.whl", hash = "sha256:f93a5b99c5e23a507b773d3a1136bd6e16c67883ccdbd9a829b0bbdb98cd7d7a", size = 107371, upload-time = "2026-01-08T13:55:45.562Z" }, +] + +[[package]] +name = "jupyter-core" +version = "5.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "platformdirs" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/02/49/9d1284d0dc65e2c757b74c6687b6d319b02f822ad039e5c512df9194d9dd/jupyter_core-5.9.1.tar.gz", hash = "sha256:4d09aaff303b9566c3ce657f580bd089ff5c91f5f89cf7d8846c3cdf465b5508", size = 89814, upload-time = "2025-10-16T19:19:18.444Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/e7/80988e32bf6f73919a113473a604f5a8f09094de312b9d52b79c2df7612b/jupyter_core-5.9.1-py3-none-any.whl", hash = "sha256:ebf87fdc6073d142e114c72c9e29a9d7ca03fad818c5d300ce2adc1fb0743407", size = 29032, upload-time = "2025-10-16T19:19:16.783Z" }, +] + [[package]] name = "jupyterlab-widgets" version = "3.0.16" @@ -1605,45 +1714,58 @@ wheels = [ [[package]] name = "langchain" -version = "0.3.28" +version = "1.3.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, - { name = "langchain-text-splitters" }, - { name = "langsmith" }, + { name = "langgraph" }, { name = "pydantic" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "sqlalchemy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/87/bb/a65e29c8e4aaf0348c2617962e427c8e760d82a67adbd197019e49c7769d/langchain-0.3.28.tar.gz", hash = "sha256:30a32f44cc6690bcc6a6fb7c14d61a15406d5eda1a0e7eab60b3660944888741", size = 10242473, upload-time = "2026-03-06T22:45:17.911Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d5/d0/c7f9d3d26c0e3f8bb146c6d707ee0fc1d30d8da65a59626e8a580085e929/langchain-1.3.2.tar.gz", hash = "sha256:ffd5f204a46b5fa1a38bf89ba3b45ca0902c02d18fa7d2a2eaeaeb1f5bf19d0a", size = 600598, upload-time = "2026-05-26T18:17:57.715Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5b/f5/ecd71e5b78e67944b2600a155ef63000bc00148e6794e8e7809b2453887a/langchain-0.3.28-py3-none-any.whl", hash = "sha256:1ba1244477b67b812b775f346209fa596e78bf055a34e45ce22acb7a45842a32", size = 1024717, upload-time = "2026-03-06T22:45:15.545Z" }, + { url = "https://files.pythonhosted.org/packages/f8/82/a54edcd1c48163de5642eb10fa2cb58b13a8889c659964f63f0306b58b1e/langchain-1.3.2-py3-none-any.whl", hash = "sha256:900f6b3f4ee08b9ba3cdbe667dbf42525bd6f66a4a07a7f1db26262673e41ed6", size = 121225, upload-time = "2026-05-26T18:17:56.075Z" }, ] [[package]] name = "langchain-anthropic" -version = "0.3.22" +version = "1.4.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anthropic" }, { name = "langchain-core" }, { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b8/ac/4791e4451e1972f80cb517e19d003678239921fc0685a4c4b265fe47e216/langchain_anthropic-0.3.22.tar.gz", hash = "sha256:6c440278bd8012bc94ae341f416bfc724fdc5d2d2b69630fe6e82fa6ee9682ac", size = 471312, upload-time = "2025-10-09T18:39:26.983Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/e3/d2f9dec95602524b1cfb4be2747ba5bc38d32501b2a56cb4bcb76e80bb45/langchain_anthropic-1.4.3.tar.gz", hash = "sha256:f8a2442463c0629b1b3110eaeaa56fdbdc87df2a802f8c7f5ecf611eb4874ec8", size = 685219, upload-time = "2026-05-03T17:33:27.118Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/55/482a1968c95275e8be6d8c1e53b54f0f7be0b8b155ce1608c947a95cf543/langchain_anthropic-1.4.3-py3-none-any.whl", hash = "sha256:65466e0f2f95909a009708f2958e917dfdbfab79c612b4484a30866a85e1f291", size = 50389, upload-time = "2026-05-03T17:33:25.671Z" }, +] + +[[package]] +name = "langchain-classic" +version = "1.0.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, + { name = "langchain-text-splitters" }, + { name = "langsmith" }, + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "sqlalchemy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/78/84b5065816f348c39fefa4316f209f0135e8410216340a953bec17d9e4e4/langchain_classic-1.0.7.tar.gz", hash = "sha256:debbec8065e69b95108d2652e8d5c44f4516e19aa8d716c02ed2211c3aee099d", size = 10554118, upload-time = "2026-05-07T15:46:56.8Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/ac/019fd9d45716a4d74c154f160665074ae49885ff4764c8313737f5fda348/langchain_anthropic-0.3.22-py3-none-any.whl", hash = "sha256:17721b240342a1a3f70bf0b2ff33520ba60d69008e3b9433190a62a52ff87cf6", size = 32592, upload-time = "2025-10-09T18:39:25.766Z" }, + { url = "https://files.pythonhosted.org/packages/f5/78/2d9980d028ff0523eea503a77c200e2ff252a3a75eb6e7842bcf5f9c979b/langchain_classic-1.0.7-py3-none-any.whl", hash = "sha256:d9d9be38f7aa534ed0259c2410432e34a1f80b1d491e686749bb55af56479be3", size = 1041386, upload-time = "2026-05-07T15:46:54.845Z" }, ] [[package]] name = "langchain-community" -version = "0.3.31" +version = "0.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, - { name = "dataclasses-json" }, { name = "httpx-sse" }, - { name = "langchain" }, + { name = "langchain-classic" }, { name = "langchain-core" }, { name = "langsmith" }, { name = "numpy" }, @@ -1653,17 +1775,18 @@ dependencies = [ { name = "sqlalchemy" }, { name = "tenacity" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/83/49/2ff5354273809e9811392bc24bcffda545a196070666aef27bc6aacf1c21/langchain_community-0.3.31.tar.gz", hash = "sha256:250e4c1041539130f6d6ac6f9386cb018354eafccd917b01a4cff1950b80fd81", size = 33241237, upload-time = "2025-10-07T20:17:57.857Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ea/0c/e3aca1f2b1c5b95f8b87cb2b6e81a6f20d538c07a128419dc01cef0617b6/langchain_community-0.4.2.tar.gz", hash = "sha256:a99308160d53d7e9b5965ee665e5173709914338210089fd5788ad724432c21e", size = 33268708, upload-time = "2026-05-22T19:42:59.374Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e6/0a/b8848db67ad7c8d4652cb6f4cb78d49b5b5e6e8e51d695d62025aa3f7dbc/langchain_community-0.3.31-py3-none-any.whl", hash = "sha256:1c727e3ebbacd4d891b07bd440647668001cea3e39cbe732499ad655ec5cb569", size = 2532920, upload-time = "2025-10-07T20:17:54.91Z" }, + { url = "https://files.pythonhosted.org/packages/8f/39/5d97e42a3e95dc2a6d71b2f902a3fae71786131e11d01bddb604accb0ebe/langchain_community-0.4.2-py3-none-any.whl", hash = "sha256:84dd8c5122532394d5b6849a5fc9995ef28e4f77227daeb09f24b3d942e9e466", size = 2364406, upload-time = "2026-05-22T19:42:57.103Z" }, ] [[package]] name = "langchain-core" -version = "0.3.84" +version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jsonpatch" }, + { name = "langchain-protocol" }, { name = "langsmith" }, { name = "packaging" }, { name = "pydantic" }, @@ -1672,40 +1795,136 @@ dependencies = [ { name = "typing-extensions" }, { name = "uuid-utils" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/3e/1e70598fac522eaeeeb22f03107da06495160533b25ba4388be9cef01d55/langchain_core-0.3.84.tar.gz", hash = "sha256:814b75bfe67a8460a53f5839bae9505bbfffc7af6f1aa0a5155715563f5cc490", size = 599092, upload-time = "2026-04-08T19:14:00.106Z" } +sdist = { url = "https://files.pythonhosted.org/packages/59/de/679a53472c25860837e32c0442c962fa86e95317a36460e2c9d5c91b17c2/langchain_core-1.4.0.tar.gz", hash = "sha256:1dc341eed802ed9c117c0df3923c991e5e9e226571e5725c194eeb5bd93d1a7f", size = 920260, upload-time = "2026-05-11T18:42:35.919Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/1a/86c38c27b81913a1c6c12448cab55defb5a1097c7dc9a4cea83f55477a2d/langchain_core-1.4.0-py3-none-any.whl", hash = "sha256:23cbbdb46e38ddd1dd5247e6167e96013eae74bea4c5949c550809970a9e565c", size = 548120, upload-time = "2026-05-11T18:42:33.992Z" }, +] + +[[package]] +name = "langchain-mcp-tools" +version = "0.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsonschema-pydantic" }, + { name = "langchain" }, + { name = "langchain-core" }, + { name = "mcp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/e4/668b7cda0109f5325d7f4ea55896ccb70328c4cda52845dfd9278cc8fca2/langchain_mcp_tools-0.3.3.tar.gz", hash = "sha256:1a3b3ce86e4651983508d626c1f2adc80029eb9a66dad32673aa38d5905b8a43", size = 28777, upload-time = "2026-03-28T03:53:37.121Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7d/1e093a543925f55f3c0c59a48870c3f216bf8ee559e89bfbd72d445cf211/langchain_mcp_tools-0.3.3-py3-none-any.whl", hash = "sha256:a0f5bd5a95ad0549c4cfdb881be20d4383fcfafa5e674abad1a1a542a3fa41db", size = 24262, upload-time = "2026-03-28T03:53:35.587Z" }, +] + +[[package]] +name = "langchain-ollama" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, + { name = "ollama" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d4/9b/6641afe8a5bf807e454fd464eddfc7eb2f2df53cb0b29744381171f9c609/langchain_ollama-1.1.0.tar.gz", hash = "sha256:f776f56f6782ae4da7692579b94a6575906118318d1023b455d7207f9d059811", size = 133075, upload-time = "2026-04-07T02:48:00.873Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/5b/ba75d5b80bd1f60ae799c8cbda5477eb7489fb21d40c967ec509bbd51933/langchain_core-0.3.84-py3-none-any.whl", hash = "sha256:d0b3a7b6473e30a2b3d4588ee09dc6471b8d38c46cd48f3e7c3d1ab6547f63cb", size = 459123, upload-time = "2026-04-08T19:13:57.818Z" }, + { url = "https://files.pythonhosted.org/packages/2c/b2/c2acb076590a98bee2816ed5f285e00df162a34238f9e276e175e14ebc35/langchain_ollama-1.1.0-py3-none-any.whl", hash = "sha256:43ac83a6eacb0f43855810739794dd55019e0d9b17bdcf3ecb3b1991ac3b59dd", size = 31413, upload-time = "2026-04-07T02:47:59.642Z" }, ] [[package]] name = "langchain-openai" -version = "0.3.35" +version = "1.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, { name = "openai" }, { name = "tiktoken" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fb/96/06d0d25a37e05a0ff2d918f0a4b0bf0732aed6a43b472b0b68426ce04ef8/langchain_openai-0.3.35.tar.gz", hash = "sha256:fa985fd041c3809da256a040c98e8a43e91c6d165b96dcfeb770d8bd457bf76f", size = 786635, upload-time = "2025-10-06T15:09:28.463Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f7/1b/c506c7f41156d3a6b4582b4c487f480001b8741deecc6e2d4931fdf4cf2c/langchain_openai-1.2.2.tar.gz", hash = "sha256:8698ffcee9a086e91ab6d207f0026181a03effcbf86bf9aee1808ee35af69dcc", size = 1147539, upload-time = "2026-05-21T22:08:31.123Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/8e/7406c99afacafc8c2ce0fa4152f9f8b9598c93ceb291959821abd053b982/langchain_openai-1.2.2-py3-none-any.whl", hash = "sha256:7da39a3c70cbafa93853456199e39a264dc70651be79b12ac49b4f6a448bce2d", size = 99631, upload-time = "2026-05-21T22:08:29.527Z" }, +] + +[[package]] +name = "langchain-protocol" +version = "0.0.15" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4f/24/9777489d6fbbee64af0c8f96d4f840239c408cf694f3394672807dafc490/langchain_protocol-0.0.15.tar.gz", hash = "sha256:9ab2d11ee73944754f10e037e717098d3a6796f0e58afa9cadda6154e7655ade", size = 5862, upload-time = "2026-05-01T22:30:04.748Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d8/d5/c90c5478215c20ee71d8feaf676f7ffd78d0568f8c98bd83f81ce7562ed7/langchain_openai-0.3.35-py3-none-any.whl", hash = "sha256:76d5707e6e81fd461d33964ad618bd326cb661a1975cef7c1cb0703576bdada5", size = 75952, upload-time = "2025-10-06T15:09:27.137Z" }, + { url = "https://files.pythonhosted.org/packages/1d/7a/9c97a7b9cbe4c5dc6a44cdb1545450c28f0c8ce89b9c1f0ee7fbad896263/langchain_protocol-0.0.15-py3-none-any.whl", hash = "sha256:461eb794358f83d5e42635a5797799ffec7b4702314e34edf73ac21e75d3ef79", size = 6982, upload-time = "2026-05-01T22:30:03.877Z" }, ] [[package]] name = "langchain-text-splitters" -version = "0.3.11" +version = "1.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/9f/6c545900fefb7b00ddfa3f16b80d61338a0ec68c31c5451eeeab99082760/langchain_text_splitters-1.1.2.tar.gz", hash = "sha256:782a723db0a4746ac91e251c7c1d57fd23636e4f38ed733074e28d7a86f41627", size = 293580, upload-time = "2026-04-16T14:20:39.162Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/26/1ef06f56198d631296d646a6223de35bcc6cf9795ceb2442816bc963b84c/langchain_text_splitters-1.1.2-py3-none-any.whl", hash = "sha256:a2de0d799ff31886429fd6e2e0032df275b60ec817c19059a7b46181cc1c2f10", size = 35903, upload-time = "2026-04-16T14:20:38.243Z" }, +] + +[[package]] +name = "langgraph" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, + { name = "langgraph-checkpoint" }, + { name = "langgraph-prebuilt" }, + { name = "langgraph-sdk" }, + { name = "pydantic" }, + { name = "xxhash" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e6/5a/ffc12434ee8aecab830d58b4d204ddea45073eae7639c963310f671a5bf5/langgraph-1.2.2.tar.gz", hash = "sha256:f54a98458976b3ff0774683867df125fb52d8dbedeb2441d0b0656a51331cee5", size = 695730, upload-time = "2026-05-26T18:07:28.49Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/9b/b08d578bba73e25351152dfd3d6d21e81210a5fff1b6f26e56f33197c8f5/langgraph-1.2.2-py3-none-any.whl", hash = "sha256:0a851bf4ba5939c5474a2fd57e6b439b5315283e254e42943bd392c2d71a5e03", size = 236376, upload-time = "2026-05-26T18:07:26.577Z" }, +] + +[[package]] +name = "langgraph-checkpoint" +version = "4.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain-core" }, + { name = "ormsgpack" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/47/886af6f886f0bff2273164a45f008694e48a96ff3cd25ff0228f2aa9480e/langgraph_checkpoint-4.1.1.tar.gz", hash = "sha256:6c2bdb530c91f91d7d9c1bd100925d0fc4f498d418c17f3587d1526279482a25", size = 184020, upload-time = "2026-05-22T16:57:38.503Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/b4/71425e3e38be92611300b9cc5e46a5bf98ab23f5ea8a75b73d02a2f1413c/langgraph_checkpoint-4.1.1-py3-none-any.whl", hash = "sha256:25d29144b082827218e7bc3f1e9b0566a4bb007895cd6cc26f66a8428739f56e", size = 56212, upload-time = "2026-05-22T16:57:37.203Z" }, +] + +[[package]] +name = "langgraph-prebuilt" +version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, + { name = "langgraph-checkpoint" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/29/66/ed9b93f56bc17ef22d551892f0ac2b225a97fe0fcf23a511b857f70d590b/langgraph_prebuilt-1.1.0.tar.gz", hash = "sha256:3c579cf6eed2d17f9c157c2d0fcaddcd8688524e7022d3b22b37a3bf4589d528", size = 178833, upload-time = "2026-05-12T03:37:49.332Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/43/3fe1a700b8490ed02679cdbbc8c915eb23a092faf496c9c1118abcd10be3/langgraph_prebuilt-1.1.0-py3-none-any.whl", hash = "sha256:51e311747d755b751d5c6b39b0c1446124d3a7643d2515017e6714b323508fc9", size = 41043, upload-time = "2026-05-12T03:37:48.007Z" }, +] + +[[package]] +name = "langgraph-sdk" +version = "0.3.15" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, + { name = "orjson" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/11/43/dcda8fd25f0b19cb2835f2f6bb67f26ad58634f04ac2d8eae00526b0fa55/langchain_text_splitters-0.3.11.tar.gz", hash = "sha256:7a50a04ada9a133bbabb80731df7f6ddac51bc9f1b9cab7fa09304d71d38a6cc", size = 46458, upload-time = "2025-08-31T23:02:58.316Z" } +sdist = { url = "https://files.pythonhosted.org/packages/66/af/cdd4d6f3c05b3c1112ed3f12ef830faf15951b21d22cbc622a4becbbe25c/langgraph_sdk-0.3.15.tar.gz", hash = "sha256:29e805003d2c6e296823dd71992610976fd0428cefaa8b3304fd91f2247037de", size = 201924, upload-time = "2026-05-22T16:54:27.678Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/58/0d/41a51b40d24ff0384ec4f7ab8dd3dcea8353c05c973836b5e289f1465d4f/langchain_text_splitters-0.3.11-py3-none-any.whl", hash = "sha256:cf079131166a487f1372c8ab5d0bfaa6c0a4291733d9c43a34a16ac9bcd6a393", size = 33845, upload-time = "2025-08-31T23:02:57.195Z" }, + { url = "https://files.pythonhosted.org/packages/be/a5/0196d9c05749c25bc198e4909d68c998bc3120297e14944921baf2f4c384/langgraph_sdk-0.3.15-py3-none-any.whl", hash = "sha256:3838773acf7456d158165385d49f48f1e856f28b56ccd99ea139a8f27004815d", size = 98166, upload-time = "2026-05-22T16:54:26.013Z" }, ] [[package]] name = "langsmith" -version = "0.7.32" +version = "0.8.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, @@ -1718,9 +1937,9 @@ dependencies = [ { name = "xxhash" }, { name = "zstandard" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2f/b4/a0b4a501bee6b8a741ce29f8c48155b132118483cddc6f9247735ddb38fa/langsmith-0.7.32.tar.gz", hash = "sha256:b59b8e106d0e4c4842e158229296086e2aa7c561e3f602acda73d3ad0062e915", size = 1184518, upload-time = "2026-04-15T23:42:41.885Z" } +sdist = { url = "https://files.pythonhosted.org/packages/17/eb/8883d1158c743d0aac350f09df7880714d27283497e8c80bb9fe3480f165/langsmith-0.8.5.tar.gz", hash = "sha256:3615243d99c12f4047f13042bdc05a373dce232d106a6511b3ca7b48c5af1c2c", size = 4462348, upload-time = "2026-05-15T21:31:41.093Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/62/bc/148f98ac7dad73ac5e1b1c985290079cfeeb9ba13d760a24f25002beb2c9/langsmith-0.7.32-py3-none-any.whl", hash = "sha256:e1fde928990c4c52f47dc5132708cec674355d9101723d564183e965f383bf5f", size = 378272, upload-time = "2026-04-15T23:42:39.905Z" }, + { url = "https://files.pythonhosted.org/packages/23/85/968c88a63e32a59b3e5c68afd2fe114ce0708a125db0be1a85efc25fb2ea/langsmith-0.8.5-py3-none-any.whl", hash = "sha256:efc779f9d450dcaf9d97bc8894f4926276509d6e730e05289af9a64debce06ae", size = 399564, upload-time = "2026-05-15T21:31:39.046Z" }, ] [[package]] @@ -1819,14 +2038,14 @@ wheels = [ [[package]] name = "markdown-it-py" -version = "4.0.0" +version = "4.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mdurl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, ] [[package]] @@ -1891,27 +2110,40 @@ wheels = [ ] [[package]] -name = "marshmallow" -version = "3.26.2" +name = "matplotlib-inline" +version = "0.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "packaging" }, + { name = "traitlets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/55/79/de6c16cc902f4fc372236926b0ce2ab7845268dcc30fb2fbb7f71b418631/marshmallow-3.26.2.tar.gz", hash = "sha256:bbe2adb5a03e6e3571b573f42527c6fe926e17467833660bebd11593ab8dfd57", size = 222095, upload-time = "2025-12-22T06:53:53.309Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/74/97e72a36efd4ae2bccb3463284300f8953f199b5ffbc04cbbb0ec78f74b1/matplotlib_inline-0.2.1.tar.gz", hash = "sha256:e1ee949c340d771fc39e241ea75683deb94762c8fa5f2927ec57c83c4dffa9fe", size = 8110, upload-time = "2025-10-23T09:00:22.126Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/be/2f/5108cb3ee4ba6501748c4908b908e55f42a5b66245b4cfe0c99326e1ef6e/marshmallow-3.26.2-py3-none-any.whl", hash = "sha256:013fa8a3c4c276c24d26d84ce934dc964e2aa794345a0f8c7e5a7191482c8a73", size = 50964, upload-time = "2025-12-22T06:53:51.801Z" }, + { url = "https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl", hash = "sha256:d56ce5156ba6085e00a9d54fead6ed29a9c47e215cd1bba2e976ef39f5710a76", size = 9516, upload-time = "2025-10-23T09:00:20.675Z" }, ] [[package]] -name = "matplotlib-inline" -version = "0.2.1" +name = "mcp" +version = "1.27.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "traitlets" }, + { name = "anyio" }, + { name = "httpx" }, + { name = "httpx-sse" }, + { name = "jsonschema" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "pyjwt", extra = ["crypto"] }, + { name = "python-multipart" }, + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "sse-starlette" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, + { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c7/74/97e72a36efd4ae2bccb3463284300f8953f199b5ffbc04cbbb0ec78f74b1/matplotlib_inline-0.2.1.tar.gz", hash = "sha256:e1ee949c340d771fc39e241ea75683deb94762c8fa5f2927ec57c83c4dffa9fe", size = 8110, upload-time = "2025-10-23T09:00:22.126Z" } +sdist = { url = "https://files.pythonhosted.org/packages/38/83/d1efe7c2980d8a3afa476f4e3d42d53dd54c0ab94c27bee5d755b45c8b73/mcp-1.27.1.tar.gz", hash = "sha256:0f47e1820f8f8f941466b39749eb1d1839a04caddca2bc60e9d46e8a99914924", size = 608458, upload-time = "2026-05-08T16:50:12.601Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl", hash = "sha256:d56ce5156ba6085e00a9d54fead6ed29a9c47e215cd1bba2e976ef39f5710a76", size = 9516, upload-time = "2025-10-23T09:00:20.675Z" }, + { url = "https://files.pythonhosted.org/packages/fd/73/42d9596facebdb533b7f0b86c1b0364ef350d1f8ba78b1052e8a58b48b65/mcp-1.27.1-py3-none-any.whl", hash = "sha256:1af3c4203b329430fde7a87b4fcb6392a041f5cb851fd68fc674016ab4e7c06f", size = 216260, upload-time = "2026-05-08T16:50:10.547Z" }, ] [[package]] @@ -1942,10 +2174,12 @@ wheels = [ [[package]] name = "minio-manager-service-client" version = "0.0.1" -source = { git = "https://github.com/BERDataLakehouse/minio_manager_service_client.git?rev=v0.0.17#30c4e526507626f8afc08be33301f8a576496288" } +source = { git = "https://github.com/BERDataLakehouse/minio_manager_service_client.git?rev=v0.0.23#1a1ab502f6c7f86e38672cce571fc5b4ea7ed8b9" } dependencies = [ + { name = "attrs" }, { name = "httpx" }, { name = "pydantic" }, + { name = "python-dateutil" }, ] [[package]] @@ -2068,67 +2302,80 @@ wheels = [ ] [[package]] -name = "mypy-extensions" -version = "1.1.0" +name = "nest-asyncio" +version = "1.6.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, + { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, ] [[package]] name = "numpy" -version = "2.4.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d7/9f/b8cef5bffa569759033adda9481211426f12f53299629b410340795c2514/numpy-2.4.4.tar.gz", hash = "sha256:2d390634c5182175533585cc89f3608a4682ccb173cc9bb940b2881c8d6f8fa0", size = 20731587, upload-time = "2026-03-29T13:22:01.298Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/14/1d/d0a583ce4fefcc3308806a749a536c201ed6b5ad6e1322e227ee4848979d/numpy-2.4.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:08f2e31ed5e6f04b118e49821397f12767934cfdd12a1ce86a058f91e004ee50", size = 16684933, upload-time = "2026-03-29T13:19:22.47Z" }, - { url = "https://files.pythonhosted.org/packages/c1/62/2b7a48fbb745d344742c0277f01286dead15f3f68e4f359fbfcf7b48f70f/numpy-2.4.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e823b8b6edc81e747526f70f71a9c0a07ac4e7ad13020aa736bb7c9d67196115", size = 14694532, upload-time = "2026-03-29T13:19:25.581Z" }, - { url = "https://files.pythonhosted.org/packages/e5/87/499737bfba066b4a3bebff24a8f1c5b2dee410b209bc6668c9be692580f0/numpy-2.4.4-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:4a19d9dba1a76618dd86b164d608566f393f8ec6ac7c44f0cc879011c45e65af", size = 5199661, upload-time = "2026-03-29T13:19:28.31Z" }, - { url = "https://files.pythonhosted.org/packages/cd/da/464d551604320d1491bc345efed99b4b7034143a85787aab78d5691d5a0e/numpy-2.4.4-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:d2a8490669bfe99a233298348acc2d824d496dee0e66e31b66a6022c2ad74a5c", size = 6547539, upload-time = "2026-03-29T13:19:30.97Z" }, - { url = "https://files.pythonhosted.org/packages/7d/90/8d23e3b0dafd024bf31bdec225b3bb5c2dbfa6912f8a53b8659f21216cbf/numpy-2.4.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45dbed2ab436a9e826e302fcdcbe9133f9b0006e5af7168afb8963a6520da103", size = 15668806, upload-time = "2026-03-29T13:19:33.887Z" }, - { url = "https://files.pythonhosted.org/packages/d1/73/a9d864e42a01896bb5974475438f16086be9ba1f0d19d0bb7a07427c4a8b/numpy-2.4.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c901b15172510173f5cb310eae652908340f8dede90fff9e3bf6c0d8dfd92f83", size = 16632682, upload-time = "2026-03-29T13:19:37.336Z" }, - { url = "https://files.pythonhosted.org/packages/34/fb/14570d65c3bde4e202a031210475ae9cde9b7686a2e7dc97ee67d2833b35/numpy-2.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:99d838547ace2c4aace6c4f76e879ddfe02bb58a80c1549928477862b7a6d6ed", size = 17019810, upload-time = "2026-03-29T13:19:40.963Z" }, - { url = "https://files.pythonhosted.org/packages/8a/77/2ba9d87081fd41f6d640c83f26fb7351e536b7ce6dd9061b6af5904e8e46/numpy-2.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0aec54fd785890ecca25a6003fd9a5aed47ad607bbac5cd64f836ad8666f4959", size = 18357394, upload-time = "2026-03-29T13:19:44.859Z" }, - { url = "https://files.pythonhosted.org/packages/a2/23/52666c9a41708b0853fa3b1a12c90da38c507a3074883823126d4e9d5b30/numpy-2.4.4-cp313-cp313-win32.whl", hash = "sha256:07077278157d02f65c43b1b26a3886bce886f95d20aabd11f87932750dfb14ed", size = 5959556, upload-time = "2026-03-29T13:19:47.661Z" }, - { url = "https://files.pythonhosted.org/packages/57/fb/48649b4971cde70d817cf97a2a2fdc0b4d8308569f1dd2f2611959d2e0cf/numpy-2.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:5c70f1cc1c4efbe316a572e2d8b9b9cc44e89b95f79ca3331553fbb63716e2bf", size = 12317311, upload-time = "2026-03-29T13:19:50.67Z" }, - { url = "https://files.pythonhosted.org/packages/ba/d8/11490cddd564eb4de97b4579ef6bfe6a736cc07e94c1598590ae25415e01/numpy-2.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:ef4059d6e5152fa1a39f888e344c73fdc926e1b2dd58c771d67b0acfbf2aa67d", size = 10222060, upload-time = "2026-03-29T13:19:54.229Z" }, - { url = "https://files.pythonhosted.org/packages/99/5d/dab4339177a905aad3e2221c915b35202f1ec30d750dd2e5e9d9a72b804b/numpy-2.4.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4bbc7f303d125971f60ec0aaad5e12c62d0d2c925f0ab1273debd0e4ba37aba5", size = 14822302, upload-time = "2026-03-29T13:19:57.585Z" }, - { url = "https://files.pythonhosted.org/packages/eb/e4/0564a65e7d3d97562ed6f9b0fd0fb0a6f559ee444092f105938b50043876/numpy-2.4.4-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:4d6d57903571f86180eb98f8f0c839fa9ebbfb031356d87f1361be91e433f5b7", size = 5327407, upload-time = "2026-03-29T13:20:00.601Z" }, - { url = "https://files.pythonhosted.org/packages/29/8d/35a3a6ce5ad371afa58b4700f1c820f8f279948cca32524e0a695b0ded83/numpy-2.4.4-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:4636de7fd195197b7535f231b5de9e4b36d2c440b6e566d2e4e4746e6af0ca93", size = 6647631, upload-time = "2026-03-29T13:20:02.855Z" }, - { url = "https://files.pythonhosted.org/packages/f4/da/477731acbd5a58a946c736edfdabb2ac5b34c3d08d1ba1a7b437fa0884df/numpy-2.4.4-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ad2e2ef14e0b04e544ea2fa0a36463f847f113d314aa02e5b402fdf910ef309e", size = 15727691, upload-time = "2026-03-29T13:20:06.004Z" }, - { url = "https://files.pythonhosted.org/packages/e6/db/338535d9b152beabeb511579598418ba0212ce77cf9718edd70262cc4370/numpy-2.4.4-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a285b3b96f951841799528cd1f4f01cd70e7e0204b4abebac9463eecfcf2a40", size = 16681241, upload-time = "2026-03-29T13:20:09.417Z" }, - { url = "https://files.pythonhosted.org/packages/e2/a9/ad248e8f58beb7a0219b413c9c7d8151c5d285f7f946c3e26695bdbbe2df/numpy-2.4.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f8474c4241bc18b750be2abea9d7a9ec84f46ef861dbacf86a4f6e043401f79e", size = 17085767, upload-time = "2026-03-29T13:20:13.126Z" }, - { url = "https://files.pythonhosted.org/packages/b5/1a/3b88ccd3694681356f70da841630e4725a7264d6a885c8d442a697e1146b/numpy-2.4.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4e874c976154687c1f71715b034739b45c7711bec81db01914770373d125e392", size = 18403169, upload-time = "2026-03-29T13:20:17.096Z" }, - { url = "https://files.pythonhosted.org/packages/c2/c9/fcfd5d0639222c6eac7f304829b04892ef51c96a75d479214d77e3ce6e33/numpy-2.4.4-cp313-cp313t-win32.whl", hash = "sha256:9c585a1790d5436a5374bac930dad6ed244c046ed91b2b2a3634eb2971d21008", size = 6083477, upload-time = "2026-03-29T13:20:20.195Z" }, - { url = "https://files.pythonhosted.org/packages/d5/e3/3938a61d1c538aaec8ed6fd6323f57b0c2d2d2219512434c5c878db76553/numpy-2.4.4-cp313-cp313t-win_amd64.whl", hash = "sha256:93e15038125dc1e5345d9b5b68aa7f996ec33b98118d18c6ca0d0b7d6198b7e8", size = 12457487, upload-time = "2026-03-29T13:20:22.946Z" }, - { url = "https://files.pythonhosted.org/packages/97/6a/7e345032cc60501721ef94e0e30b60f6b0bd601f9174ebd36389a2b86d40/numpy-2.4.4-cp313-cp313t-win_arm64.whl", hash = "sha256:0dfd3f9d3adbe2920b68b5cd3d51444e13a10792ec7154cd0a2f6e74d4ab3233", size = 10292002, upload-time = "2026-03-29T13:20:25.909Z" }, - { url = "https://files.pythonhosted.org/packages/6e/06/c54062f85f673dd5c04cbe2f14c3acb8c8b95e3384869bb8cc9bff8cb9df/numpy-2.4.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:f169b9a863d34f5d11b8698ead99febeaa17a13ca044961aa8e2662a6c7766a0", size = 16684353, upload-time = "2026-03-29T13:20:29.504Z" }, - { url = "https://files.pythonhosted.org/packages/4c/39/8a320264a84404c74cc7e79715de85d6130fa07a0898f67fb5cd5bd79908/numpy-2.4.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:2483e4584a1cb3092da4470b38866634bafb223cbcd551ee047633fd2584599a", size = 14704914, upload-time = "2026-03-29T13:20:33.547Z" }, - { url = "https://files.pythonhosted.org/packages/91/fb/287076b2614e1d1044235f50f03748f31fa287e3dbe6abeb35cdfa351eca/numpy-2.4.4-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:2d19e6e2095506d1736b7d80595e0f252d76b89f5e715c35e06e937679ea7d7a", size = 5210005, upload-time = "2026-03-29T13:20:36.45Z" }, - { url = "https://files.pythonhosted.org/packages/63/eb/fcc338595309910de6ecabfcef2419a9ce24399680bfb149421fa2df1280/numpy-2.4.4-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:6a246d5914aa1c820c9443ddcee9c02bec3e203b0c080349533fae17727dfd1b", size = 6544974, upload-time = "2026-03-29T13:20:39.014Z" }, - { url = "https://files.pythonhosted.org/packages/44/5d/e7e9044032a716cdfaa3fba27a8e874bf1c5f1912a1ddd4ed071bf8a14a6/numpy-2.4.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:989824e9faf85f96ec9c7761cd8d29c531ad857bfa1daa930cba85baaecf1a9a", size = 15684591, upload-time = "2026-03-29T13:20:42.146Z" }, - { url = "https://files.pythonhosted.org/packages/98/7c/21252050676612625449b4807d6b695b9ce8a7c9e1c197ee6216c8a65c7c/numpy-2.4.4-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:27a8d92cd10f1382a67d7cf4db7ce18341b66438bdd9f691d7b0e48d104c2a9d", size = 16637700, upload-time = "2026-03-29T13:20:46.204Z" }, - { url = "https://files.pythonhosted.org/packages/b1/29/56d2bbef9465db24ef25393383d761a1af4f446a1df9b8cded4fe3a5a5d7/numpy-2.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e44319a2953c738205bf3354537979eaa3998ed673395b964c1176083dd46252", size = 17035781, upload-time = "2026-03-29T13:20:50.242Z" }, - { url = "https://files.pythonhosted.org/packages/e3/2b/a35a6d7589d21f44cea7d0a98de5ddcbb3d421b2622a5c96b1edf18707c3/numpy-2.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e892aff75639bbef0d2a2cfd55535510df26ff92f63c92cd84ef8d4ba5a5557f", size = 18362959, upload-time = "2026-03-29T13:20:54.019Z" }, - { url = "https://files.pythonhosted.org/packages/64/c9/d52ec581f2390e0f5f85cbfd80fb83d965fc15e9f0e1aec2195faa142cde/numpy-2.4.4-cp314-cp314-win32.whl", hash = "sha256:1378871da56ca8943c2ba674530924bb8ca40cd228358a3b5f302ad60cf875fc", size = 6008768, upload-time = "2026-03-29T13:20:56.912Z" }, - { url = "https://files.pythonhosted.org/packages/fa/22/4cc31a62a6c7b74a8730e31a4274c5dc80e005751e277a2ce38e675e4923/numpy-2.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:715d1c092715954784bc79e1174fc2a90093dc4dc84ea15eb14dad8abdcdeb74", size = 12449181, upload-time = "2026-03-29T13:20:59.548Z" }, - { url = "https://files.pythonhosted.org/packages/70/2e/14cda6f4d8e396c612d1bf97f22958e92148801d7e4f110cabebdc0eef4b/numpy-2.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:2c194dd721e54ecad9ad387c1d35e63dce5c4450c6dc7dd5611283dda239aabb", size = 10496035, upload-time = "2026-03-29T13:21:02.524Z" }, - { url = "https://files.pythonhosted.org/packages/b1/e8/8fed8c8d848d7ecea092dc3469643f9d10bc3a134a815a3b033da1d2039b/numpy-2.4.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2aa0613a5177c264ff5921051a5719d20095ea586ca88cc802c5c218d1c67d3e", size = 14824958, upload-time = "2026-03-29T13:21:05.671Z" }, - { url = "https://files.pythonhosted.org/packages/05/1a/d8007a5138c179c2bf33ef44503e83d70434d2642877ee8fbb230e7c0548/numpy-2.4.4-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:42c16925aa5a02362f986765f9ebabf20de75cdefdca827d14315c568dcab113", size = 5330020, upload-time = "2026-03-29T13:21:08.635Z" }, - { url = "https://files.pythonhosted.org/packages/99/64/ffb99ac6ae93faf117bcbd5c7ba48a7f45364a33e8e458545d3633615dda/numpy-2.4.4-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:874f200b2a981c647340f841730fc3a2b54c9d940566a3c4149099591e2c4c3d", size = 6650758, upload-time = "2026-03-29T13:21:10.949Z" }, - { url = "https://files.pythonhosted.org/packages/6e/6e/795cc078b78a384052e73b2f6281ff7a700e9bf53bcce2ee579d4f6dd879/numpy-2.4.4-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c9b39d38a9bd2ae1becd7eac1303d031c5c110ad31f2b319c6e7d98b135c934d", size = 15729948, upload-time = "2026-03-29T13:21:14.047Z" }, - { url = "https://files.pythonhosted.org/packages/5f/86/2acbda8cc2af5f3d7bfc791192863b9e3e19674da7b5e533fded124d1299/numpy-2.4.4-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b268594bccac7d7cf5844c7732e3f20c50921d94e36d7ec9b79e9857694b1b2f", size = 16679325, upload-time = "2026-03-29T13:21:17.561Z" }, - { url = "https://files.pythonhosted.org/packages/bc/59/cafd83018f4aa55e0ac6fa92aa066c0a1877b77a615ceff1711c260ffae8/numpy-2.4.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ac6b31e35612a26483e20750126d30d0941f949426974cace8e6b5c58a3657b0", size = 17084883, upload-time = "2026-03-29T13:21:21.106Z" }, - { url = "https://files.pythonhosted.org/packages/f0/85/a42548db84e65ece46ab2caea3d3f78b416a47af387fcbb47ec28e660dc2/numpy-2.4.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8e3ed142f2728df44263aaf5fb1f5b0b99f4070c553a0d7f033be65338329150", size = 18403474, upload-time = "2026-03-29T13:21:24.828Z" }, - { url = "https://files.pythonhosted.org/packages/ed/ad/483d9e262f4b831000062e5d8a45e342166ec8aaa1195264982bca267e62/numpy-2.4.4-cp314-cp314t-win32.whl", hash = "sha256:dddbbd259598d7240b18c9d87c56a9d2fb3b02fe266f49a7c101532e78c1d871", size = 6155500, upload-time = "2026-03-29T13:21:28.205Z" }, - { url = "https://files.pythonhosted.org/packages/c7/03/2fc4e14c7bd4ff2964b74ba90ecb8552540b6315f201df70f137faa5c589/numpy-2.4.4-cp314-cp314t-win_amd64.whl", hash = "sha256:a7164afb23be6e37ad90b2f10426149fd75aee07ca55653d2aa41e66c4ef697e", size = 12637755, upload-time = "2026-03-29T13:21:31.107Z" }, - { url = "https://files.pythonhosted.org/packages/58/78/548fb8e07b1a341746bfbecb32f2c268470f45fa028aacdbd10d9bc73aab/numpy-2.4.4-cp314-cp314t-win_arm64.whl", hash = "sha256:ba203255017337d39f89bdd58417f03c4426f12beed0440cfd933cb15f8669c7", size = 10566643, upload-time = "2026-03-29T13:21:34.339Z" }, +version = "2.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/82/bdab26d7438c6791ca31b7c024ca37c1eab8b726ba236129005cd4a06e45/numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0", size = 16684648, upload-time = "2026-05-18T23:34:29.41Z" }, + { url = "https://files.pythonhosted.org/packages/1b/30/a80189bcc7f5e4258b3fbc3968d909d1756f54d023299ecc39ad6fdb9ef8/numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb", size = 14693902, upload-time = "2026-05-18T23:34:33.013Z" }, + { url = "https://files.pythonhosted.org/packages/97/12/70b5d0d7c15e1ebb8a6a84a8caa1d19e181d84fb58bb6d70aca29099dec1/numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f", size = 5198992, upload-time = "2026-05-18T23:34:36.132Z" }, + { url = "https://files.pythonhosted.org/packages/ba/8c/ebd2a8f8a83541f8d38cc5667e8c2b69cecfd30da6e45693e8158857d44b/numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3", size = 6546944, upload-time = "2026-05-18T23:34:38.484Z" }, + { url = "https://files.pythonhosted.org/packages/bb/c5/7b863a97a91671a0338f4253bd3b5a3d3852f0692dae91711c9f4a10e787/numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b", size = 15669392, upload-time = "2026-05-18T23:34:41.257Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9d/3584b9984ca4c047aea75214ce1a4c4c73d849bd71b604264b7f5653f8a8/numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089", size = 16633220, upload-time = "2026-05-18T23:34:45.075Z" }, + { url = "https://files.pythonhosted.org/packages/05/ae/7c67fba23bd98caec7c99261f3a16072ade14813486b0282cb29846de832/numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a", size = 17020800, upload-time = "2026-05-18T23:34:49.065Z" }, + { url = "https://files.pythonhosted.org/packages/d9/5d/3b6725cb31d983c5e66916f5d36f6d7e5521129e4c4404d64f918292a5b6/numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605", size = 18357600, upload-time = "2026-05-18T23:34:52.709Z" }, + { url = "https://files.pythonhosted.org/packages/f7/da/2ccc6c2fe8898dee01d90c75c5f5f914a23daf99e3e0f59516a08760c8b5/numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91", size = 5961134, upload-time = "2026-05-18T23:34:55.618Z" }, + { url = "https://files.pythonhosted.org/packages/b5/cd/9cc4dc876fb065d5c220aae4d5e14826b2715331bb7618ce1fb07a679d99/numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359", size = 12318598, upload-time = "2026-05-18T23:34:58.928Z" }, + { url = "https://files.pythonhosted.org/packages/39/1e/c0bcba1f8694116485fe28fd1be698c278fcda4141c5b0e53a2aed8b12a8/numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778", size = 10222272, upload-time = "2026-05-18T23:35:02.167Z" }, + { url = "https://files.pythonhosted.org/packages/63/6d/cc5619247c8f4204e507f5883528372e4ac4bb189e579fb859a12e480b1f/numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1", size = 14821197, upload-time = "2026-05-18T23:35:05.468Z" }, + { url = "https://files.pythonhosted.org/packages/00/58/f1c39161c87d9e9bed660f1ed4bafc0e403d5ec9650b6dd77aead07d489b/numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe", size = 5326287, upload-time = "2026-05-18T23:35:08.693Z" }, + { url = "https://files.pythonhosted.org/packages/af/57/3917ab0fd97f271a8694513581b8a36c655f111c446852c302f04ccdb6fc/numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997", size = 6646763, upload-time = "2026-05-18T23:35:11.459Z" }, + { url = "https://files.pythonhosted.org/packages/eb/0f/037e64c494b67581ae18193d770adef354c41f3f2c8ebf865602d949bf8f/numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20", size = 15728070, upload-time = "2026-05-18T23:35:14.79Z" }, + { url = "https://files.pythonhosted.org/packages/21/a6/5d2bae9c9542eb4df16dc9c46dc79c186e9bad53805dfa5399a6023c6db0/numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d", size = 16681752, upload-time = "2026-05-18T23:35:18.836Z" }, + { url = "https://files.pythonhosted.org/packages/92/14/23d1dfb410ae362cd59ce53e936b1513d545eb40db3949ced632e19a459e/numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67", size = 17086024, upload-time = "2026-05-18T23:35:22.52Z" }, + { url = "https://files.pythonhosted.org/packages/4b/6e/23595a2c642cdf3bc567877064bdd7f91c8b0038a4453cf2daf7248eafe9/numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd", size = 18403398, upload-time = "2026-05-18T23:35:26.398Z" }, + { url = "https://files.pythonhosted.org/packages/8a/90/0ac3bc947217e66dec77e7cbc6a1979d1af70b6461b82f620d3bccd5e4c8/numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab", size = 6084971, upload-time = "2026-05-18T23:35:29.387Z" }, + { url = "https://files.pythonhosted.org/packages/77/71/5673e351671a1d2bd6063b91b44f70c0affea7d1516fa7a6572941ba4aa1/numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75", size = 12458532, upload-time = "2026-05-18T23:35:32.175Z" }, + { url = "https://files.pythonhosted.org/packages/3f/88/19d3503c5046e688f049274b27a3ef3d771152fa80d3ba3d01a3dff61abe/numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd", size = 10291881, upload-time = "2026-05-18T23:35:35.465Z" }, + { url = "https://files.pythonhosted.org/packages/f8/91/3ab2044d05fd16d343c5ac2e69b127f1b2854040dd20b193257c78028bd3/numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079", size = 16683458, upload-time = "2026-05-18T23:35:38.353Z" }, + { url = "https://files.pythonhosted.org/packages/8e/62/764ce66fa4147ae6d73071a3abf804ffe606f174618697c571acdf26a7c9/numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7", size = 14704559, upload-time = "2026-05-18T23:35:42.14Z" }, + { url = "https://files.pythonhosted.org/packages/60/61/23f27c172f022e04025b7dc2367f4d63c1a398120607ec896228649a6f48/numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5", size = 5209716, upload-time = "2026-05-18T23:35:45.377Z" }, + { url = "https://files.pythonhosted.org/packages/03/71/21cf70dc6ea3e3acb95fc53a265b2fc248b981f0194ceb5b475271b8809d/numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096", size = 6543947, upload-time = "2026-05-18T23:35:47.926Z" }, + { url = "https://files.pythonhosted.org/packages/d5/91/64288395ee1799bd2e0b04a305dce9666da90c961e1f3fe982a05ee1c036/numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b", size = 15685197, upload-time = "2026-05-18T23:35:50.863Z" }, + { url = "https://files.pythonhosted.org/packages/f3/eb/ebffaa97dc55502df69584a8f0dcf07f69a3e0b3e2323670a2722db9aa39/numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8", size = 16638245, upload-time = "2026-05-18T23:35:54.752Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0b/54f9da33128d7e350fab89c7455902eeae70349ee52bddb448dc4a576f45/numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402", size = 17036587, upload-time = "2026-05-18T23:35:58.355Z" }, + { url = "https://files.pythonhosted.org/packages/b6/f0/fdebc1052db1cc37c64beb22072d67cd6d1c71adca1299f53dec2b5e20d3/numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb", size = 18363226, upload-time = "2026-05-18T23:36:02.845Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b4/298628d98c72b57e57f7165ae6a481a1deaf6f3c28262a6e4c739c275930/numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1", size = 6010196, upload-time = "2026-05-18T23:36:05.92Z" }, + { url = "https://files.pythonhosted.org/packages/df/ac/46de6dda46478f7942f839e094970be2d4a861e005c4b3bf07c92e291a09/numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261", size = 12450334, upload-time = "2026-05-18T23:36:09.107Z" }, + { url = "https://files.pythonhosted.org/packages/78/92/b8b798ac784102c0da830d2257d59358e3d3d90d1e2b3f2575dad976c5cf/numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6", size = 10495678, upload-time = "2026-05-18T23:36:12.766Z" }, + { url = "https://files.pythonhosted.org/packages/30/34/ec28d1aa8115971537c01469ab2011ee96827930f0a124de1000cc2a7ed7/numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a", size = 14823672, upload-time = "2026-05-18T23:36:16.473Z" }, + { url = "https://files.pythonhosted.org/packages/16/bd/f6d1fede4e54e8042a7ff97bb495510f3c220f94bcd9e8b228e87c92cc0d/numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e", size = 5328731, upload-time = "2026-05-18T23:36:19.767Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f0/e105b9e2fd728a9910103884decd6951d9dd73896b914a98d9a231de02ee/numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e", size = 6649805, upload-time = "2026-05-18T23:36:22.266Z" }, + { url = "https://files.pythonhosted.org/packages/82/dd/1206a7ca6ab15e3f02069707ca96222e202af681bb73756da7527f3cb837/numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43", size = 15730496, upload-time = "2026-05-18T23:36:25.713Z" }, + { url = "https://files.pythonhosted.org/packages/51/e7/38d3ea825dcab85a591734decb2f6c67caa7c8367d374df1a1c3842f9b07/numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e", size = 16679616, upload-time = "2026-05-18T23:36:29.652Z" }, + { url = "https://files.pythonhosted.org/packages/93/b7/caabfdf53edf663e0b4eb74d7d405d83baef09eb5e83bcd32d601d72b93e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895", size = 17085145, upload-time = "2026-05-18T23:36:33.449Z" }, + { url = "https://files.pythonhosted.org/packages/f9/45/68d7c33a6bcf3e5aa3bdbd57a367e6f615286dfd6482f97e8ffeb734306e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4", size = 18403813, upload-time = "2026-05-18T23:36:37.369Z" }, + { url = "https://files.pythonhosted.org/packages/9c/50/0753655aa844c99cd9e018aacf76f130f1bd81d881bb74bc0aef5d73a8ba/numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063", size = 6156982, upload-time = "2026-05-18T23:36:40.817Z" }, + { url = "https://files.pythonhosted.org/packages/b2/d4/7c67becf668f973cb490cec3e98dfd799d866f9c989a54d355672cfa0db6/numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627", size = 12638908, upload-time = "2026-05-18T23:36:43.996Z" }, + { url = "https://files.pythonhosted.org/packages/43/bb/e1c71a4295b1b1d1393d50dbb4f2a36283c6859d9d3892e84f00ec5a91d5/numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66", size = 10565867, upload-time = "2026-05-18T23:36:47.114Z" }, +] + +[[package]] +name = "ollama" +version = "0.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/72/5f12423b6b39ca8430fbe56f77fcf4ef60f63067c7c4a2e30e200ed9ec16/ollama-0.6.2.tar.gz", hash = "sha256:936d55daa684f474364c098611c933626f8d6c7d67065c5b7ae0c477b508b07f", size = 53145, upload-time = "2026-04-29T21:21:15.018Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/ab/d6722beeb2d10f7a3b9ff49375708904fde18f82b5609a0bc4aeb5996a4d/ollama-0.6.2-py3-none-any.whl", hash = "sha256:3ad7daab28e5a973445c36a73882a3ef698c2ebb00e21e308652741577509f7d", size = 15115, upload-time = "2026-04-29T21:21:13.794Z" }, ] [[package]] name = "openai" -version = "2.32.0" +version = "2.38.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -2140,9 +2387,9 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ed/59/bdcc6b759b8c42dd73afaf5bf8f902c04b37987a5514dbc1c64dba390fef/openai-2.32.0.tar.gz", hash = "sha256:c54b27a9e4cb8d51f0dd94972ffd1a04437efeb259a9e60d8922b8bd26fe55e0", size = 693286, upload-time = "2026-04-15T22:28:19.434Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8f/12/cfa322c5f5dd8fa21aab9a7a8e979e7a11123800f86ca8d82eb68a83d213/openai-2.38.0.tar.gz", hash = "sha256:798694c6cf74145541fda94325b6f8f72d8e1fd0262cc137c8d728177a6a4ce3", size = 772764, upload-time = "2026-05-21T21:23:42.105Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/c1/d6e64ccd0536bf616556f0cad2b6d94a8125f508d25cfd814b1d2db4e2f1/openai-2.32.0-py3-none-any.whl", hash = "sha256:4dcc9badeb4bf54ad0d187453742f290226d30150890b7890711bda4f32f192f", size = 1162570, upload-time = "2026-04-15T22:28:17.714Z" }, + { url = "https://files.pythonhosted.org/packages/0a/bf/ccff9be562e24207716d04ef9dc931c76aff0c89a7265da43e2104d7fe06/openai-2.38.0-py3-none-any.whl", hash = "sha256:ec6661c57b2dcc47414a767e6e3335c7ed3d19c9696999283a3c82e95c756a3c", size = 1344910, upload-time = "2026-05-21T21:23:39.636Z" }, ] [[package]] @@ -2156,40 +2403,70 @@ wheels = [ [[package]] name = "orjson" -version = "3.11.8" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9d/1b/2024d06792d0779f9dbc51531b61c24f76c75b9f4ce05e6f3377a1814cea/orjson-3.11.8.tar.gz", hash = "sha256:96163d9cdc5a202703e9ad1b9ae757d5f0ca62f4fa0cc93d1f27b0e180cc404e", size = 5603832, upload-time = "2026-03-31T16:16:27.878Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/66/7f/95fba509bb2305fab0073558f1e8c3a2ec4b2afe58ed9fcb7d3b8beafe94/orjson-3.11.8-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:3f23426851d98478c8970da5991f84784a76682213cd50eb73a1da56b95239dc", size = 229180, upload-time = "2026-03-31T16:15:36.426Z" }, - { url = "https://files.pythonhosted.org/packages/f6/9d/b237215c743ca073697d759b5503abd2cb8a0d7b9c9e21f524bcf176ab66/orjson-3.11.8-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:ebaed4cef74a045b83e23537b52ef19a367c7e3f536751e355a2a394f8648559", size = 128754, upload-time = "2026-03-31T16:15:38.049Z" }, - { url = "https://files.pythonhosted.org/packages/42/3d/27d65b6d11e63f133781425f132807aef793ed25075fec686fc8e46dd528/orjson-3.11.8-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97c8f5d3b62380b70c36ffacb2a356b7c6becec86099b177f73851ba095ef623", size = 131877, upload-time = "2026-03-31T16:15:39.484Z" }, - { url = "https://files.pythonhosted.org/packages/dd/cc/faee30cd8f00421999e40ef0eba7332e3a625ce91a58200a2f52c7fef235/orjson-3.11.8-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:436c4922968a619fb7fef1ccd4b8b3a76c13b67d607073914d675026e911a65c", size = 130361, upload-time = "2026-03-31T16:15:41.274Z" }, - { url = "https://files.pythonhosted.org/packages/5c/bb/a6c55896197f97b6d4b4e7c7fd77e7235517c34f5d6ad5aadd43c54c6d7c/orjson-3.11.8-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1ab359aff0436d80bfe8a23b46b5fea69f1e18aaf1760a709b4787f1318b317f", size = 135521, upload-time = "2026-03-31T16:15:42.758Z" }, - { url = "https://files.pythonhosted.org/packages/9c/7c/ca3a3525aa32ff636ebb1778e77e3587b016ab2edb1b618b36ba96f8f2c0/orjson-3.11.8-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f89b6d0b3a8d81e1929d3ab3d92bbc225688bd80a770c49432543928fe09ac55", size = 146862, upload-time = "2026-03-31T16:15:44.341Z" }, - { url = "https://files.pythonhosted.org/packages/3c/0c/18a9d7f18b5edd37344d1fd5be17e94dc652c67826ab749c6e5948a78112/orjson-3.11.8-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:29c009e7a2ca9ad0ed1376ce20dd692146a5d9fe4310848904b6b4fee5c5c137", size = 132847, upload-time = "2026-03-31T16:15:46.368Z" }, - { url = "https://files.pythonhosted.org/packages/23/91/7e722f352ad67ca573cee44de2a58fb810d0f4eb4e33276c6a557979fd8a/orjson-3.11.8-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:705b895b781b3e395c067129d8551655642dfe9437273211d5404e87ac752b53", size = 133637, upload-time = "2026-03-31T16:15:48.123Z" }, - { url = "https://files.pythonhosted.org/packages/af/04/32845ce13ac5bd1046ddb02ac9432ba856cc35f6d74dde95864fe0ad5523/orjson-3.11.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:88006eda83858a9fdf73985ce3804e885c2befb2f506c9a3723cdeb5a2880e3e", size = 141906, upload-time = "2026-03-31T16:15:49.626Z" }, - { url = "https://files.pythonhosted.org/packages/02/5e/c551387ddf2d7106d9039369862245c85738b828844d13b99ccb8d61fd06/orjson-3.11.8-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:55120759e61309af7fcf9e961c6f6af3dde5921cdb3ee863ef63fd9db126cae6", size = 423722, upload-time = "2026-03-31T16:15:51.176Z" }, - { url = "https://files.pythonhosted.org/packages/00/a3/ecfe62434096f8a794d4976728cb59bcfc4a643977f21c2040545d37eb4c/orjson-3.11.8-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:98bdc6cb889d19bed01de46e67574a2eab61f5cc6b768ed50e8ac68e9d6ffab6", size = 147801, upload-time = "2026-03-31T16:15:52.939Z" }, - { url = "https://files.pythonhosted.org/packages/18/6d/0dce10b9f6643fdc59d99333871a38fa5a769d8e2fc34a18e5d2bfdee900/orjson-3.11.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:708c95f925a43ab9f34625e45dcdadf09ec8a6e7b664a938f2f8d5650f6c090b", size = 136460, upload-time = "2026-03-31T16:15:54.431Z" }, - { url = "https://files.pythonhosted.org/packages/01/d6/6dde4f31842d87099238f1f07b459d24edc1a774d20687187443ab044191/orjson-3.11.8-cp313-cp313-win32.whl", hash = "sha256:01c4e5a6695dc09098f2e6468a251bc4671c50922d4d745aff1a0a33a0cf5b8d", size = 131956, upload-time = "2026-03-31T16:15:56.081Z" }, - { url = "https://files.pythonhosted.org/packages/c1/f9/4e494a56e013db957fb77186b818b916d4695b8fa2aa612364974160e91b/orjson-3.11.8-cp313-cp313-win_amd64.whl", hash = "sha256:c154a35dd1330707450bb4d4e7dd1f17fa6f42267a40c1e8a1daa5e13719b4b8", size = 127410, upload-time = "2026-03-31T16:15:57.54Z" }, - { url = "https://files.pythonhosted.org/packages/57/7f/803203d00d6edb6e9e7eef421d4e1adbb5ea973e40b3533f3cfd9aeb374e/orjson-3.11.8-cp313-cp313-win_arm64.whl", hash = "sha256:4861bde57f4d253ab041e374f44023460e60e71efaa121f3c5f0ed457c3a701e", size = 127338, upload-time = "2026-03-31T16:15:59.106Z" }, - { url = "https://files.pythonhosted.org/packages/6d/35/b01910c3d6b85dc882442afe5060cbf719c7d1fc85749294beda23d17873/orjson-3.11.8-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:ec795530a73c269a55130498842aaa762e4a939f6ce481a7e986eeaa790e9da4", size = 229171, upload-time = "2026-03-31T16:16:00.651Z" }, - { url = "https://files.pythonhosted.org/packages/c2/56/c9ec97bd11240abef39b9e5d99a15462809c45f677420fd148a6c5e6295e/orjson-3.11.8-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:c492a0e011c0f9066e9ceaa896fbc5b068c54d365fea5f3444b697ee01bc8625", size = 128746, upload-time = "2026-03-31T16:16:02.673Z" }, - { url = "https://files.pythonhosted.org/packages/3b/e4/66d4f30a90de45e2f0cbd9623588e8ae71eef7679dbe2ae954ed6d66a41f/orjson-3.11.8-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:883206d55b1bd5f5679ad5e6ddd3d1a5e3cac5190482927fdb8c78fb699193b5", size = 131867, upload-time = "2026-03-31T16:16:04.342Z" }, - { url = "https://files.pythonhosted.org/packages/19/30/2a645fc9286b928675e43fa2a3a16fb7b6764aa78cc719dc82141e00f30b/orjson-3.11.8-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5774c1fdcc98b2259800b683b19599c133baeb11d60033e2095fd9d4667b82db", size = 124664, upload-time = "2026-03-31T16:16:05.837Z" }, - { url = "https://files.pythonhosted.org/packages/db/44/77b9a86d84a28d52ba3316d77737f6514e17118119ade3f91b639e859029/orjson-3.11.8-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ac7381c83dd3d4a6347e6635950aa448f54e7b8406a27c7ecb4a37e9f1ae08b", size = 129701, upload-time = "2026-03-31T16:16:07.407Z" }, - { url = "https://files.pythonhosted.org/packages/b3/ea/eff3d9bfe47e9bc6969c9181c58d9f71237f923f9c86a2d2f490cd898c82/orjson-3.11.8-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:14439063aebcb92401c11afc68ee4e407258d2752e62d748b6942dad20d2a70d", size = 141202, upload-time = "2026-03-31T16:16:09.48Z" }, - { url = "https://files.pythonhosted.org/packages/52/c8/90d4b4c60c84d62068d0cf9e4d8f0a4e05e76971d133ac0c60d818d4db20/orjson-3.11.8-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fa72e71977bff96567b0f500fc5bfd2fdf915f34052c782a4c6ebbdaa97aa858", size = 127194, upload-time = "2026-03-31T16:16:11.02Z" }, - { url = "https://files.pythonhosted.org/packages/8d/c7/ea9e08d1f0ba981adffb629811148b44774d935171e7b3d780ae43c4c254/orjson-3.11.8-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7679bc2f01bb0d219758f1a5f87bb7c8a81c0a186824a393b366876b4948e14f", size = 133639, upload-time = "2026-03-31T16:16:13.434Z" }, - { url = "https://files.pythonhosted.org/packages/6c/8c/ddbbfd6ba59453c8fc7fe1d0e5983895864e264c37481b2a791db635f046/orjson-3.11.8-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:14f7b8fcb35ef403b42fa5ecfa4ed032332a91f3dc7368fbce4184d59e1eae0d", size = 141914, upload-time = "2026-03-31T16:16:14.955Z" }, - { url = "https://files.pythonhosted.org/packages/4e/31/dbfbefec9df060d34ef4962cd0afcb6fa7a9ec65884cb78f04a7859526c3/orjson-3.11.8-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:c2bdf7b2facc80b5e34f48a2d557727d5c5c57a8a450de122ae81fa26a81c1bc", size = 423800, upload-time = "2026-03-31T16:16:16.594Z" }, - { url = "https://files.pythonhosted.org/packages/87/cf/f74e9ae9803d4ab46b163494adba636c6d7ea955af5cc23b8aaa94cfd528/orjson-3.11.8-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ccd7ba1b0605813a0715171d39ec4c314cb97a9c85893c2c5c0c3a3729df38bf", size = 147837, upload-time = "2026-03-31T16:16:18.585Z" }, - { url = "https://files.pythonhosted.org/packages/64/e6/9214f017b5db85e84e68602792f742e5dc5249e963503d1b356bee611e01/orjson-3.11.8-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cdbc8c9c02463fef4d3c53a9ba3336d05496ec8e1f1c53326a1e4acc11f5c600", size = 136441, upload-time = "2026-03-31T16:16:20.151Z" }, - { url = "https://files.pythonhosted.org/packages/24/dd/3590348818f58f837a75fb969b04cdf187ae197e14d60b5e5a794a38b79d/orjson-3.11.8-cp314-cp314-win32.whl", hash = "sha256:0b57f67710a8cd459e4e54eb96d5f77f3624eba0c661ba19a525807e42eccade", size = 131983, upload-time = "2026-03-31T16:16:21.823Z" }, - { url = "https://files.pythonhosted.org/packages/3f/0f/b6cb692116e05d058f31ceee819c70f097fa9167c82f67fabe7516289abc/orjson-3.11.8-cp314-cp314-win_amd64.whl", hash = "sha256:735e2262363dcbe05c35e3a8869898022af78f89dde9e256924dc02e99fe69ca", size = 127396, upload-time = "2026-03-31T16:16:23.685Z" }, - { url = "https://files.pythonhosted.org/packages/c0/d1/facb5b5051fabb0ef9d26c6544d87ef19a939a9a001198655d0d891062dd/orjson-3.11.8-cp314-cp314-win_arm64.whl", hash = "sha256:6ccdea2c213cf9f3d9490cbd5d427693c870753df41e6cb375bd79bcbafc8817", size = 127330, upload-time = "2026-03-31T16:16:25.496Z" }, +version = "3.11.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/0c/964746fcafbd16f8ff53219ad9f6b412b34f345c75f384ad434ceaadb538/orjson-3.11.9.tar.gz", hash = "sha256:4fef17e1f8722c11587a6ef18e35902450221da0028e65dbaaa543619e68e48f", size = 5599163, upload-time = "2026-05-06T15:11:08.309Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/33/93fcc25907235c344ae73122f8a4e01d2d393ef062b4af7d2e2487a32c37/orjson-3.11.9-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:4bab1b2d6141fe7b32ae71dac905666ece4f94936efbfb13d55bb7739a3a6021", size = 228458, upload-time = "2026-05-06T15:10:20.079Z" }, + { url = "https://files.pythonhosted.org/packages/8f/27/b1e6dadb3c080313c03fdd8067b85e6a0460c7d8d6a1c3984ef77b904e4d/orjson-3.11.9-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:844417969855fc7a41be124aafe83dc424592a7f77cd4501900c67307122b92c", size = 128368, upload-time = "2026-05-06T15:10:21.549Z" }, + { url = "https://files.pythonhosted.org/packages/21/0f/c9ede0bf052f6b4051e64a7d4fa91b725cccf8321a6a786e86eb03519f00/orjson-3.11.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffe02797b5e9f3a9d8292ddcd289b474ad13e81ad83cd1891a240811f1d2cb81", size = 132070, upload-time = "2026-05-06T15:10:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/fd/26/d398e28048dc18205bbe812f2c88cb9b40313db2470778e25964796458fe/orjson-3.11.9-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e4eed3b200023042814d2fc8a5d2e880f13b52e1ed2485e83da4f3962f7dc1a", size = 127892, upload-time = "2026-05-06T15:10:24.714Z" }, + { url = "https://files.pythonhosted.org/packages/66/60/52b0054c4c700d5aa7fc5b7ca96917400d8f061307778578e67a10e25852/orjson-3.11.9-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aff7da9952a5ad1cef8e68017724d96c7b9a66e99e91d6252e1b133d67a7b10", size = 135217, upload-time = "2026-05-06T15:10:26.084Z" }, + { url = "https://files.pythonhosted.org/packages/d5/97/1e3dc2b2a28b7b2528f403d2fc1d79ec5f39af3bc143ab65d3ec26426385/orjson-3.11.9-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4d4e98d6f3b8afed8bc8cd9718ec0cdf46661826beefb53fe8eafb37f2bf0362", size = 145980, upload-time = "2026-05-06T15:10:28.062Z" }, + { url = "https://files.pythonhosted.org/packages/fc/39/31fbfe7850f2de32dee7e7e5c09f26d403ab01e440ac96001c6b01ad3c99/orjson-3.11.9-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a81d52442a7c99b3662333235b3adf96a1715864658b35bb797212be7bddb97", size = 132738, upload-time = "2026-05-06T15:10:29.727Z" }, + { url = "https://files.pythonhosted.org/packages/a1/08/dca0082dd2a194acb93e5457e73455388e2e2ca464a2672449a9ddbb679d/orjson-3.11.9-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e39364e726a8fff737309aff059ff67d8a8c8d5b677be7bb49a8b3e84b7e218", size = 134033, upload-time = "2026-05-06T15:10:31.152Z" }, + { url = "https://files.pythonhosted.org/packages/11/d4/5bdb0626801230139987385554c5d4c42255218ac906525bf4347f22cd95/orjson-3.11.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4fd66214623f1b17501df9f0543bef0b833979ab5b6ded1e1d123222866aa8c9", size = 141492, upload-time = "2026-05-06T15:10:32.641Z" }, + { url = "https://files.pythonhosted.org/packages/fa/88/a21fb53b3ede6703aede6dce4710ed4111e5b201cfa6bbff5e544f9d47d7/orjson-3.11.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:8ecc30f10465fa1e0ce13fd01d9e22c316e5053a719a8d915d4545a09a5ff677", size = 415087, upload-time = "2026-05-06T15:10:34.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/57/1b30daf70f0d8180e9a73cefbfbdd99e4bf19eb020466502b01fba7e0e50/orjson-3.11.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:97db4c94a7db398a5bd636273324f0b3fd58b350bbbac8bb380ceb825a9b40f4", size = 148031, upload-time = "2026-05-06T15:10:36.358Z" }, + { url = "https://files.pythonhosted.org/packages/04/83/45fbb6d962e260807f99441db9613cee868ceda4baceda59b3720a563f97/orjson-3.11.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9f78cf8fec5bd627f4082b8dfeac7871b43d7f3274904492a43dab39f18a19a0", size = 136915, upload-time = "2026-05-06T15:10:38.013Z" }, + { url = "https://files.pythonhosted.org/packages/5f/cc/2d10025f9056d376e4127ec05a5808b218d46f035fdc08178a5411b34250/orjson-3.11.9-cp313-cp313-win32.whl", hash = "sha256:d4087e5c0209a0a8efe4de3303c234b9c44d1174161dcd851e8eea07c7560b32", size = 131613, upload-time = "2026-05-06T15:10:39.569Z" }, + { url = "https://files.pythonhosted.org/packages/67/bd/2775ff28bfe883b9aa1ff348300542eb2ef1ee18d8ae0e3a49846817a865/orjson-3.11.9-cp313-cp313-win_amd64.whl", hash = "sha256:051b102c93b4f634e89f3866b07b9a9a98915ada541f4ec30f177067b2694979", size = 127086, upload-time = "2026-05-06T15:10:41.262Z" }, + { url = "https://files.pythonhosted.org/packages/91/2b/d26799e580939e32a7da9a39531bc9e58e15ca32ffaa6a8cb3e9bb0d22cd/orjson-3.11.9-cp313-cp313-win_arm64.whl", hash = "sha256:cce9127885941bd28f080cecf1f1d288336b7e0d812c345b08be88b572796254", size = 126696, upload-time = "2026-05-06T15:10:42.651Z" }, + { url = "https://files.pythonhosted.org/packages/8e/eb/5da01e356015aee6ecfa1187ced87aef51364e306f5e695dd52719bf0e78/orjson-3.11.9-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b6ef1979adc4bc243523f1a2ba91418030a8e29b0a99cbe7e0e2d6807d4dce6e", size = 228465, upload-time = "2026-05-06T15:10:44.097Z" }, + { url = "https://files.pythonhosted.org/packages/64/62/3e0e0c14c957133bcd855395c62b55ed4e3b0af23ffea11b032cb1dcbdb1/orjson-3.11.9-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:f36b7f32c7c0db4a719f1fc5824db4a9c6f8bd1a354debb91faf26ebf3a4c71e", size = 128364, upload-time = "2026-05-06T15:10:45.839Z" }, + { url = "https://files.pythonhosted.org/packages/5a/5a/07d8aa117211a8ed7630bda80c8c0b14d04e0f8dcf99bcf49656e4a710eb/orjson-3.11.9-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08f4d8ebb44925c794e535b2bebc507cebf32209df81de22ae285fb0d8d66de0", size = 132063, upload-time = "2026-05-06T15:10:47.267Z" }, + { url = "https://files.pythonhosted.org/packages/d6/ec/4acaf21483e18aa945be74a474c74b434f284b549f275a0a39b9f98956e9/orjson-3.11.9-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6cc7923789694fd58f001cbcac7e47abc13af4d560ebbfcf3b41a8b1a0748124", size = 122356, upload-time = "2026-05-06T15:10:48.765Z" }, + { url = "https://files.pythonhosted.org/packages/13/d8/5f0555e7638801323b7a75850f92e7dfa891bc84fe27a1ba4449170d1200/orjson-3.11.9-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea5c46eb2d3af39e806b986f4b09d5c2706a1f5afde3cbf7544ce6616127173c", size = 129592, upload-time = "2026-05-06T15:10:50.13Z" }, + { url = "https://files.pythonhosted.org/packages/b6/30/ed9860412a3603ceb3c5955bfd72d28b9d0e7ba6ed81add14f83d7114236/orjson-3.11.9-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f5d89a2ed90731df3be64bab0aa44f78bff39fdc9d71c291f4a8023aa46425b7", size = 140491, upload-time = "2026-05-06T15:10:51.582Z" }, + { url = "https://files.pythonhosted.org/packages/d0/17/adc514dea7ac7c505527febf884934b815d34f0c7b8693c1a8b39c5c4a57/orjson-3.11.9-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:25e4aed0312d292c09f61af25bba34e0b2c88546041472b09088c39a4d828af1", size = 127309, upload-time = "2026-05-06T15:10:53.329Z" }, + { url = "https://files.pythonhosted.org/packages/76/3e/c0b690253f0b82d86e99949af13533363acfb5432ecb5d53dd5b3bce9c34/orjson-3.11.9-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaea64f3f467d22e70eeed68bdccb3bc4f83f650446c4a03c59f2cba28a108db", size = 134030, upload-time = "2026-05-06T15:10:54.988Z" }, + { url = "https://files.pythonhosted.org/packages/c1/7a/bc82a0bb25e9faaf92dc4d9ef002732efc09737706af83e346788641d4a7/orjson-3.11.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a028425d1b440c5d92a6be1e1a020739dfe67ea87d96c6dbe828c1b30041728b", size = 141482, upload-time = "2026-05-06T15:10:56.663Z" }, + { url = "https://files.pythonhosted.org/packages/01/55/e69188b939f77d5d32a9833745ace31ea5ccae3ab613a1ec185d3cd2c4fb/orjson-3.11.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5b192c6cf397e4455b11523c5cf2b18ed084c1bbd61b6c0926344d2129481972", size = 415178, upload-time = "2026-05-06T15:10:58.446Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1a/b8a5a7ac527e80b9cb11d51e3f6689b709279183264b9ec5c7bc680bb8b5/orjson-3.11.9-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea407d4ccf5891d667d045fecae97a7a1e5e87b3b97f97ae1803c2e741130be0", size = 148089, upload-time = "2026-05-06T15:11:00.441Z" }, + { url = "https://files.pythonhosted.org/packages/97/4e/00503f64204bf859b37213a63927028f30fb6268cd8677fb0a5ad48155e1/orjson-3.11.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f63aaf97afd9f6dec5b1a68e1b8da12bfccb4cb9a9a65c3e0b6c847849e7586", size = 136921, upload-time = "2026-05-06T15:11:02.176Z" }, + { url = "https://files.pythonhosted.org/packages/0d/ba/a23b82a0a8d0ed7bed4e5f5035aae751cad4ff6a1e8d2ecd14d8860f5929/orjson-3.11.9-cp314-cp314-win32.whl", hash = "sha256:e30ab17845bb9fa54ccf67fa4f9f5282652d54faa6d17452f47d0f369d038673", size = 131638, upload-time = "2026-05-06T15:11:03.696Z" }, + { url = "https://files.pythonhosted.org/packages/f3/c3/0c6798456bade745c75c452342dabacce5798196483e77e643be1f53877d/orjson-3.11.9-cp314-cp314-win_amd64.whl", hash = "sha256:32ef5f4283a3be81913947d19608eacb7c6608026851123790cd9cc8982af34b", size = 127078, upload-time = "2026-05-06T15:11:05.123Z" }, + { url = "https://files.pythonhosted.org/packages/16/21/5a3f1e8913103b703a436a5664238e5b965ec392b555fe68943ea3691e6b/orjson-3.11.9-cp314-cp314-win_arm64.whl", hash = "sha256:eebdbdeef0094e4f5aefa20dcd4eb2368ab5e7a3b4edea27f1e7b2892e009cf9", size = 126687, upload-time = "2026-05-06T15:11:06.602Z" }, +] + +[[package]] +name = "ormsgpack" +version = "1.12.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/12/0c/f1761e21486942ab9bb6feaebc610fa074f7c5e496e6962dea5873348077/ormsgpack-1.12.2.tar.gz", hash = "sha256:944a2233640273bee67521795a73cf1e959538e0dfb7ac635505010455e53b33", size = 39031, upload-time = "2026-01-18T20:55:28.023Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/29/bb0eba3288c0449efbb013e9c6f58aea79cf5cb9ee1921f8865f04c1a9d7/ormsgpack-1.12.2-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:5ea60cb5f210b1cfbad8c002948d73447508e629ec375acb82910e3efa8ff355", size = 378661, upload-time = "2026-01-18T20:55:57.765Z" }, + { url = "https://files.pythonhosted.org/packages/6e/31/5efa31346affdac489acade2926989e019e8ca98129658a183e3add7af5e/ormsgpack-1.12.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3601f19afdbea273ed70b06495e5794606a8b690a568d6c996a90d7255e51c1", size = 203194, upload-time = "2026-01-18T20:56:08.252Z" }, + { url = "https://files.pythonhosted.org/packages/eb/56/d0087278beef833187e0167f8527235ebe6f6ffc2a143e9de12a98b1ce87/ormsgpack-1.12.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29a9f17a3dac6054c0dce7925e0f4995c727f7c41859adf9b5572180f640d172", size = 210778, upload-time = "2026-01-18T20:55:17.694Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a2/072343e1413d9443e5a252a8eb591c2d5b1bffbe5e7bfc78c069361b92eb/ormsgpack-1.12.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39c1bd2092880e413902910388be8715f70b9f15f20779d44e673033a6146f2d", size = 212592, upload-time = "2026-01-18T20:55:32.747Z" }, + { url = "https://files.pythonhosted.org/packages/a2/8b/a0da3b98a91d41187a63b02dda14267eefc2a74fcb43cc2701066cf1510e/ormsgpack-1.12.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:50b7249244382209877deedeee838aef1542f3d0fc28b8fe71ca9d7e1896a0d7", size = 387164, upload-time = "2026-01-18T20:55:40.853Z" }, + { url = "https://files.pythonhosted.org/packages/19/bb/6d226bc4cf9fc20d8eb1d976d027a3f7c3491e8f08289a2e76abe96a65f3/ormsgpack-1.12.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:5af04800d844451cf102a59c74a841324868d3f1625c296a06cc655c542a6685", size = 482516, upload-time = "2026-01-18T20:55:42.033Z" }, + { url = "https://files.pythonhosted.org/packages/fb/f1/bb2c7223398543dedb3dbf8bb93aaa737b387de61c5feaad6f908841b782/ormsgpack-1.12.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cec70477d4371cd524534cd16472d8b9cc187e0e3043a8790545a9a9b296c258", size = 425539, upload-time = "2026-01-18T20:55:24.727Z" }, + { url = "https://files.pythonhosted.org/packages/7b/e8/0fb45f57a2ada1fed374f7494c8cd55e2f88ccd0ab0a669aa3468716bf5f/ormsgpack-1.12.2-cp313-cp313-win_amd64.whl", hash = "sha256:21f4276caca5c03a818041d637e4019bc84f9d6ca8baa5ea03e5cc8bf56140e9", size = 117459, upload-time = "2026-01-18T20:55:56.876Z" }, + { url = "https://files.pythonhosted.org/packages/7a/d4/0cfeea1e960d550a131001a7f38a5132c7ae3ebde4c82af1f364ccc5d904/ormsgpack-1.12.2-cp313-cp313-win_arm64.whl", hash = "sha256:baca4b6773d20a82e36d6fd25f341064244f9f86a13dead95dd7d7f996f51709", size = 111577, upload-time = "2026-01-18T20:55:43.605Z" }, + { url = "https://files.pythonhosted.org/packages/94/16/24d18851334be09c25e87f74307c84950f18c324a4d3c0b41dabdbf19c29/ormsgpack-1.12.2-cp314-cp314-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:bc68dd5915f4acf66ff2010ee47c8906dc1cf07399b16f4089f8c71733f6e36c", size = 378717, upload-time = "2026-01-18T20:55:26.164Z" }, + { url = "https://files.pythonhosted.org/packages/b5/a2/88b9b56f83adae8032ac6a6fa7f080c65b3baf9b6b64fd3d37bd202991d4/ormsgpack-1.12.2-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:46d084427b4132553940070ad95107266656cb646ea9da4975f85cb1a6676553", size = 203183, upload-time = "2026-01-18T20:55:18.815Z" }, + { url = "https://files.pythonhosted.org/packages/a9/80/43e4555963bf602e5bdc79cbc8debd8b6d5456c00d2504df9775e74b450b/ormsgpack-1.12.2-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c010da16235806cf1d7bc4c96bf286bfa91c686853395a299b3ddb49499a3e13", size = 210814, upload-time = "2026-01-18T20:55:33.973Z" }, + { url = "https://files.pythonhosted.org/packages/78/e1/7cfbf28de8bca6efe7e525b329c31277d1b64ce08dcba723971c241a9d60/ormsgpack-1.12.2-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18867233df592c997154ff942a6503df274b5ac1765215bceba7a231bea2745d", size = 212634, upload-time = "2026-01-18T20:55:28.634Z" }, + { url = "https://files.pythonhosted.org/packages/95/f8/30ae5716e88d792a4e879debee195653c26ddd3964c968594ddef0a3cc7e/ormsgpack-1.12.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b009049086ddc6b8f80c76b3955df1aa22a5fbd7673c525cd63bf91f23122ede", size = 387139, upload-time = "2026-01-18T20:56:02.013Z" }, + { url = "https://files.pythonhosted.org/packages/dc/81/aee5b18a3e3a0e52f718b37ab4b8af6fae0d9d6a65103036a90c2a8ffb5d/ormsgpack-1.12.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1dcc17d92b6390d4f18f937cf0b99054824a7815818012ddca925d6e01c2e49e", size = 482578, upload-time = "2026-01-18T20:55:35.117Z" }, + { url = "https://files.pythonhosted.org/packages/bd/17/71c9ba472d5d45f7546317f467a5fc941929cd68fb32796ca3d13dcbaec2/ormsgpack-1.12.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f04b5e896d510b07c0ad733d7fce2d44b260c5e6c402d272128f8941984e4285", size = 425539, upload-time = "2026-01-18T20:56:04.009Z" }, + { url = "https://files.pythonhosted.org/packages/2e/a6/ac99cd7fe77e822fed5250ff4b86fa66dd4238937dd178d2299f10b69816/ormsgpack-1.12.2-cp314-cp314-win_amd64.whl", hash = "sha256:ae3aba7eed4ca7cb79fd3436eddd29140f17ea254b91604aa1eb19bfcedb990f", size = 117493, upload-time = "2026-01-18T20:56:07.343Z" }, + { url = "https://files.pythonhosted.org/packages/3a/67/339872846a1ae4592535385a1c1f93614138566d7af094200c9c3b45d1e5/ormsgpack-1.12.2-cp314-cp314-win_arm64.whl", hash = "sha256:118576ea6006893aea811b17429bfc561b4778fad393f5f538c84af70b01260c", size = 111579, upload-time = "2026-01-18T20:55:21.161Z" }, + { url = "https://files.pythonhosted.org/packages/49/c2/6feb972dc87285ad381749d3882d8aecbde9f6ecf908dd717d33d66df095/ormsgpack-1.12.2-cp314-cp314t-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:7121b3d355d3858781dc40dafe25a32ff8a8242b9d80c692fd548a4b1f7fd3c8", size = 378721, upload-time = "2026-01-18T20:55:52.12Z" }, + { url = "https://files.pythonhosted.org/packages/a3/9a/900a6b9b413e0f8a471cf07830f9cf65939af039a362204b36bd5b581d8b/ormsgpack-1.12.2-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ee766d2e78251b7a63daf1cddfac36a73562d3ddef68cacfb41b2af64698033", size = 203170, upload-time = "2026-01-18T20:55:44.469Z" }, + { url = "https://files.pythonhosted.org/packages/87/4c/27a95466354606b256f24fad464d7c97ab62bce6cc529dd4673e1179b8fb/ormsgpack-1.12.2-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:292410a7d23de9b40444636b9b8f1e4e4b814af7f1ef476e44887e52a123f09d", size = 212816, upload-time = "2026-01-18T20:55:23.501Z" }, + { url = "https://files.pythonhosted.org/packages/73/cd/29cee6007bddf7a834e6cd6f536754c0535fcb939d384f0f37a38b1cddb8/ormsgpack-1.12.2-cp314-cp314t-win_amd64.whl", hash = "sha256:837dd316584485b72ef451d08dd3e96c4a11d12e4963aedb40e08f89685d8ec2", size = 117232, upload-time = "2026-01-18T20:55:45.448Z" }, ] [[package]] @@ -2203,46 +2480,46 @@ wheels = [ [[package]] name = "pandas" -version = "3.0.2" +version = "3.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, { name = "python-dateutil" }, { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/da/99/b342345300f13440fe9fe385c3c481e2d9a595ee3bab4d3219247ac94e9a/pandas-3.0.2.tar.gz", hash = "sha256:f4753e73e34c8d83221ba58f232433fca2748be8b18dbca02d242ed153945043", size = 4645855, upload-time = "2026-03-31T06:48:30.816Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/ca/3e639a1ea6fcd0617ca4e8ca45f62a74de33a56ae6cd552735470b22c8d3/pandas-3.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b5918ba197c951dec132b0c5929a00c0bf05d5942f590d3c10a807f6e15a57d3", size = 10321105, upload-time = "2026-03-31T06:46:57.327Z" }, - { url = "https://files.pythonhosted.org/packages/0b/77/dbc82ff2fb0e63c6564356682bf201edff0ba16c98630d21a1fb312a8182/pandas-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d606a041c89c0a474a4702d532ab7e73a14fe35c8d427b972a625c8e46373668", size = 9864088, upload-time = "2026-03-31T06:46:59.935Z" }, - { url = "https://files.pythonhosted.org/packages/5c/2b/341f1b04bbca2e17e13cd3f08c215b70ef2c60c5356ef1e8c6857449edc7/pandas-3.0.2-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:710246ba0616e86891b58ab95f2495143bb2bc83ab6b06747c74216f583a6ac9", size = 10369066, upload-time = "2026-03-31T06:47:02.792Z" }, - { url = "https://files.pythonhosted.org/packages/12/c5/cbb1ffefb20a93d3f0e1fdcda699fb84976210d411b008f97f48bf6ce27e/pandas-3.0.2-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5d3cfe227c725b1f3dff4278b43d8c784656a42a9325b63af6b1492a8232209e", size = 10876780, upload-time = "2026-03-31T06:47:06.205Z" }, - { url = "https://files.pythonhosted.org/packages/98/fe/2249ae5e0a69bd0ddf17353d0a5d26611d70970111f5b3600cdc8be883e7/pandas-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c3b723df9087a9a9a840e263ebd9f88b64a12075d1bf2ea401a5a42f254f084d", size = 11375181, upload-time = "2026-03-31T06:47:09.383Z" }, - { url = "https://files.pythonhosted.org/packages/de/64/77a38b09e70b6464883b8d7584ab543e748e42c1b5d337a2ee088e0df741/pandas-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a3096110bf9eac0070b7208465f2740e2d8a670d5cb6530b5bb884eca495fd39", size = 11928899, upload-time = "2026-03-31T06:47:12.686Z" }, - { url = "https://files.pythonhosted.org/packages/5e/52/42855bf626868413f761addd574acc6195880ae247a5346477a4361c3acb/pandas-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:07a10f5c36512eead51bc578eb3354ad17578b22c013d89a796ab5eee90cd991", size = 9746574, upload-time = "2026-03-31T06:47:15.64Z" }, - { url = "https://files.pythonhosted.org/packages/88/39/21304ae06a25e8bf9fc820d69b29b2c495b2ae580d1e143146c309941760/pandas-3.0.2-cp313-cp313-win_arm64.whl", hash = "sha256:5fdbfa05931071aba28b408e59226186b01eb5e92bea2ab78b65863ca3228d84", size = 9047156, upload-time = "2026-03-31T06:47:18.595Z" }, - { url = "https://files.pythonhosted.org/packages/72/20/7defa8b27d4f330a903bb68eea33be07d839c5ea6bdda54174efcec0e1d2/pandas-3.0.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:dbc20dea3b9e27d0e66d74c42b2d0c1bed9c2ffe92adea33633e3bedeb5ac235", size = 10756238, upload-time = "2026-03-31T06:47:22.012Z" }, - { url = "https://files.pythonhosted.org/packages/e9/95/49433c14862c636afc0e9b2db83ff16b3ad92959364e52b2955e44c8e94c/pandas-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b75c347eff42497452116ce05ef461822d97ce5b9ff8df6edacb8076092c855d", size = 10408520, upload-time = "2026-03-31T06:47:25.197Z" }, - { url = "https://files.pythonhosted.org/packages/3b/f8/462ad2b5881d6b8ec8e5f7ed2ea1893faa02290d13870a1600fe72ad8efc/pandas-3.0.2-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d1478075142e83a5571782ad007fb201ed074bdeac7ebcc8890c71442e96adf7", size = 10324154, upload-time = "2026-03-31T06:47:28.097Z" }, - { url = "https://files.pythonhosted.org/packages/0a/65/d1e69b649cbcddda23ad6e4c40ef935340f6f652a006e5cbc3555ac8adb3/pandas-3.0.2-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5880314e69e763d4c8b27937090de570f1fb8d027059a7ada3f7f8e98bdcb677", size = 10714449, upload-time = "2026-03-31T06:47:30.85Z" }, - { url = "https://files.pythonhosted.org/packages/47/a4/85b59bc65b8190ea3689882db6cdf32a5003c0ccd5a586c30fdcc3ffc4fc/pandas-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b5329e26898896f06035241a626d7c335daa479b9bbc82be7c2742d048e41172", size = 11338475, upload-time = "2026-03-31T06:47:34.026Z" }, - { url = "https://files.pythonhosted.org/packages/1e/c4/bc6966c6e38e5d9478b935272d124d80a589511ed1612a5d21d36f664c68/pandas-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:81526c4afd31971f8b62671442a4b2b51e0aa9acc3819c9f0f12a28b6fcf85f1", size = 11786568, upload-time = "2026-03-31T06:47:36.941Z" }, - { url = "https://files.pythonhosted.org/packages/e8/74/09298ca9740beed1d3504e073d67e128aa07e5ca5ca2824b0c674c0b8676/pandas-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:7cadd7e9a44ec13b621aec60f9150e744cfc7a3dd32924a7e2f45edff31823b0", size = 10488652, upload-time = "2026-03-31T06:47:40.612Z" }, - { url = "https://files.pythonhosted.org/packages/bb/40/c6ea527147c73b24fc15c891c3fcffe9c019793119c5742b8784a062c7db/pandas-3.0.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:db0dbfd2a6cdf3770aa60464d50333d8f3d9165b2f2671bcc299b72de5a6677b", size = 10326084, upload-time = "2026-03-31T06:47:43.834Z" }, - { url = "https://files.pythonhosted.org/packages/95/25/bdb9326c3b5455f8d4d3549fce7abcf967259de146fe2cf7a82368141948/pandas-3.0.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0555c5882688a39317179ab4a0ed41d3ebc8812ab14c69364bbee8fb7a3f6288", size = 9914146, upload-time = "2026-03-31T06:47:46.67Z" }, - { url = "https://files.pythonhosted.org/packages/8d/77/3a227ff3337aa376c60d288e1d61c5d097131d0ac71f954d90a8f369e422/pandas-3.0.2-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:01f31a546acd5574ef77fe199bc90b55527c225c20ccda6601cf6b0fd5ed597c", size = 10444081, upload-time = "2026-03-31T06:47:49.681Z" }, - { url = "https://files.pythonhosted.org/packages/15/88/3cdd54fa279341afa10acf8d2b503556b1375245dccc9315659f795dd2e9/pandas-3.0.2-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:deeca1b5a931fdf0c2212c8a659ade6d3b1edc21f0914ce71ef24456ca7a6535", size = 10897535, upload-time = "2026-03-31T06:47:53.033Z" }, - { url = "https://files.pythonhosted.org/packages/06/9d/98cc7a7624f7932e40f434299260e2917b090a579d75937cb8a57b9d2de3/pandas-3.0.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0f48afd9bb13300ffb5a3316973324c787054ba6665cda0da3fbd67f451995db", size = 11446992, upload-time = "2026-03-31T06:47:56.193Z" }, - { url = "https://files.pythonhosted.org/packages/9a/cd/19ff605cc3760e80602e6826ddef2824d8e7050ed80f2e11c4b079741dc3/pandas-3.0.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6c4d8458b97a35717b62469a4ea0e85abd5ed8687277f5ccfc67f8a5126f8c53", size = 11968257, upload-time = "2026-03-31T06:47:59.137Z" }, - { url = "https://files.pythonhosted.org/packages/db/60/aba6a38de456e7341285102bede27514795c1eaa353bc0e7638b6b785356/pandas-3.0.2-cp314-cp314-win_amd64.whl", hash = "sha256:b35d14bb5d8285d9494fe93815a9e9307c0876e10f1e8e89ac5b88f728ec8dcf", size = 9865893, upload-time = "2026-03-31T06:48:02.038Z" }, - { url = "https://files.pythonhosted.org/packages/08/71/e5ec979dd2e8a093dacb8864598c0ff59a0cee0bbcdc0bfec16a51684d4f/pandas-3.0.2-cp314-cp314-win_arm64.whl", hash = "sha256:63d141b56ef686f7f0d714cfb8de4e320475b86bf4b620aa0b7da89af8cbdbbb", size = 9188644, upload-time = "2026-03-31T06:48:05.045Z" }, - { url = "https://files.pythonhosted.org/packages/f1/6c/7b45d85db19cae1eb524f2418ceaa9d85965dcf7b764ed151386b7c540f0/pandas-3.0.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:140f0cffb1fa2524e874dde5b477d9defe10780d8e9e220d259b2c0874c89d9d", size = 10776246, upload-time = "2026-03-31T06:48:07.789Z" }, - { url = "https://files.pythonhosted.org/packages/a8/3e/7b00648b086c106e81766f25322b48aa8dfa95b55e621dbdf2fdd413a117/pandas-3.0.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ae37e833ff4fed0ba352f6bdd8b73ba3ab3256a85e54edfd1ab51ae40cca0af8", size = 10424801, upload-time = "2026-03-31T06:48:10.897Z" }, - { url = "https://files.pythonhosted.org/packages/da/6e/558dd09a71b53b4008e7fc8a98ec6d447e9bfb63cdaeea10e5eb9b2dabe8/pandas-3.0.2-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d888a5c678a419a5bb41a2a93818e8ed9fd3172246555c0b37b7cc27027effd", size = 10345643, upload-time = "2026-03-31T06:48:13.7Z" }, - { url = "https://files.pythonhosted.org/packages/be/e3/921c93b4d9a280409451dc8d07b062b503bbec0531d2627e73a756e99a82/pandas-3.0.2-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b444dc64c079e84df91baa8bf613d58405645461cabca929d9178f2cd392398d", size = 10743641, upload-time = "2026-03-31T06:48:16.659Z" }, - { url = "https://files.pythonhosted.org/packages/56/ca/fd17286f24fa3b4d067965d8d5d7e14fe557dd4f979a0b068ac0deaf8228/pandas-3.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4544c7a54920de8eeacaa1466a6b7268ecfbc9bc64ab4dbb89c6bbe94d5e0660", size = 11361993, upload-time = "2026-03-31T06:48:19.475Z" }, - { url = "https://files.pythonhosted.org/packages/e4/a5/2f6ed612056819de445a433ca1f2821ac3dab7f150d569a59e9cc105de1d/pandas-3.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:734be7551687c00fbd760dc0522ed974f82ad230d4a10f54bf51b80d44a08702", size = 11815274, upload-time = "2026-03-31T06:48:22.695Z" }, - { url = "https://files.pythonhosted.org/packages/00/2f/b622683e99ec3ce00b0854bac9e80868592c5b051733f2cf3a868e5fea26/pandas-3.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:57a07209bebcbcf768d2d13c9b78b852f9a15978dac41b9e6421a81ad4cdd276", size = 10888530, upload-time = "2026-03-31T06:48:25.806Z" }, - { url = "https://files.pythonhosted.org/packages/cb/2b/f8434233fab2bd66a02ec014febe4e5adced20e2693e0e90a07d118ed30e/pandas-3.0.2-cp314-cp314t-win_arm64.whl", hash = "sha256:5371b72c2d4d415d08765f32d689217a43227484e81b2305b52076e328f6f482", size = 9455341, upload-time = "2026-03-31T06:48:28.418Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/90/62d8302883c44308c477e222c3daf7c813a34c8e96985882fbd53d964352/pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa", size = 10331071, upload-time = "2026-05-11T18:52:58.838Z" }, + { url = "https://files.pythonhosted.org/packages/7f/ae/6a6493c783a101f165e4356953ba3c74d6f77f0042fa7d753da9dfbb640c/pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7", size = 9875690, upload-time = "2026-05-11T18:53:01.431Z" }, + { url = "https://files.pythonhosted.org/packages/62/7c/5df8e9f56c69a2769fbe9382a5ef8f2658c007e376434e1e2cbb57ad895f/pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8", size = 10381634, upload-time = "2026-05-11T18:53:04.393Z" }, + { url = "https://files.pythonhosted.org/packages/99/68/1237369725aa617bb358263d535803e3053fdbc593513ec5ed9c9896b5b6/pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a", size = 10891243, upload-time = "2026-05-11T18:53:07.643Z" }, + { url = "https://files.pythonhosted.org/packages/25/93/77d108e8af7222b4a503ebde0e30215b1c2e4f8e53a526431890f22d5586/pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb", size = 11388659, upload-time = "2026-05-11T18:53:10.634Z" }, + { url = "https://files.pythonhosted.org/packages/d0/bd/eff5b4399f332ac386c853f6cd2bd3fa2ca0061b9f36ecd9c4d7c4265649/pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2", size = 11942880, upload-time = "2026-05-11T18:53:13.536Z" }, + { url = "https://files.pythonhosted.org/packages/2c/20/559ace4200982c3887d0b86bfd0d856a2143ef8ddab63cc07934951a964c/pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44", size = 9757091, upload-time = "2026-05-11T18:53:16.306Z" }, + { url = "https://files.pythonhosted.org/packages/3a/66/69055a09fe200f29f922a3eeec4804611900b95f52d932ece3393c3c0c19/pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e", size = 9057282, upload-time = "2026-05-11T18:53:18.768Z" }, + { url = "https://files.pythonhosted.org/packages/57/0e/efe801b0e6811e8e650cd21b7f2608e30f08a7067e2bf6e8752b0d56ee3c/pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d", size = 10767016, upload-time = "2026-05-11T18:53:21.227Z" }, + { url = "https://files.pythonhosted.org/packages/ea/dc/eb55135a1d5f0f0519f28da1f609a206d2cad1f9c35c32d51e38dd7261ae/pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066", size = 10420210, upload-time = "2026-05-11T18:53:23.982Z" }, + { url = "https://files.pythonhosted.org/packages/c6/3e/b1d5d955ce33ffecb407465a60bc32769d74fcf68224b7ae67ae11d4dea4/pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd", size = 10336126, upload-time = "2026-05-11T18:53:26.731Z" }, + { url = "https://files.pythonhosted.org/packages/f5/76/a01261711ab60a22d71b862f0de20e4c504bf80457270ad8cb42110f6abc/pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085", size = 10728051, upload-time = "2026-05-11T18:53:29.125Z" }, + { url = "https://files.pythonhosted.org/packages/e9/21/ea191195e587b18cf682e97f433f81b2d0fbe341380e80a3e0d6e4403c8e/pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870", size = 11350796, upload-time = "2026-05-11T18:53:32.056Z" }, + { url = "https://files.pythonhosted.org/packages/64/69/f0eaaf54939f0e8c6768fd06be9af2cef9b36048b96dfb9e1b2c685a807e/pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f", size = 11799741, upload-time = "2026-05-11T18:53:34.985Z" }, + { url = "https://files.pythonhosted.org/packages/45/a4/865e0e510cae5fc2194de4db28be638952de942571ba9125934fd9c01d47/pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13", size = 10499958, upload-time = "2026-05-11T18:53:37.857Z" }, + { url = "https://files.pythonhosted.org/packages/86/54/effdcc3c0ff7a08037889200e148ebe94c16c4f653be078c7b3675955df1/pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac", size = 10336065, upload-time = "2026-05-11T18:53:41.099Z" }, + { url = "https://files.pythonhosted.org/packages/68/10/bf2d6738d72748b961a3751ab89522d58c54efc36a8e1a12161216cd45cf/pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f", size = 9926101, upload-time = "2026-05-11T18:53:43.515Z" }, + { url = "https://files.pythonhosted.org/packages/ae/e9/e35cf11c8a136e757b956f5f0efdcaa50aecde85ea055f1898dfc68262f3/pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb", size = 10457553, upload-time = "2026-05-11T18:53:46.394Z" }, + { url = "https://files.pythonhosted.org/packages/58/3b/1cdec6772bdbaf7b25dab360c59f03cadf05492dd724c6540af905389b07/pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a", size = 10914065, upload-time = "2026-05-11T18:53:49.134Z" }, + { url = "https://files.pythonhosted.org/packages/c4/c2/1ef644445fcd72e3627bceec77e3560636f87ddce4ed841afe76b83b5bf9/pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360", size = 11459188, upload-time = "2026-05-11T18:53:52.527Z" }, + { url = "https://files.pythonhosted.org/packages/7e/49/4d8d4f42cbc9c4adc7a1870f269c02cbd6cd40d059622c06fb298addcbad/pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76", size = 11982966, upload-time = "2026-05-11T18:53:55.043Z" }, + { url = "https://files.pythonhosted.org/packages/38/55/792619469bab9882d8bbd5865d45a72f6478762d04a9af4bf0d08c503e95/pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5", size = 9876755, upload-time = "2026-05-11T18:53:58.067Z" }, + { url = "https://files.pythonhosted.org/packages/2a/af/33c469653b0ba03b50c3a98192d4c07f0c75c66b263ceb097fce0ee97d31/pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977", size = 9198658, upload-time = "2026-05-11T18:54:00.733Z" }, + { url = "https://files.pythonhosted.org/packages/a2/fa/b8c257bd76b8bd060c3a9151c1fca05e9b9c5e3af5d0f549c0356f6d143d/pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04", size = 10787242, upload-time = "2026-05-11T18:54:03.564Z" }, + { url = "https://files.pythonhosted.org/packages/54/eb/f19206ffb0bf1919002969aa448b4702c6594845156a6f8050674855aac3/pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6", size = 10436369, upload-time = "2026-05-11T18:54:06.311Z" }, + { url = "https://files.pythonhosted.org/packages/fd/24/c7c39fb4fe22b71a0c2d78bf0c585c600092d85f94f086d2b3b2f6ca27e2/pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c", size = 10358306, upload-time = "2026-05-11T18:54:09.085Z" }, + { url = "https://files.pythonhosted.org/packages/16/ec/dd2a9eb7fa1204df88c0864164e35b228ac581062ac612ba0a67fd812e4c/pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028", size = 10758394, upload-time = "2026-05-11T18:54:11.956Z" }, + { url = "https://files.pythonhosted.org/packages/95/6e/00c61ea8e85b4f6d8d35e11852a1a4998fc7fafc91c6a602d1cc9c972d64/pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d", size = 11375717, upload-time = "2026-05-11T18:54:14.539Z" }, + { url = "https://files.pythonhosted.org/packages/31/89/8fc1c268969fac43688d65fd92e67df24bd128d53cb4d2eee534cd307399/pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a", size = 11828897, upload-time = "2026-05-11T18:54:17.146Z" }, + { url = "https://files.pythonhosted.org/packages/56/3b/e7d20dea247a3e6dc0bd8a6953854afbedc03951def4e7371e05e7263e25/pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1", size = 10900855, upload-time = "2026-05-11T18:54:19.72Z" }, + { url = "https://files.pythonhosted.org/packages/0f/54/68a0978d1ef8502b8492099beaa6e7a0c1b32e3b5d4f677f5810cb08711c/pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1", size = 9466464, upload-time = "2026-05-11T18:54:22.754Z" }, ] [[package]] @@ -2317,6 +2594,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772, upload-time = "2023-11-25T06:56:14.81Z" }, ] +[[package]] +name = "platformdirs" +version = "4.9.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9f/4a/0883b8e3802965322523f0b200ecf33d31f10991d0401162f4b23c698b42/platformdirs-4.9.6.tar.gz", hash = "sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a", size = 29400, upload-time = "2026-04-09T00:04:10.812Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917", size = 21348, upload-time = "2026-04-09T00:04:09.463Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -2349,71 +2635,79 @@ wheels = [ [[package]] name = "propcache" -version = "0.4.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9e/da/e9fc233cf63743258bff22b3dfa7ea5baef7b5bc324af47a0ad89b8ffc6f/propcache-0.4.1.tar.gz", hash = "sha256:f48107a8c637e80362555f37ecf49abe20370e557cc4ab374f04ec4423c97c3d", size = 46442, upload-time = "2025-10-08T19:49:02.291Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/df/6d9c1b6ac12b003837dde8a10231a7344512186e87b36e855bef32241942/propcache-0.4.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:43eedf29202c08550aac1d14e0ee619b0430aaef78f85864c1a892294fbc28cf", size = 77750, upload-time = "2025-10-08T19:47:07.648Z" }, - { url = "https://files.pythonhosted.org/packages/8b/e8/677a0025e8a2acf07d3418a2e7ba529c9c33caf09d3c1f25513023c1db56/propcache-0.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d62cdfcfd89ccb8de04e0eda998535c406bf5e060ffd56be6c586cbcc05b3311", size = 44780, upload-time = "2025-10-08T19:47:08.851Z" }, - { url = "https://files.pythonhosted.org/packages/89/a4/92380f7ca60f99ebae761936bc48a72a639e8a47b29050615eef757cb2a7/propcache-0.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cae65ad55793da34db5f54e4029b89d3b9b9490d8abe1b4c7ab5d4b8ec7ebf74", size = 46308, upload-time = "2025-10-08T19:47:09.982Z" }, - { url = "https://files.pythonhosted.org/packages/2d/48/c5ac64dee5262044348d1d78a5f85dd1a57464a60d30daee946699963eb3/propcache-0.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:333ddb9031d2704a301ee3e506dc46b1fe5f294ec198ed6435ad5b6a085facfe", size = 208182, upload-time = "2025-10-08T19:47:11.319Z" }, - { url = "https://files.pythonhosted.org/packages/c6/0c/cd762dd011a9287389a6a3eb43aa30207bde253610cca06824aeabfe9653/propcache-0.4.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fd0858c20f078a32cf55f7e81473d96dcf3b93fd2ccdb3d40fdf54b8573df3af", size = 211215, upload-time = "2025-10-08T19:47:13.146Z" }, - { url = "https://files.pythonhosted.org/packages/30/3e/49861e90233ba36890ae0ca4c660e95df565b2cd15d4a68556ab5865974e/propcache-0.4.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:678ae89ebc632c5c204c794f8dab2837c5f159aeb59e6ed0539500400577298c", size = 218112, upload-time = "2025-10-08T19:47:14.913Z" }, - { url = "https://files.pythonhosted.org/packages/f1/8b/544bc867e24e1bd48f3118cecd3b05c694e160a168478fa28770f22fd094/propcache-0.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d472aeb4fbf9865e0c6d622d7f4d54a4e101a89715d8904282bb5f9a2f476c3f", size = 204442, upload-time = "2025-10-08T19:47:16.277Z" }, - { url = "https://files.pythonhosted.org/packages/50/a6/4282772fd016a76d3e5c0df58380a5ea64900afd836cec2c2f662d1b9bb3/propcache-0.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4d3df5fa7e36b3225954fba85589da77a0fe6a53e3976de39caf04a0db4c36f1", size = 199398, upload-time = "2025-10-08T19:47:17.962Z" }, - { url = "https://files.pythonhosted.org/packages/3e/ec/d8a7cd406ee1ddb705db2139f8a10a8a427100347bd698e7014351c7af09/propcache-0.4.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:ee17f18d2498f2673e432faaa71698032b0127ebf23ae5974eeaf806c279df24", size = 196920, upload-time = "2025-10-08T19:47:19.355Z" }, - { url = "https://files.pythonhosted.org/packages/f6/6c/f38ab64af3764f431e359f8baf9e0a21013e24329e8b85d2da32e8ed07ca/propcache-0.4.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:580e97762b950f993ae618e167e7be9256b8353c2dcd8b99ec100eb50f5286aa", size = 203748, upload-time = "2025-10-08T19:47:21.338Z" }, - { url = "https://files.pythonhosted.org/packages/d6/e3/fa846bd70f6534d647886621388f0a265254d30e3ce47e5c8e6e27dbf153/propcache-0.4.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:501d20b891688eb8e7aa903021f0b72d5a55db40ffaab27edefd1027caaafa61", size = 205877, upload-time = "2025-10-08T19:47:23.059Z" }, - { url = "https://files.pythonhosted.org/packages/e2/39/8163fc6f3133fea7b5f2827e8eba2029a0277ab2c5beee6c1db7b10fc23d/propcache-0.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9a0bd56e5b100aef69bd8562b74b46254e7c8812918d3baa700c8a8009b0af66", size = 199437, upload-time = "2025-10-08T19:47:24.445Z" }, - { url = "https://files.pythonhosted.org/packages/93/89/caa9089970ca49c7c01662bd0eeedfe85494e863e8043565aeb6472ce8fe/propcache-0.4.1-cp313-cp313-win32.whl", hash = "sha256:bcc9aaa5d80322bc2fb24bb7accb4a30f81e90ab8d6ba187aec0744bc302ad81", size = 37586, upload-time = "2025-10-08T19:47:25.736Z" }, - { url = "https://files.pythonhosted.org/packages/f5/ab/f76ec3c3627c883215b5c8080debb4394ef5a7a29be811f786415fc1e6fd/propcache-0.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:381914df18634f5494334d201e98245c0596067504b9372d8cf93f4bb23e025e", size = 40790, upload-time = "2025-10-08T19:47:26.847Z" }, - { url = "https://files.pythonhosted.org/packages/59/1b/e71ae98235f8e2ba5004d8cb19765a74877abf189bc53fc0c80d799e56c3/propcache-0.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:8873eb4460fd55333ea49b7d189749ecf6e55bf85080f11b1c4530ed3034cba1", size = 37158, upload-time = "2025-10-08T19:47:27.961Z" }, - { url = "https://files.pythonhosted.org/packages/83/ce/a31bbdfc24ee0dcbba458c8175ed26089cf109a55bbe7b7640ed2470cfe9/propcache-0.4.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:92d1935ee1f8d7442da9c0c4fa7ac20d07e94064184811b685f5c4fada64553b", size = 81451, upload-time = "2025-10-08T19:47:29.445Z" }, - { url = "https://files.pythonhosted.org/packages/25/9c/442a45a470a68456e710d96cacd3573ef26a1d0a60067e6a7d5e655621ed/propcache-0.4.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:473c61b39e1460d386479b9b2f337da492042447c9b685f28be4f74d3529e566", size = 46374, upload-time = "2025-10-08T19:47:30.579Z" }, - { url = "https://files.pythonhosted.org/packages/f4/bf/b1d5e21dbc3b2e889ea4327044fb16312a736d97640fb8b6aa3f9c7b3b65/propcache-0.4.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:c0ef0aaafc66fbd87842a3fe3902fd889825646bc21149eafe47be6072725835", size = 48396, upload-time = "2025-10-08T19:47:31.79Z" }, - { url = "https://files.pythonhosted.org/packages/f4/04/5b4c54a103d480e978d3c8a76073502b18db0c4bc17ab91b3cb5092ad949/propcache-0.4.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f95393b4d66bfae908c3ca8d169d5f79cd65636ae15b5e7a4f6e67af675adb0e", size = 275950, upload-time = "2025-10-08T19:47:33.481Z" }, - { url = "https://files.pythonhosted.org/packages/b4/c1/86f846827fb969c4b78b0af79bba1d1ea2156492e1b83dea8b8a6ae27395/propcache-0.4.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c07fda85708bc48578467e85099645167a955ba093be0a2dcba962195676e859", size = 273856, upload-time = "2025-10-08T19:47:34.906Z" }, - { url = "https://files.pythonhosted.org/packages/36/1d/fc272a63c8d3bbad6878c336c7a7dea15e8f2d23a544bda43205dfa83ada/propcache-0.4.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:af223b406d6d000830c6f65f1e6431783fc3f713ba3e6cc8c024d5ee96170a4b", size = 280420, upload-time = "2025-10-08T19:47:36.338Z" }, - { url = "https://files.pythonhosted.org/packages/07/0c/01f2219d39f7e53d52e5173bcb09c976609ba30209912a0680adfb8c593a/propcache-0.4.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a78372c932c90ee474559c5ddfffd718238e8673c340dc21fe45c5b8b54559a0", size = 263254, upload-time = "2025-10-08T19:47:37.692Z" }, - { url = "https://files.pythonhosted.org/packages/2d/18/cd28081658ce597898f0c4d174d4d0f3c5b6d4dc27ffafeef835c95eb359/propcache-0.4.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:564d9f0d4d9509e1a870c920a89b2fec951b44bf5ba7d537a9e7c1ccec2c18af", size = 261205, upload-time = "2025-10-08T19:47:39.659Z" }, - { url = "https://files.pythonhosted.org/packages/7a/71/1f9e22eb8b8316701c2a19fa1f388c8a3185082607da8e406a803c9b954e/propcache-0.4.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:17612831fda0138059cc5546f4d12a2aacfb9e47068c06af35c400ba58ba7393", size = 247873, upload-time = "2025-10-08T19:47:41.084Z" }, - { url = "https://files.pythonhosted.org/packages/4a/65/3d4b61f36af2b4eddba9def857959f1016a51066b4f1ce348e0cf7881f58/propcache-0.4.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:41a89040cb10bd345b3c1a873b2bf36413d48da1def52f268a055f7398514874", size = 262739, upload-time = "2025-10-08T19:47:42.51Z" }, - { url = "https://files.pythonhosted.org/packages/2a/42/26746ab087faa77c1c68079b228810436ccd9a5ce9ac85e2b7307195fd06/propcache-0.4.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:e35b88984e7fa64aacecea39236cee32dd9bd8c55f57ba8a75cf2399553f9bd7", size = 263514, upload-time = "2025-10-08T19:47:43.927Z" }, - { url = "https://files.pythonhosted.org/packages/94/13/630690fe201f5502d2403dd3cfd451ed8858fe3c738ee88d095ad2ff407b/propcache-0.4.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6f8b465489f927b0df505cbe26ffbeed4d6d8a2bbc61ce90eb074ff129ef0ab1", size = 257781, upload-time = "2025-10-08T19:47:45.448Z" }, - { url = "https://files.pythonhosted.org/packages/92/f7/1d4ec5841505f423469efbfc381d64b7b467438cd5a4bbcbb063f3b73d27/propcache-0.4.1-cp313-cp313t-win32.whl", hash = "sha256:2ad890caa1d928c7c2965b48f3a3815c853180831d0e5503d35cf00c472f4717", size = 41396, upload-time = "2025-10-08T19:47:47.202Z" }, - { url = "https://files.pythonhosted.org/packages/48/f0/615c30622316496d2cbbc29f5985f7777d3ada70f23370608c1d3e081c1f/propcache-0.4.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f7ee0e597f495cf415bcbd3da3caa3bd7e816b74d0d52b8145954c5e6fd3ff37", size = 44897, upload-time = "2025-10-08T19:47:48.336Z" }, - { url = "https://files.pythonhosted.org/packages/fd/ca/6002e46eccbe0e33dcd4069ef32f7f1c9e243736e07adca37ae8c4830ec3/propcache-0.4.1-cp313-cp313t-win_arm64.whl", hash = "sha256:929d7cbe1f01bb7baffb33dc14eb5691c95831450a26354cd210a8155170c93a", size = 39789, upload-time = "2025-10-08T19:47:49.876Z" }, - { url = "https://files.pythonhosted.org/packages/8e/5c/bca52d654a896f831b8256683457ceddd490ec18d9ec50e97dfd8fc726a8/propcache-0.4.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3f7124c9d820ba5548d431afb4632301acf965db49e666aa21c305cbe8c6de12", size = 78152, upload-time = "2025-10-08T19:47:51.051Z" }, - { url = "https://files.pythonhosted.org/packages/65/9b/03b04e7d82a5f54fb16113d839f5ea1ede58a61e90edf515f6577c66fa8f/propcache-0.4.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:c0d4b719b7da33599dfe3b22d3db1ef789210a0597bc650b7cee9c77c2be8c5c", size = 44869, upload-time = "2025-10-08T19:47:52.594Z" }, - { url = "https://files.pythonhosted.org/packages/b2/fa/89a8ef0468d5833a23fff277b143d0573897cf75bd56670a6d28126c7d68/propcache-0.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9f302f4783709a78240ebc311b793f123328716a60911d667e0c036bc5dcbded", size = 46596, upload-time = "2025-10-08T19:47:54.073Z" }, - { url = "https://files.pythonhosted.org/packages/86/bd/47816020d337f4a746edc42fe8d53669965138f39ee117414c7d7a340cfe/propcache-0.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c80ee5802e3fb9ea37938e7eecc307fb984837091d5fd262bb37238b1ae97641", size = 206981, upload-time = "2025-10-08T19:47:55.715Z" }, - { url = "https://files.pythonhosted.org/packages/df/f6/c5fa1357cc9748510ee55f37173eb31bfde6d94e98ccd9e6f033f2fc06e1/propcache-0.4.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ed5a841e8bb29a55fb8159ed526b26adc5bdd7e8bd7bf793ce647cb08656cdf4", size = 211490, upload-time = "2025-10-08T19:47:57.499Z" }, - { url = "https://files.pythonhosted.org/packages/80/1e/e5889652a7c4a3846683401a48f0f2e5083ce0ec1a8a5221d8058fbd1adf/propcache-0.4.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:55c72fd6ea2da4c318e74ffdf93c4fe4e926051133657459131a95c846d16d44", size = 215371, upload-time = "2025-10-08T19:47:59.317Z" }, - { url = "https://files.pythonhosted.org/packages/b2/f2/889ad4b2408f72fe1a4f6a19491177b30ea7bf1a0fd5f17050ca08cfc882/propcache-0.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8326e144341460402713f91df60ade3c999d601e7eb5ff8f6f7862d54de0610d", size = 201424, upload-time = "2025-10-08T19:48:00.67Z" }, - { url = "https://files.pythonhosted.org/packages/27/73/033d63069b57b0812c8bd19f311faebeceb6ba31b8f32b73432d12a0b826/propcache-0.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:060b16ae65bc098da7f6d25bf359f1f31f688384858204fe5d652979e0015e5b", size = 197566, upload-time = "2025-10-08T19:48:02.604Z" }, - { url = "https://files.pythonhosted.org/packages/dc/89/ce24f3dc182630b4e07aa6d15f0ff4b14ed4b9955fae95a0b54c58d66c05/propcache-0.4.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:89eb3fa9524f7bec9de6e83cf3faed9d79bffa560672c118a96a171a6f55831e", size = 193130, upload-time = "2025-10-08T19:48:04.499Z" }, - { url = "https://files.pythonhosted.org/packages/a9/24/ef0d5fd1a811fb5c609278d0209c9f10c35f20581fcc16f818da959fc5b4/propcache-0.4.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:dee69d7015dc235f526fe80a9c90d65eb0039103fe565776250881731f06349f", size = 202625, upload-time = "2025-10-08T19:48:06.213Z" }, - { url = "https://files.pythonhosted.org/packages/f5/02/98ec20ff5546f68d673df2f7a69e8c0d076b5abd05ca882dc7ee3a83653d/propcache-0.4.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:5558992a00dfd54ccbc64a32726a3357ec93825a418a401f5cc67df0ac5d9e49", size = 204209, upload-time = "2025-10-08T19:48:08.432Z" }, - { url = "https://files.pythonhosted.org/packages/a0/87/492694f76759b15f0467a2a93ab68d32859672b646aa8a04ce4864e7932d/propcache-0.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c9b822a577f560fbd9554812526831712c1436d2c046cedee4c3796d3543b144", size = 197797, upload-time = "2025-10-08T19:48:09.968Z" }, - { url = "https://files.pythonhosted.org/packages/ee/36/66367de3575db1d2d3f3d177432bd14ee577a39d3f5d1b3d5df8afe3b6e2/propcache-0.4.1-cp314-cp314-win32.whl", hash = "sha256:ab4c29b49d560fe48b696cdcb127dd36e0bc2472548f3bf56cc5cb3da2b2984f", size = 38140, upload-time = "2025-10-08T19:48:11.232Z" }, - { url = "https://files.pythonhosted.org/packages/0c/2a/a758b47de253636e1b8aef181c0b4f4f204bf0dd964914fb2af90a95b49b/propcache-0.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:5a103c3eb905fcea0ab98be99c3a9a5ab2de60228aa5aceedc614c0281cf6153", size = 41257, upload-time = "2025-10-08T19:48:12.707Z" }, - { url = "https://files.pythonhosted.org/packages/34/5e/63bd5896c3fec12edcbd6f12508d4890d23c265df28c74b175e1ef9f4f3b/propcache-0.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:74c1fb26515153e482e00177a1ad654721bf9207da8a494a0c05e797ad27b992", size = 38097, upload-time = "2025-10-08T19:48:13.923Z" }, - { url = "https://files.pythonhosted.org/packages/99/85/9ff785d787ccf9bbb3f3106f79884a130951436f58392000231b4c737c80/propcache-0.4.1-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:824e908bce90fb2743bd6b59db36eb4f45cd350a39637c9f73b1c1ea66f5b75f", size = 81455, upload-time = "2025-10-08T19:48:15.16Z" }, - { url = "https://files.pythonhosted.org/packages/90/85/2431c10c8e7ddb1445c1f7c4b54d886e8ad20e3c6307e7218f05922cad67/propcache-0.4.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:c2b5e7db5328427c57c8e8831abda175421b709672f6cfc3d630c3b7e2146393", size = 46372, upload-time = "2025-10-08T19:48:16.424Z" }, - { url = "https://files.pythonhosted.org/packages/01/20/b0972d902472da9bcb683fa595099911f4d2e86e5683bcc45de60dd05dc3/propcache-0.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6f6ff873ed40292cd4969ef5310179afd5db59fdf055897e282485043fc80ad0", size = 48411, upload-time = "2025-10-08T19:48:17.577Z" }, - { url = "https://files.pythonhosted.org/packages/e2/e3/7dc89f4f21e8f99bad3d5ddb3a3389afcf9da4ac69e3deb2dcdc96e74169/propcache-0.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49a2dc67c154db2c1463013594c458881a069fcf98940e61a0569016a583020a", size = 275712, upload-time = "2025-10-08T19:48:18.901Z" }, - { url = "https://files.pythonhosted.org/packages/20/67/89800c8352489b21a8047c773067644e3897f02ecbbd610f4d46b7f08612/propcache-0.4.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:005f08e6a0529984491e37d8dbc3dd86f84bd78a8ceb5fa9a021f4c48d4984be", size = 273557, upload-time = "2025-10-08T19:48:20.762Z" }, - { url = "https://files.pythonhosted.org/packages/e2/a1/b52b055c766a54ce6d9c16d9aca0cad8059acd9637cdf8aa0222f4a026ef/propcache-0.4.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5c3310452e0d31390da9035c348633b43d7e7feb2e37be252be6da45abd1abcc", size = 280015, upload-time = "2025-10-08T19:48:22.592Z" }, - { url = "https://files.pythonhosted.org/packages/48/c8/33cee30bd890672c63743049f3c9e4be087e6780906bfc3ec58528be59c1/propcache-0.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4c3c70630930447f9ef1caac7728c8ad1c56bc5015338b20fed0d08ea2480b3a", size = 262880, upload-time = "2025-10-08T19:48:23.947Z" }, - { url = "https://files.pythonhosted.org/packages/0c/b1/8f08a143b204b418285c88b83d00edbd61afbc2c6415ffafc8905da7038b/propcache-0.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8e57061305815dfc910a3634dcf584f08168a8836e6999983569f51a8544cd89", size = 260938, upload-time = "2025-10-08T19:48:25.656Z" }, - { url = "https://files.pythonhosted.org/packages/cf/12/96e4664c82ca2f31e1c8dff86afb867348979eb78d3cb8546a680287a1e9/propcache-0.4.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:521a463429ef54143092c11a77e04056dd00636f72e8c45b70aaa3140d639726", size = 247641, upload-time = "2025-10-08T19:48:27.207Z" }, - { url = "https://files.pythonhosted.org/packages/18/ed/e7a9cfca28133386ba52278136d42209d3125db08d0a6395f0cba0c0285c/propcache-0.4.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:120c964da3fdc75e3731aa392527136d4ad35868cc556fd09bb6d09172d9a367", size = 262510, upload-time = "2025-10-08T19:48:28.65Z" }, - { url = "https://files.pythonhosted.org/packages/f5/76/16d8bf65e8845dd62b4e2b57444ab81f07f40caa5652b8969b87ddcf2ef6/propcache-0.4.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:d8f353eb14ee3441ee844ade4277d560cdd68288838673273b978e3d6d2c8f36", size = 263161, upload-time = "2025-10-08T19:48:30.133Z" }, - { url = "https://files.pythonhosted.org/packages/e7/70/c99e9edb5d91d5ad8a49fa3c1e8285ba64f1476782fed10ab251ff413ba1/propcache-0.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ab2943be7c652f09638800905ee1bab2c544e537edb57d527997a24c13dc1455", size = 257393, upload-time = "2025-10-08T19:48:31.567Z" }, - { url = "https://files.pythonhosted.org/packages/08/02/87b25304249a35c0915d236575bc3574a323f60b47939a2262b77632a3ee/propcache-0.4.1-cp314-cp314t-win32.whl", hash = "sha256:05674a162469f31358c30bcaa8883cb7829fa3110bf9c0991fe27d7896c42d85", size = 42546, upload-time = "2025-10-08T19:48:32.872Z" }, - { url = "https://files.pythonhosted.org/packages/cb/ef/3c6ecf8b317aa982f309835e8f96987466123c6e596646d4e6a1dfcd080f/propcache-0.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:990f6b3e2a27d683cb7602ed6c86f15ee6b43b1194736f9baaeb93d0016633b1", size = 46259, upload-time = "2025-10-08T19:48:34.226Z" }, - { url = "https://files.pythonhosted.org/packages/c4/2d/346e946d4951f37eca1e4f55be0f0174c52cd70720f84029b02f296f4a38/propcache-0.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ecef2343af4cc68e05131e45024ba34f6095821988a9d0a02aa7c73fcc448aa9", size = 40428, upload-time = "2025-10-08T19:48:35.441Z" }, - { url = "https://files.pythonhosted.org/packages/5b/5a/bc7b4a4ef808fa59a816c17b20c4bef6884daebbdf627ff2a161da67da19/propcache-0.4.1-py3-none-any.whl", hash = "sha256:af2a6052aeb6cf17d3e46ee169099044fd8224cbaf75c76a2ef596e8163e2237", size = 13305, upload-time = "2025-10-08T19:49:00.792Z" }, +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/44/c87281c333769159c50594f22610f77398a47ccbfbbf23074e744e86f87c/propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427", size = 50208, upload-time = "2026-05-08T21:02:12.199Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/09/f049e45385503fe67db75a6b6186a7b9f0c3930366dc960522c312a825b1/propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a", size = 94457, upload-time = "2026-05-08T21:00:36.355Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/83d1d05655baf63113731bd5a1008435e14f8d1e5a06cbe4ec5b23ad7a31/propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117", size = 53835, upload-time = "2026-05-08T21:00:38.072Z" }, + { url = "https://files.pythonhosted.org/packages/a9/12/a6ba6482bb5ea3260c000c9b20881c95fa11c6b30173715668259f844ed7/propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098", size = 54545, upload-time = "2026-05-08T21:00:39.319Z" }, + { url = "https://files.pythonhosted.org/packages/a9/19/7fa086f5764c59ec8a8e157cd93aa8497acc00aba9dcdec56bfffb32602d/propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4", size = 59886, upload-time = "2026-05-08T21:00:40.621Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e4/5d7663dc8235956c8f5281698a3af1d351d8820341ddd890f59d9a9127f2/propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e", size = 63261, upload-time = "2026-05-08T21:00:41.775Z" }, + { url = "https://files.pythonhosted.org/packages/4a/4a/15a03adee24d6350da4292caeac44c34c033d2afe5e87eb370f38854560f/propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7", size = 64184, upload-time = "2026-05-08T21:00:43.018Z" }, + { url = "https://files.pythonhosted.org/packages/8b/c6/979176efdaa3d239e36d503d5af63a0a773b36662ed8f52e5b6a6d9fd40e/propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d", size = 61534, upload-time = "2026-05-08T21:00:44.507Z" }, + { url = "https://files.pythonhosted.org/packages/c8/22/63e8cd1bae4c2d2be6493b6b7d10566ddafad88137cfbc99964a1119853c/propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a", size = 61500, upload-time = "2026-05-08T21:00:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/60/5a/28e5d9acbac1cc9ccb67045e8c1b943aa8d79fdf39c93bd73cacd68008ea/propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2", size = 59994, upload-time = "2026-05-08T21:00:47.093Z" }, + { url = "https://files.pythonhosted.org/packages/f3/40/db650677f554a95b9c01a7c9d93d629e93a15562f5deb4573c9ee136fed2/propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa", size = 56884, upload-time = "2026-05-08T21:00:48.376Z" }, + { url = "https://files.pythonhosted.org/packages/80/45/70b39b89516ff8b96bf732fa6fded8cef20f293cb1508690101c3c07ec51/propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853", size = 63464, upload-time = "2026-05-08T21:00:49.954Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e2/fa59d3a89eac5534293124af4f1d0d0ada091ce4a0ab4610ce03fd2bdd8d/propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a", size = 61588, upload-time = "2026-05-08T21:00:51.281Z" }, + { url = "https://files.pythonhosted.org/packages/0b/97/efb547a55c4bc7381cfb202d6a2239ac621045277bc1ea5dfd3a7f0516c0/propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704", size = 64667, upload-time = "2026-05-08T21:00:52.602Z" }, + { url = "https://files.pythonhosted.org/packages/92/56/f5c7d9b4b7595d5127da38974d791b2153f3d1eae6c674af3583ace92ad3/propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4", size = 62463, upload-time = "2026-05-08T21:00:54.303Z" }, + { url = "https://files.pythonhosted.org/packages/bd/3b/484a3a65fc9f9f60c41dcd17b428bace5389544e2c680994534a20755066/propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d", size = 38621, upload-time = "2026-05-08T21:00:55.808Z" }, + { url = "https://files.pythonhosted.org/packages/1c/fd/3f0f10dba4dabad3bf53102be007abf55481067952bde0fdddff439e7c61/propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757", size = 41649, upload-time = "2026-05-08T21:00:57.061Z" }, + { url = "https://files.pythonhosted.org/packages/90/ec/6ce619cc32bb500a482f811f9cd509368b4e58e638d13f2c68f370d6b475/propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f", size = 37636, upload-time = "2026-05-08T21:00:58.646Z" }, + { url = "https://files.pythonhosted.org/packages/1b/82/c1d268bbbf2ef981c5bf0fbbe746db617c66e3bcefe431a1aa8943fbe23a/propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d", size = 98872, upload-time = "2026-05-08T21:00:59.889Z" }, + { url = "https://files.pythonhosted.org/packages/f4/d4/52c871e73e864e6b34c0e2d58ac1ec5ccd149497ddc7ad2137ae98323a35/propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa", size = 56257, upload-time = "2026-05-08T21:01:01.195Z" }, + { url = "https://files.pythonhosted.org/packages/67/f0/9b90ca2a210b3d09bcfcd96ecd0f55545c091535abce2a45de2775cfd357/propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94", size = 56696, upload-time = "2026-05-08T21:01:02.941Z" }, + { url = "https://files.pythonhosted.org/packages/9d/0e/6e9d4ba07c8e56e21ddec1e75f12148142b21ca83a51871babce095334f4/propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164", size = 62378, upload-time = "2026-05-08T21:01:04.475Z" }, + { url = "https://files.pythonhosted.org/packages/65/19/c10badaa463dde8a27ce884f8ee2ec37e6035b7c9f5ff0c8f74f06f08dac/propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f", size = 65283, upload-time = "2026-05-08T21:01:05.959Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b6/93bea99ca80e19cef6512a8580e5b7857bbe09422d9daa7fd4ef5723306c/propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c", size = 66616, upload-time = "2026-05-08T21:01:07.228Z" }, + { url = "https://files.pythonhosted.org/packages/83/e4/5c7462e50625f051f37fb38b8224f7639f667184bbd34424ec83819bb1b7/propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc", size = 63773, upload-time = "2026-05-08T21:01:08.514Z" }, + { url = "https://files.pythonhosted.org/packages/ca/b6/99238894047b13c823be25027e736626cd414a52a5e30d2c3347c2733529/propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f", size = 63664, upload-time = "2026-05-08T21:01:09.874Z" }, + { url = "https://files.pythonhosted.org/packages/85/1e/a3a1a63116a2b8edb415a8bb9a6f0c34bd03830b1e18e8ce2904e1dc1cf4/propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb", size = 62643, upload-time = "2026-05-08T21:01:11.132Z" }, + { url = "https://files.pythonhosted.org/packages/e4/03/893cf147de2fc6543c5eaa07ad833170e7e2a2385725bbebe8c0503723bb/propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751", size = 59595, upload-time = "2026-05-08T21:01:12.387Z" }, + { url = "https://files.pythonhosted.org/packages/86/3b/04c1a2e12c57766568ba75ba72b3bf2042818d4c1425fab6fc07155c7cff/propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836", size = 65711, upload-time = "2026-05-08T21:01:13.676Z" }, + { url = "https://files.pythonhosted.org/packages/1c/34/80f8d0099f8d6bacc4de1624c85672681c8cd1149ca2da0e38fd120b817f/propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f", size = 64247, upload-time = "2026-05-08T21:01:14.936Z" }, + { url = "https://files.pythonhosted.org/packages/f3/1a/8b08f3a5f1037e9e370c55883ceeeee0f6dd0416fb2d2d67b8bfc91f2a79/propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55", size = 67102, upload-time = "2026-05-08T21:01:16.281Z" }, + { url = "https://files.pythonhosted.org/packages/34/68/8bdb7bb7756d76e005490649d10e4a8369e610c74d619f71e1aedf889e9c/propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568", size = 64964, upload-time = "2026-05-08T21:01:17.57Z" }, + { url = "https://files.pythonhosted.org/packages/0a/aa/50fb0b5d3968b61a510926ff8b8465f1d6e976b3ab74496d7a4b9fc42515/propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191", size = 42546, upload-time = "2026-05-08T21:01:18.946Z" }, + { url = "https://files.pythonhosted.org/packages/ae/4c/0ddbae64321bd4a95bcbfc19307238016b5b1fee645c84626c8d539e5b74/propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7", size = 46330, upload-time = "2026-05-08T21:01:20.162Z" }, + { url = "https://files.pythonhosted.org/packages/00/d9/9cddc8efb78d8af264c5ec9f6d10b62f57c515feda8d321595f56010fb23/propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96", size = 40521, upload-time = "2026-05-08T21:01:21.399Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ea/23ee535d90ce8bcc465a3028eb3cc0ce3bd1005f4bb27710b30587de798d/propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999", size = 94662, upload-time = "2026-05-08T21:01:22.683Z" }, + { url = "https://files.pythonhosted.org/packages/b5/06/c5a52f419b5d8972f8d46a7577476090d8e3263ff589ce40b5ca4968d5be/propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e", size = 53928, upload-time = "2026-05-08T21:01:23.986Z" }, + { url = "https://files.pythonhosted.org/packages/63/b1/4260d67d6bd85e58a66b72d54ce15d5de789b6f3870cc6bedf8ff9667401/propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539", size = 54650, upload-time = "2026-05-08T21:01:25.305Z" }, + { url = "https://files.pythonhosted.org/packages/70/06/2f46c318e3307cd7a6a7481def374ce838c0fe20084b39dd54b0879d0e99/propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e", size = 59912, upload-time = "2026-05-08T21:01:26.545Z" }, + { url = "https://files.pythonhosted.org/packages/4c/29/fe1aebec2ce57ab985a9c382bded1124431f85078113aa222c5d278430d4/propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979", size = 63300, upload-time = "2026-05-08T21:01:27.937Z" }, + { url = "https://files.pythonhosted.org/packages/b4/18/2334b26768b6c82be8c69e83671b767d5ef426aa09b0cba6c2ea47816774/propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80", size = 64208, upload-time = "2026-05-08T21:01:29.484Z" }, + { url = "https://files.pythonhosted.org/packages/2b/76/7f1bfd6afff4c5e38e36a3c6d68eb5f4b7311ea80baf693db78d95b603c4/propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825", size = 61633, upload-time = "2026-05-08T21:01:31.068Z" }, + { url = "https://files.pythonhosted.org/packages/c4/46/b3ff8aba2b4953a3e50de2cf72f1b5748b8eca93b15f3dc2c84339084c09/propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39", size = 61724, upload-time = "2026-05-08T21:01:32.374Z" }, + { url = "https://files.pythonhosted.org/packages/c5/01/814cfcafbcff954f94c01cf30e097ddc88a076b5440fbcf4570753437d40/propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4", size = 60069, upload-time = "2026-05-08T21:01:33.67Z" }, + { url = "https://files.pythonhosted.org/packages/da/68/5c6f7622d510cc666a300687e06fd060c1a43361c0c9b20d284f06d8096a/propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5", size = 57099, upload-time = "2026-05-08T21:01:34.915Z" }, + { url = "https://files.pythonhosted.org/packages/55/27/9cb0b4c679124085327957d42521c99dba04c88c90c3e55a6f0b633ebccc/propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702", size = 63391, upload-time = "2026-05-08T21:01:36.231Z" }, + { url = "https://files.pythonhosted.org/packages/f0/9d/7258aaa5bdf60fc6f27591eef6fe52768cb0beda7140be477c8b12c9794a/propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3", size = 61626, upload-time = "2026-05-08T21:01:37.545Z" }, + { url = "https://files.pythonhosted.org/packages/8e/0d/41c602003e8a9b16fe1e7eadf62c7bfba9d5474370b24200bf48b315f45f/propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5", size = 64781, upload-time = "2026-05-08T21:01:38.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f3/38e66b1856e9bd079deea015bc4a55f7767c0e4db2f7dcf69e7e680ba4ce/propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4", size = 62570, upload-time = "2026-05-08T21:01:40.415Z" }, + { url = "https://files.pythonhosted.org/packages/95/ca/bbfe9b910ce57dde8bb4876b4520fc02a4e89497c10de26be936758a3aaa/propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0", size = 39436, upload-time = "2026-05-08T21:01:41.654Z" }, + { url = "https://files.pythonhosted.org/packages/61/d2/45c9defbaa1ea297035d9d4cce9e8f80daafbf19319c6007f157c6256ea9/propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c", size = 42373, upload-time = "2026-05-08T21:01:43.041Z" }, + { url = "https://files.pythonhosted.org/packages/44/68/9ea5103f41d5217d7d6ec24db90018e23aebec070c3f9a6e54d12b841fd8/propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0", size = 38554, upload-time = "2026-05-08T21:01:44.336Z" }, + { url = "https://files.pythonhosted.org/packages/8a/81/fadf555f42d3b762eea8a53950b0489fdc0aa9da5f8ed9e10ce0a4e01b48/propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb", size = 99395, upload-time = "2026-05-08T21:01:45.883Z" }, + { url = "https://files.pythonhosted.org/packages/f5/c9/c61e134a686949cf7971af3a390148b1156f7be81c73bc0cd12c873e2d48/propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078", size = 56653, upload-time = "2026-05-08T21:01:47.307Z" }, + { url = "https://files.pythonhosted.org/packages/cb/73/daf935ea7048ddd7ec8eec5345b4a40b619d2d178b3c0a0900796bc3c794/propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa", size = 56914, upload-time = "2026-05-08T21:01:48.573Z" }, + { url = "https://files.pythonhosted.org/packages/79/9f/aba959b435ea18617edd7cf0a7ad0b9c574b8fc7e3d2cd55fb59cb255d33/propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917", size = 62567, upload-time = "2026-05-08T21:01:49.903Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a1/859942de9a791ff42f6141736f5b37749b8f53e65edfa49638c67dd67e6a/propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe", size = 65542, upload-time = "2026-05-08T21:01:51.204Z" }, + { url = "https://files.pythonhosted.org/packages/b5/61/315bc0fd6c0fc7f80a528b8afd209e5fc4a875ea79571b91b8f50f442907/propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03", size = 66845, upload-time = "2026-05-08T21:01:52.539Z" }, + { url = "https://files.pythonhosted.org/packages/47/f7/9f8122e3132e8e354ac41975ef8f1099be7d5a16bc7ae562734e993665c0/propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335", size = 63985, upload-time = "2026-05-08T21:01:53.847Z" }, + { url = "https://files.pythonhosted.org/packages/c8/54/c317819ec157cbf6f35df9df9657a6f82daf34d5faf15948b2f639c2192e/propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285", size = 63999, upload-time = "2026-05-08T21:01:55.179Z" }, + { url = "https://files.pythonhosted.org/packages/5a/56/387e3f7dfce0a9233df41fb888aa1c30222cb4bbbf09537c02dd9bd85fe2/propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837", size = 62779, upload-time = "2026-05-08T21:01:57.489Z" }, + { url = "https://files.pythonhosted.org/packages/a1/9c/596784cb5824ed61ee960d3f8655a3f0993e107c6e98ab6c818b7fb92ccb/propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8", size = 59796, upload-time = "2026-05-08T21:01:58.736Z" }, + { url = "https://files.pythonhosted.org/packages/c2/3d/1a6cfa1726a48542c1e8784a0761421476a5b68e09b7f36bf95eb954aaba/propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366", size = 66023, upload-time = "2026-05-08T21:02:00.228Z" }, + { url = "https://files.pythonhosted.org/packages/e4/0e/05fd6990369477076e4e280bcb970de760fddf0161a46e988bc95f7940ec/propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56", size = 64448, upload-time = "2026-05-08T21:02:01.888Z" }, + { url = "https://files.pythonhosted.org/packages/cd/86/5f8da315a4309c62c10c0b2516b17492d5d3bbe1bb862b96604db67e2a37/propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d", size = 67329, upload-time = "2026-05-08T21:02:03.484Z" }, + { url = "https://files.pythonhosted.org/packages/da/d3/3368efe79ab21f0cdf86ef49895811c9cc933131d4cde1f28a624e22e712/propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2", size = 65172, upload-time = "2026-05-08T21:02:04.745Z" }, + { url = "https://files.pythonhosted.org/packages/d5/07/127e8b0bacfb325396196f9d976a22453049b89b9b2b08477cc3145faa44/propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821", size = 43813, upload-time = "2026-05-08T21:02:06.025Z" }, + { url = "https://files.pythonhosted.org/packages/88/fb/46dad6c0ae49ed230ab1b16c890c2b6314e2403e6c412976f4a72d64a527/propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370", size = 47764, upload-time = "2026-05-08T21:02:07.353Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c4/a47d0a63aa309d10d59ede6e9d4cff03a344a79d1f0f4cd0cd74997b53e0/propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6", size = 41140, upload-time = "2026-05-08T21:02:09.065Z" }, + { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, ] [[package]] @@ -2431,6 +2725,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, ] +[[package]] +name = "psutil" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/08/510cbdb69c25a96f4ae523f733cdc963ae654904e8db864c07585ef99875/psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b", size = 130595, upload-time = "2026-01-28T18:14:57.293Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f5/97baea3fe7a5a9af7436301f85490905379b1c6f2dd51fe3ecf24b4c5fbf/psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea", size = 131082, upload-time = "2026-01-28T18:14:59.732Z" }, + { url = "https://files.pythonhosted.org/packages/37/d6/246513fbf9fa174af531f28412297dd05241d97a75911ac8febefa1a53c6/psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63", size = 181476, upload-time = "2026-01-28T18:15:01.884Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b5/9182c9af3836cca61696dabe4fd1304e17bc56cb62f17439e1154f225dd3/psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312", size = 184062, upload-time = "2026-01-28T18:15:04.436Z" }, + { url = "https://files.pythonhosted.org/packages/16/ba/0756dca669f5a9300d0cbcbfae9a4c30e446dfc7440ffe43ded5724bfd93/psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b", size = 139893, upload-time = "2026-01-28T18:15:06.378Z" }, + { url = "https://files.pythonhosted.org/packages/1c/61/8fa0e26f33623b49949346de05ec1ddaad02ed8ba64af45f40a147dbfa97/psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9", size = 135589, upload-time = "2026-01-28T18:15:08.03Z" }, + { url = "https://files.pythonhosted.org/packages/81/69/ef179ab5ca24f32acc1dac0c247fd6a13b501fd5534dbae0e05a1c48b66d/psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00", size = 130664, upload-time = "2026-01-28T18:15:09.469Z" }, + { url = "https://files.pythonhosted.org/packages/7b/64/665248b557a236d3fa9efc378d60d95ef56dd0a490c2cd37dafc7660d4a9/psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9", size = 131087, upload-time = "2026-01-28T18:15:11.724Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" }, + { url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2c/ff9bfb544f283ba5f83ba725a3c5fec6d6b10b8f27ac1dc641c473dc390d/psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1", size = 141228, upload-time = "2026-01-28T18:15:18.385Z" }, + { url = "https://files.pythonhosted.org/packages/f2/fc/f8d9c31db14fcec13748d373e668bc3bed94d9077dbc17fb0eebc073233c/psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841", size = 136284, upload-time = "2026-01-28T18:15:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, + { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, + { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, + { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, + { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, +] + [[package]] name = "ptyprocess" version = "0.7.0" @@ -2469,38 +2791,38 @@ wheels = [ [[package]] name = "pyarrow" -version = "23.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/88/22/134986a4cc224d593c1afde5494d18ff629393d74cc2eddb176669f234a4/pyarrow-23.0.1.tar.gz", hash = "sha256:b8c5873e33440b2bc2f4a79d2b47017a89c5a24116c055625e6f2ee50523f019", size = 1167336, upload-time = "2026-02-16T10:14:12.39Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/47/10/2cbe4c6f0fb83d2de37249567373d64327a5e4d8db72f486db42875b08f6/pyarrow-23.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6b8fda694640b00e8af3c824f99f789e836720aa8c9379fb435d4c4953a756b8", size = 34210066, upload-time = "2026-02-16T10:10:45.487Z" }, - { url = "https://files.pythonhosted.org/packages/cb/4f/679fa7e84dadbaca7a65f7cdba8d6c83febbd93ca12fa4adf40ba3b6362b/pyarrow-23.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:8ff51b1addc469b9444b7c6f3548e19dc931b172ab234e995a60aea9f6e6025f", size = 35825526, upload-time = "2026-02-16T10:10:52.266Z" }, - { url = "https://files.pythonhosted.org/packages/f9/63/d2747d930882c9d661e9398eefc54f15696547b8983aaaf11d4a2e8b5426/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:71c5be5cbf1e1cb6169d2a0980850bccb558ddc9b747b6206435313c47c37677", size = 44473279, upload-time = "2026-02-16T10:11:01.557Z" }, - { url = "https://files.pythonhosted.org/packages/b3/93/10a48b5e238de6d562a411af6467e71e7aedbc9b87f8d3a35f1560ae30fb/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9b6f4f17b43bc39d56fec96e53fe89d94bac3eb134137964371b45352d40d0c2", size = 47585798, upload-time = "2026-02-16T10:11:09.401Z" }, - { url = "https://files.pythonhosted.org/packages/5c/20/476943001c54ef078dbf9542280e22741219a184a0632862bca4feccd666/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9fc13fc6c403d1337acab46a2c4346ca6c9dec5780c3c697cf8abfd5e19b6b37", size = 48179446, upload-time = "2026-02-16T10:11:17.781Z" }, - { url = "https://files.pythonhosted.org/packages/4b/b6/5dd0c47b335fcd8edba9bfab78ad961bd0fd55ebe53468cc393f45e0be60/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c16ed4f53247fa3ffb12a14d236de4213a4415d127fe9cebed33d51671113e2", size = 50623972, upload-time = "2026-02-16T10:11:26.185Z" }, - { url = "https://files.pythonhosted.org/packages/d5/09/a532297c9591a727d67760e2e756b83905dd89adb365a7f6e9c72578bcc1/pyarrow-23.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:cecfb12ef629cf6be0b1887f9f86463b0dd3dc3195ae6224e74006be4736035a", size = 27540749, upload-time = "2026-02-16T10:12:23.297Z" }, - { url = "https://files.pythonhosted.org/packages/a5/8e/38749c4b1303e6ae76b3c80618f84861ae0c55dd3c2273842ea6f8258233/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:29f7f7419a0e30264ea261fdc0e5fe63ce5a6095003db2945d7cd78df391a7e1", size = 34471544, upload-time = "2026-02-16T10:11:32.535Z" }, - { url = "https://files.pythonhosted.org/packages/a3/73/f237b2bc8c669212f842bcfd842b04fc8d936bfc9d471630569132dc920d/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:33d648dc25b51fd8055c19e4261e813dfc4d2427f068bcecc8b53d01b81b0500", size = 35949911, upload-time = "2026-02-16T10:11:39.813Z" }, - { url = "https://files.pythonhosted.org/packages/0c/86/b912195eee0903b5611bf596833def7d146ab2d301afeb4b722c57ffc966/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:cd395abf8f91c673dd3589cadc8cc1ee4e8674fa61b2e923c8dd215d9c7d1f41", size = 44520337, upload-time = "2026-02-16T10:11:47.764Z" }, - { url = "https://files.pythonhosted.org/packages/69/c2/f2a717fb824f62d0be952ea724b4f6f9372a17eed6f704b5c9526f12f2f1/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:00be9576d970c31defb5c32eb72ef585bf600ef6d0a82d5eccaae96639cf9d07", size = 47548944, upload-time = "2026-02-16T10:11:56.607Z" }, - { url = "https://files.pythonhosted.org/packages/84/a7/90007d476b9f0dc308e3bc57b832d004f848fd6c0da601375d20d92d1519/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c2139549494445609f35a5cda4eb94e2c9e4d704ce60a095b342f82460c73a83", size = 48236269, upload-time = "2026-02-16T10:12:04.47Z" }, - { url = "https://files.pythonhosted.org/packages/b0/3f/b16fab3e77709856eb6ac328ce35f57a6d4a18462c7ca5186ef31b45e0e0/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7044b442f184d84e2351e5084600f0d7343d6117aabcbc1ac78eb1ae11eb4125", size = 50604794, upload-time = "2026-02-16T10:12:11.797Z" }, - { url = "https://files.pythonhosted.org/packages/e9/a1/22df0620a9fac31d68397a75465c344e83c3dfe521f7612aea33e27ab6c0/pyarrow-23.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:a35581e856a2fafa12f3f54fce4331862b1cfb0bef5758347a858a4aa9d6bae8", size = 27660642, upload-time = "2026-02-16T10:12:17.746Z" }, - { url = "https://files.pythonhosted.org/packages/8d/1b/6da9a89583ce7b23ac611f183ae4843cd3a6cf54f079549b0e8c14031e73/pyarrow-23.0.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:5df1161da23636a70838099d4aaa65142777185cc0cdba4037a18cee7d8db9ca", size = 34238755, upload-time = "2026-02-16T10:12:32.819Z" }, - { url = "https://files.pythonhosted.org/packages/ae/b5/d58a241fbe324dbaeb8df07be6af8752c846192d78d2272e551098f74e88/pyarrow-23.0.1-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:fa8e51cb04b9f8c9c5ace6bab63af9a1f88d35c0d6cbf53e8c17c098552285e1", size = 35847826, upload-time = "2026-02-16T10:12:38.949Z" }, - { url = "https://files.pythonhosted.org/packages/54/a5/8cbc83f04aba433ca7b331b38f39e000efd9f0c7ce47128670e737542996/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0b95a3994f015be13c63148fef8832e8a23938128c185ee951c98908a696e0eb", size = 44536859, upload-time = "2026-02-16T10:12:45.467Z" }, - { url = "https://files.pythonhosted.org/packages/36/2e/c0f017c405fcdc252dbccafbe05e36b0d0eb1ea9a958f081e01c6972927f/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4982d71350b1a6e5cfe1af742c53dfb759b11ce14141870d05d9e540d13bc5d1", size = 47614443, upload-time = "2026-02-16T10:12:55.525Z" }, - { url = "https://files.pythonhosted.org/packages/af/6b/2314a78057912f5627afa13ba43809d9d653e6630859618b0fd81a4e0759/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c250248f1fe266db627921c89b47b7c06fee0489ad95b04d50353537d74d6886", size = 48232991, upload-time = "2026-02-16T10:13:04.729Z" }, - { url = "https://files.pythonhosted.org/packages/40/f2/1bcb1d3be3460832ef3370d621142216e15a2c7c62602a4ea19ec240dd64/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f4763b83c11c16e5f4c15601ba6dfa849e20723b46aa2617cb4bffe8768479f", size = 50645077, upload-time = "2026-02-16T10:13:14.147Z" }, - { url = "https://files.pythonhosted.org/packages/eb/3f/b1da7b61cd66566a4d4c8383d376c606d1c34a906c3f1cb35c479f59d1aa/pyarrow-23.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:3a4c85ef66c134161987c17b147d6bffdca4566f9a4c1d81a0a01cdf08414ea5", size = 28234271, upload-time = "2026-02-16T10:14:09.397Z" }, - { url = "https://files.pythonhosted.org/packages/b5/78/07f67434e910a0f7323269be7bfbf58699bd0c1d080b18a1ab49ba943fe8/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:17cd28e906c18af486a499422740298c52d7c6795344ea5002a7720b4eadf16d", size = 34488692, upload-time = "2026-02-16T10:13:21.541Z" }, - { url = "https://files.pythonhosted.org/packages/50/76/34cf7ae93ece1f740a04910d9f7e80ba166b9b4ab9596a953e9e62b90fe1/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:76e823d0e86b4fb5e1cf4a58d293036e678b5a4b03539be933d3b31f9406859f", size = 35964383, upload-time = "2026-02-16T10:13:28.63Z" }, - { url = "https://files.pythonhosted.org/packages/46/90/459b827238936d4244214be7c684e1b366a63f8c78c380807ae25ed92199/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a62e1899e3078bf65943078b3ad2a6ddcacf2373bc06379aac61b1e548a75814", size = 44538119, upload-time = "2026-02-16T10:13:35.506Z" }, - { url = "https://files.pythonhosted.org/packages/28/a1/93a71ae5881e99d1f9de1d4554a87be37da11cd6b152239fb5bd924fdc64/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:df088e8f640c9fae3b1f495b3c64755c4e719091caf250f3a74d095ddf3c836d", size = 47571199, upload-time = "2026-02-16T10:13:42.504Z" }, - { url = "https://files.pythonhosted.org/packages/88/a3/d2c462d4ef313521eaf2eff04d204ac60775263f1fb08c374b543f79f610/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:46718a220d64677c93bc243af1d44b55998255427588e400677d7192671845c7", size = 48259435, upload-time = "2026-02-16T10:13:49.226Z" }, - { url = "https://files.pythonhosted.org/packages/cc/f1/11a544b8c3d38a759eb3fbb022039117fd633e9a7b19e4841cc3da091915/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a09f3876e87f48bc2f13583ab551f0379e5dfb83210391e68ace404181a20690", size = 50629149, upload-time = "2026-02-16T10:13:57.238Z" }, - { url = "https://files.pythonhosted.org/packages/50/f2/c0e76a0b451ffdf0cf788932e182758eb7558953f4f27f1aff8e2518b653/pyarrow-23.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:527e8d899f14bd15b740cd5a54ad56b7f98044955373a17179d5956ddb93d9ce", size = 28365807, upload-time = "2026-02-16T10:14:03.892Z" }, +version = "24.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/13/13e1069b351bdc3881266e11147ffccf687505dbb0ea74036237f5d454a5/pyarrow-24.0.0.tar.gz", hash = "sha256:85fe721a14dd823aca09127acbb06c3ca723efbd436c004f16bca601b04dcc83", size = 1180261, upload-time = "2026-04-21T10:51:25.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6f/d3/a1abf004482026ddc17f4503db227787fa3cfe41ec5091ff20e4fea55e57/pyarrow-24.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:02b001b3ed4723caa44f6cd1af2d5c86aa2cf9971dacc2ffa55b21237713dfba", size = 34976759, upload-time = "2026-04-21T10:48:07.258Z" }, + { url = "https://files.pythonhosted.org/packages/4f/4a/34f0a36d28a2dd32225301b79daad44e243dc1a2bb77d43b60749be255c4/pyarrow-24.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:04920d6a71aabd08a0417709efce97d45ea8e6fb733d9ca9ecffb13c67839f68", size = 36658471, upload-time = "2026-04-21T10:48:13.347Z" }, + { url = "https://files.pythonhosted.org/packages/1f/78/543b94712ae8bb1a6023bcc1acf1a740fbff8286747c289cd9468fced2a5/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a964266397740257f16f7bb2e4f08a0c81454004beab8ff59dd531b73610e9f2", size = 45675981, upload-time = "2026-04-21T10:48:20.201Z" }, + { url = "https://files.pythonhosted.org/packages/84/9f/8fb7c222b100d314137fa40ec050de56cd8c6d957d1cfff685ce72f15b17/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f066b179d68c413374294bc1735f68475457c933258df594443bb9d88ddc2a0", size = 48859172, upload-time = "2026-04-21T10:48:27.541Z" }, + { url = "https://files.pythonhosted.org/packages/a7/d3/1ea72538e6c8b3b475ed78d1049a2c518e655761ea50fe1171fc855fcab7/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1183baeb14c5f587b1ec52831e665718ce632caab84b7cd6b85fd44f96114495", size = 49385733, upload-time = "2026-04-21T10:48:34.7Z" }, + { url = "https://files.pythonhosted.org/packages/c3/be/c3d8b06a1ba35f2260f8e1f771abbee7d5e345c0937aab90675706b1690a/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:806f24b4085453c197a5078218d1ee08783ebbba271badd153d1ae22a3ee804f", size = 51934335, upload-time = "2026-04-21T10:48:42.099Z" }, + { url = "https://files.pythonhosted.org/packages/9c/62/89e07a1e7329d2cde3e3c6994ba0839a24977a2beda8be6005ea3d860b99/pyarrow-24.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:e4505fc6583f7b05ab854934896bcac8253b04ac1171a77dfb73efef92076d91", size = 27271748, upload-time = "2026-04-21T10:49:42.532Z" }, + { url = "https://files.pythonhosted.org/packages/17/1a/cff3a59f80b5b1658549d46611b67163f65e0664431c076ad728bf9d5af4/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:1a4e45017efbf115032e4475ee876d525e0e36c742214fbe405332480ecd6275", size = 35238554, upload-time = "2026-04-21T10:48:48.526Z" }, + { url = "https://files.pythonhosted.org/packages/a8/99/cce0f42a327bfef2c420fb6078a3eb834826e5d6697bf3009fe11d2ad051/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:7986f1fa71cee060ad00758bcc79d3a93bab8559bf978fab9e53472a2e25a17b", size = 36782301, upload-time = "2026-04-21T10:48:55.181Z" }, + { url = "https://files.pythonhosted.org/packages/2a/66/8e560d5ff6793ca29aca213c53eec0dd482dd46cb93b2819e5aab52e4252/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d3e0b61e8efb24ed38898e5cdc5fffa9124be480008d401a1f8071500494ae42", size = 45721929, upload-time = "2026-04-21T10:49:03.676Z" }, + { url = "https://files.pythonhosted.org/packages/27/0c/a26e25505d030716e078d9f16eb74973cbf0b33b672884e9f9da1c83b871/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:55a3bc1e3df3b5567b7d27ef551b2283f0c68a5e86f1cd56abc569da4f31335b", size = 48825365, upload-time = "2026-04-21T10:49:11.714Z" }, + { url = "https://files.pythonhosted.org/packages/5f/eb/771f9ecb0c65e73fe9dccdd1717901b9594f08c4515d000c7c62df573811/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:641f795b361874ac9da5294f8f443dfdbee355cf2bd9e3b8d97aaac2306b9b37", size = 49451819, upload-time = "2026-04-21T10:49:21.474Z" }, + { url = "https://files.pythonhosted.org/packages/48/da/61ae89a88732f5a785646f3ec6125dbb640fa98a540eb2b9889caa561403/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8adc8e6ce5fccf5dc707046ae4914fd537def529709cc0d285d37a7f9cd442ca", size = 51909252, upload-time = "2026-04-21T10:49:31.164Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1a/8dd5cafab7b66573fa91c03d06d213356ad4edd71813aa75e08ce2b3a844/pyarrow-24.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:9b18371ad2f44044b81a8d23bc2d8a9b6a6226dca775e8e16cfee640473d6c5d", size = 27388127, upload-time = "2026-04-21T10:49:37.334Z" }, + { url = "https://files.pythonhosted.org/packages/ad/80/d022a34ff05d2cbedd8ccf841fc1f532ecfa9eb5ed1711b56d0e0ea71fc9/pyarrow-24.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:1cc9057f0319e26333b357e17f3c2c022f1a83739b48a88b25bfd5fa2dc18838", size = 35007997, upload-time = "2026-04-21T10:49:48.796Z" }, + { url = "https://files.pythonhosted.org/packages/1a/ff/f01485fda6f4e5d441afb8dd5e7681e4db18826c1e271852f5d3957d6a80/pyarrow-24.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e6f1278ee4785b6db21229374a1c9e54ec7c549de5d1efc9630b6207de7e170b", size = 36678720, upload-time = "2026-04-21T10:49:55.858Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c2/2d2d5fea814237923f71b36495211f20b43a1576f9a4d6da7e751a64ec6f/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:adbbedc55506cbdabb830890444fb856bfb0060c46c6f8026c6c2f2cf86ae795", size = 45741852, upload-time = "2026-04-21T10:50:04.624Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3a/28ba9c1c1ebdbb5f1b94dfebb46f207e52e6a554b7fe4132540fde29a3a0/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:ae8a1145af31d903fa9bb166824d7abe9b4681a000b0159c9fb99c11bc11ad26", size = 48889852, upload-time = "2026-04-21T10:50:12.293Z" }, + { url = "https://files.pythonhosted.org/packages/df/51/4a389acfd31dca009f8fb82d7f510bb4130f2b3a8e18cf00194d0687d8ac/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d7027eba1df3b2069e2e8d80f644fa0918b68c46432af3d088ddd390d063ecde", size = 49445207, upload-time = "2026-04-21T10:50:20.677Z" }, + { url = "https://files.pythonhosted.org/packages/19/4b/0bab2b23d2ae901b1b9a03c0efd4b2d070256f8ce3fc43f6e58c167b2081/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e56a1ffe9bf7b727432b89104cc0849c21582949dd7bdcb34f17b2001a351a76", size = 51954117, upload-time = "2026-04-21T10:50:29.14Z" }, + { url = "https://files.pythonhosted.org/packages/29/88/f4e9145da0417b3d2c12035a8492b35ff4a3dbc653e614fcfb51d9dedb38/pyarrow-24.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:38be1808cdd068605b787e6ca9119b27eb275a0234e50212c3492331680c3b1e", size = 28001155, upload-time = "2026-04-21T10:51:22.337Z" }, + { url = "https://files.pythonhosted.org/packages/79/4f/46a49a63f43526da895b1a45bbb51d5baf8e4d77159f8528fc3e5490007f/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:418e48ce50a45a6a6c73c454677203a9c75c966cb1e92ca3370959185f197a05", size = 35250387, upload-time = "2026-04-21T10:50:35.552Z" }, + { url = "https://files.pythonhosted.org/packages/a0/da/d5e0cd5ef00796922404806d5f00325cdadc3441ce2c13fe7115f2df9a64/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:2f16197705a230a78270cdd4ea8a1d57e86b2fdcbc34a1f6aebc72e65c986f9a", size = 36797102, upload-time = "2026-04-21T10:50:42.417Z" }, + { url = "https://files.pythonhosted.org/packages/34/c7/5904145b0a593a05236c882933d439b5720f0a145381179063722fbfc123/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:fb24ac194bfc5e86839d7dcd52092ee31e5fe6733fe11f5e3b06ef0812b20072", size = 45745118, upload-time = "2026-04-21T10:50:49.324Z" }, + { url = "https://files.pythonhosted.org/packages/13/d3/cca42fe166d1c6e4d5b80e530b7949104d10e17508a90ae202dac205ce2a/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:9700ebd9a51f5895ce75ff4ac4b3c47a7d4b42bc618be8e713e5d56bacf5f931", size = 48844765, upload-time = "2026-04-21T10:50:55.579Z" }, + { url = "https://files.pythonhosted.org/packages/b0/49/942c3b79878ba928324d1e17c274ed84581db8c0a749b24bcf4cbdf15bd3/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d8ddd2768da81d3ee08cfea9b597f4abb4e8e1dc8ae7e204b608d23a0d3ab699", size = 49471890, upload-time = "2026-04-21T10:51:02.439Z" }, + { url = "https://files.pythonhosted.org/packages/76/97/ff71431000a75d84135a1ace5ca4ba11726a231a8007bbb320a4c54075d5/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:61a3d7eaa97a14768b542f3d284dc6400dd2470d9f080708b13cd46b6ae18136", size = 51932250, upload-time = "2026-04-21T10:51:10.576Z" }, + { url = "https://files.pythonhosted.org/packages/51/be/6f79d55816d5c22557cf27533543d5d70dfe692adfbee4b99f2760674f38/pyarrow-24.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:c91d00057f23b8d353039520dc3a6c09d8608164c692e9f59a175a42b2ae0c19", size = 28131282, upload-time = "2026-04-21T10:51:16.815Z" }, ] [[package]] @@ -2544,7 +2866,7 @@ wheels = [ [[package]] name = "pydantic" -version = "2.13.1" +version = "2.13.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-types" }, @@ -2552,9 +2874,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f3/6b/1353beb3d1cd5cf61cdec5b6f87a9872399de3bc5cae0b7ce07ff4de2ab0/pydantic-2.13.1.tar.gz", hash = "sha256:a0f829b279ddd1e39291133fe2539d2aa46cc6b150c1706a270ff0879e3774d2", size = 843746, upload-time = "2026-04-15T14:57:19.398Z" } +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/81/5a/2225f4c176dbfed0d809e848b50ef08f70e61daa667b7fa14b0d311ae44d/pydantic-2.13.1-py3-none-any.whl", hash = "sha256:9557ecc2806faaf6037f85b1fbd963d01e30511c48085f0d573650fdeaad378a", size = 471917, upload-time = "2026-04-15T14:57:17.277Z" }, + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, ] [package.optional-dependencies] @@ -2564,72 +2886,72 @@ email = [ [[package]] name = "pydantic-core" -version = "2.46.1" +version = "2.46.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a1/93/f97a86a7eb28faa1d038af2fd5d6166418b4433659108a4c311b57128b2d/pydantic_core-2.46.1.tar.gz", hash = "sha256:d408153772d9f298098fb5d620f045bdf0f017af0d5cb6e309ef8c205540caa4", size = 471230, upload-time = "2026-04-15T14:49:34.52Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/d2/bda39bad2f426cb5078e6ad28076614d3926704196efe0d7a2a19a99025d/pydantic_core-2.46.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:cdc8a5762a9c4b9d86e204d555444e3227507c92daba06259ee66595834de47a", size = 2119092, upload-time = "2026-04-15T14:49:50.392Z" }, - { url = "https://files.pythonhosted.org/packages/ee/f3/69631e64d69cb3481494b2bddefe0ddd07771209f74e9106d066f9138c2a/pydantic_core-2.46.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ba381dfe9c85692c566ecb60fa5a77a697a2a8eebe274ec5e4d6ec15fafad799", size = 1951400, upload-time = "2026-04-15T14:51:06.588Z" }, - { url = "https://files.pythonhosted.org/packages/53/1c/21cb3db6ae997df31be8e91f213081f72ffa641cb45c89b8a1986832b1f9/pydantic_core-2.46.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1593d8de98207466dc070118322fef68307a0cc6a5625e7b386f6fdae57f9ab6", size = 1976864, upload-time = "2026-04-15T14:50:54.804Z" }, - { url = "https://files.pythonhosted.org/packages/91/9c/05c819f734318ce5a6ca24da300d93696c105af4adb90494ee571303afd8/pydantic_core-2.46.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8262c74a1af5b0fdf795f5537f7145785a63f9fbf9e15405f547440c30017ed8", size = 2066669, upload-time = "2026-04-15T14:51:42.346Z" }, - { url = "https://files.pythonhosted.org/packages/cb/23/fadddf1c7f2f517f58731aea9b35c914e6005250f08dac9b8e53904cdbaa/pydantic_core-2.46.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b88949a24182e83fbbb3f7ca9b7858d0d37b735700ea91081434b7d37b3b444", size = 2238737, upload-time = "2026-04-15T14:50:45.558Z" }, - { url = "https://files.pythonhosted.org/packages/23/07/0cd4f95cb0359c8b1ec71e89c3777e7932c8dfeb9cd54740289f310aaead/pydantic_core-2.46.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8f3708cd55537aeaf3fd0ea55df0d68d0da51dcb07cbc8508745b34acc4c6e0", size = 2316258, upload-time = "2026-04-15T14:51:08.471Z" }, - { url = "https://files.pythonhosted.org/packages/0c/40/6fc24c3766a19c222a0d60d652b78f0283339d4cd4c173fab06b7ee76571/pydantic_core-2.46.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f79292435fff1d4f0c18d9cfaf214025cc88e4f5104bfaed53f173621da1c743", size = 2097474, upload-time = "2026-04-15T14:49:56.543Z" }, - { url = "https://files.pythonhosted.org/packages/4b/af/f39795d1ce549e35d0841382b9c616ae211caffb88863147369a8d74fba9/pydantic_core-2.46.1-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:a2e607aeb59cf4575bb364470288db3b9a1f0e7415d053a322e3e154c1a0802e", size = 2168383, upload-time = "2026-04-15T14:51:29.269Z" }, - { url = "https://files.pythonhosted.org/packages/e6/32/0d563f74582795779df6cc270c3fc220f49f4daf7860d74a5a6cda8491ff/pydantic_core-2.46.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ec5ca190b75878a9f6ae1fc8f5eb678497934475aef3d93204c9fa01e97370b6", size = 2186182, upload-time = "2026-04-15T14:50:19.097Z" }, - { url = "https://files.pythonhosted.org/packages/5c/07/1c10d5ce312fc4cf86d1e50bdcdbb8ef248409597b099cab1b4bb3a093f7/pydantic_core-2.46.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:1f80535259dcdd517d7b8ca588d5ca24b4f337228e583bebedf7a3adcdf5f721", size = 2187859, upload-time = "2026-04-15T14:49:22.974Z" }, - { url = "https://files.pythonhosted.org/packages/92/01/e1f62d4cb39f0913dbf5c95b9b119ef30ddba9493dff8c2b012f0cdd67dc/pydantic_core-2.46.1-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:24820b3c82c43df61eca30147e42853e6c127d8b868afdc0c162df829e011eb4", size = 2338372, upload-time = "2026-04-15T14:49:53.316Z" }, - { url = "https://files.pythonhosted.org/packages/44/ed/218dfeea6127fb1781a6ceca241ec6edf00e8a8933ff331af2215975a534/pydantic_core-2.46.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:f12794b1dd8ac9fb66619e0b3a0427189f5d5638e55a3de1385121a9b7bf9b39", size = 2384039, upload-time = "2026-04-15T14:53:04.929Z" }, - { url = "https://files.pythonhosted.org/packages/6c/1e/011e763cd059238249fbd5780e0f8d0b04b47f86c8925e22784f3e5fc977/pydantic_core-2.46.1-cp313-cp313-win32.whl", hash = "sha256:9bc09aed935cdf50f09e908923f9efbcca54e9244bd14a5a0e2a6c8d2c21b4e9", size = 1977943, upload-time = "2026-04-15T14:52:17.969Z" }, - { url = "https://files.pythonhosted.org/packages/8c/06/b559a490d3ed106e9b1777b8d5c8112dd8d31716243cd662616f66c1f8ea/pydantic_core-2.46.1-cp313-cp313-win_amd64.whl", hash = "sha256:fac2d6c8615b8b42bee14677861ba09d56ee076ba4a65cfb9c3c3d0cc89042f2", size = 2068729, upload-time = "2026-04-15T14:53:07.288Z" }, - { url = "https://files.pythonhosted.org/packages/9f/52/32a198946e2e19508532aa9da02a61419eb15bd2d96bab57f810f2713e31/pydantic_core-2.46.1-cp313-cp313-win_arm64.whl", hash = "sha256:f978329f12ace9f3cb814a5e44d98bbeced2e36f633132bafa06d2d71332e33e", size = 2029550, upload-time = "2026-04-15T14:52:22.707Z" }, - { url = "https://files.pythonhosted.org/packages/bd/2b/6793fe89ab66cb2d3d6e5768044eab80bba1d0fae8fd904d0a1574712e17/pydantic_core-2.46.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:9917cb61effac7ec0f448ef491ec7584526d2193be84ff981e85cbf18b68c42a", size = 2118110, upload-time = "2026-04-15T14:50:52.947Z" }, - { url = "https://files.pythonhosted.org/packages/d2/87/e9a905ddfcc2fd7bd862b340c02be6ab1f827922822d425513635d0ac774/pydantic_core-2.46.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e749679ca9f8a9d0bff95fb7f6b57bb53f2207fa42ffcc1ec86de7e0029ab89", size = 1948645, upload-time = "2026-04-15T14:51:55.577Z" }, - { url = "https://files.pythonhosted.org/packages/15/23/26e67f86ed62ac9d6f7f3091ee5220bf14b5ac36fb811851d601365ef896/pydantic_core-2.46.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2ecacee70941e233a2dad23f7796a06f86cc10cc2fbd1c97c7dd5b5a79ffa4f", size = 1977576, upload-time = "2026-04-15T14:49:37.58Z" }, - { url = "https://files.pythonhosted.org/packages/b8/78/813c13c0de323d4de54ee2e6fdd69a0271c09ac8dd65a8a000931aa487a5/pydantic_core-2.46.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:647d0a2475b8ed471962eed92fa69145b864942f9c6daa10f95ac70676637ae7", size = 2060358, upload-time = "2026-04-15T14:51:40.087Z" }, - { url = "https://files.pythonhosted.org/packages/09/5e/4caf2a15149271fbd2b4d968899a450853c800b85152abcf54b11531417f/pydantic_core-2.46.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac9cde61965b0697fce6e6cc372df9e1ad93734828aac36e9c1c42a22ad02897", size = 2235980, upload-time = "2026-04-15T14:50:34.535Z" }, - { url = "https://files.pythonhosted.org/packages/c2/c1/a2cdabb5da6f5cb63a3558bcafffc20f790fa14ccffbefbfb1370fadc93f/pydantic_core-2.46.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a2eb0864085f8b641fb3f54a2fb35c58aff24b175b80bc8a945050fcde03204", size = 2316800, upload-time = "2026-04-15T14:52:46.999Z" }, - { url = "https://files.pythonhosted.org/packages/76/fd/19d711e4e9331f9d77f222bffc202bf30ea0d74f6419046376bb82f244c8/pydantic_core-2.46.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b83ce9fede4bc4fb649281d9857f06d30198b8f70168f18b987518d713111572", size = 2101762, upload-time = "2026-04-15T14:49:24.278Z" }, - { url = "https://files.pythonhosted.org/packages/dc/64/ce95625448e1a4e219390a2923fd594f3fa368599c6b42ac71a5df7238c9/pydantic_core-2.46.1-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:cb33192753c60f269d2f4a1db8253c95b0df6e04f2989631a8cc1b0f4f6e2e92", size = 2167737, upload-time = "2026-04-15T14:50:41.637Z" }, - { url = "https://files.pythonhosted.org/packages/ad/31/413572d03ca3e73b408f00f54418b91a8be6401451bc791eaeff210328e5/pydantic_core-2.46.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:96611d51f953f87e1ae97637c01ee596a08b7f494ea00a5afb67ea6547b9f53b", size = 2185658, upload-time = "2026-04-15T14:51:46.799Z" }, - { url = "https://files.pythonhosted.org/packages/36/09/e4f581353bdf3f0c7de8a8b27afd14fc761da29d78146376315a6fedc487/pydantic_core-2.46.1-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:9b176fa55f9107db5e6c86099aa5bfd934f1d3ba6a8b43f714ddeebaed3f42b7", size = 2184154, upload-time = "2026-04-15T14:52:49.629Z" }, - { url = "https://files.pythonhosted.org/packages/1a/a4/d0d52849933f5a4bf1ad9d8da612792f96469b37e286a269e3ee9c60bbb1/pydantic_core-2.46.1-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:79a59f63a4ce4f3330e27e6f3ce281dd1099453b637350e97d7cf24c207cd120", size = 2332379, upload-time = "2026-04-15T14:49:55.009Z" }, - { url = "https://files.pythonhosted.org/packages/30/93/25bfb08fdbef419f73290e573899ce938a327628c34e8f3a4bafeea30126/pydantic_core-2.46.1-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:f200fce071808a385a314b7343f5e3688d7c45746be3d64dc71ee2d3e2a13268", size = 2377964, upload-time = "2026-04-15T14:51:59.649Z" }, - { url = "https://files.pythonhosted.org/packages/15/36/b777766ff83fef1cf97473d64764cd44f38e0d8c269ed06faace9ae17666/pydantic_core-2.46.1-cp314-cp314-win32.whl", hash = "sha256:3a07eccc0559fb9acc26d55b16bf8ebecd7f237c74a9e2c5741367db4e6d8aff", size = 1976450, upload-time = "2026-04-15T14:51:57.665Z" }, - { url = "https://files.pythonhosted.org/packages/7b/4b/4cd19d2437acfc18ca166db5a2067040334991eb862c4ecf2db098c91fbf/pydantic_core-2.46.1-cp314-cp314-win_amd64.whl", hash = "sha256:1706d270309ac7d071ffe393988c471363705feb3d009186e55d17786ada9622", size = 2067750, upload-time = "2026-04-15T14:49:38.941Z" }, - { url = "https://files.pythonhosted.org/packages/7f/a0/490751c0ef8f5b27aae81731859aed1508e72c1a9b5774c6034269db773b/pydantic_core-2.46.1-cp314-cp314-win_arm64.whl", hash = "sha256:22d4e7457ade8af06528012f382bc994a97cc2ce6e119305a70b3deff1e409d6", size = 2021109, upload-time = "2026-04-15T14:50:27.728Z" }, - { url = "https://files.pythonhosted.org/packages/36/3a/2a018968245fffd25d5f1972714121ad309ff2de19d80019ad93494844f9/pydantic_core-2.46.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:607ff9db0b7e2012e7eef78465e69f9a0d7d1c3e7c6a84cf0c4011db0fcc3feb", size = 2111548, upload-time = "2026-04-15T14:52:08.273Z" }, - { url = "https://files.pythonhosted.org/packages/77/5b/4103b6192213217e874e764e5467d2ff10d8873c1147d01fa432ac281880/pydantic_core-2.46.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8cda3eacaea13bd02a1bea7e457cc9fc30b91c5a91245cef9b215140f80dd78c", size = 1926745, upload-time = "2026-04-15T14:50:03.045Z" }, - { url = "https://files.pythonhosted.org/packages/c3/70/602a667cf4be4bec6c3334512b12ae4ea79ce9bfe41dc51be1fd34434453/pydantic_core-2.46.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9493279cdc7997fe19e5ed9b41f30cbc3806bd4722adb402fedb6f6d41bd72a", size = 1965922, upload-time = "2026-04-15T14:51:12.555Z" }, - { url = "https://files.pythonhosted.org/packages/a9/24/06a89ce5323e755b7d2812189f9706b87aaebe49b34d247b380502f7992c/pydantic_core-2.46.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3644e5e10059999202355b6c6616e624909e23773717d8f76deb8a6e2a72328c", size = 2043221, upload-time = "2026-04-15T14:51:18.995Z" }, - { url = "https://files.pythonhosted.org/packages/2c/6e/b1d9ad907d9d76964903903349fd2e33c87db4b993cc44713edcad0fc488/pydantic_core-2.46.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4ad6c9de57683e26c92730991960c0c3571b8053263b042de2d3e105930b2767", size = 2243655, upload-time = "2026-04-15T14:50:10.718Z" }, - { url = "https://files.pythonhosted.org/packages/ef/73/787abfaad51174641abb04c8aa125322279b40ad7ce23c495f5a69f76554/pydantic_core-2.46.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:557ebaa27c7617e7088002318c679a8ce685fa048523417cd1ca52b7f516d955", size = 2295976, upload-time = "2026-04-15T14:53:09.694Z" }, - { url = "https://files.pythonhosted.org/packages/56/0b/b7c5a631b6d5153d4a1ea4923b139aea256dc3bd99c8e6c7b312c7733146/pydantic_core-2.46.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3cd37e39b22b796ba0298fe81e9421dd7b65f97acfbb0fb19b33ffdda7b9a7b4", size = 2103439, upload-time = "2026-04-15T14:50:08.32Z" }, - { url = "https://files.pythonhosted.org/packages/2a/3f/952ee470df69e5674cdec1cbde22331adf643b5cc2ff79f4292d80146ee4/pydantic_core-2.46.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:6689443b59714992e67d62505cdd2f952d6cf1c14cc9fd9aeec6719befc6f23b", size = 2132871, upload-time = "2026-04-15T14:50:24.445Z" }, - { url = "https://files.pythonhosted.org/packages/e3/8b/1dea3b1e683c60c77a60f710215f90f486755962aa8939dbcb7c0f975ac3/pydantic_core-2.46.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6f32c41ca1e3456b5dd691827b7c1433c12d5f0058cc186afbb3615bc07d97b8", size = 2168658, upload-time = "2026-04-15T14:52:24.897Z" }, - { url = "https://files.pythonhosted.org/packages/67/97/32ae283810910d274d5ba9f48f856f5f2f612410b78b249f302d297816f5/pydantic_core-2.46.1-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:88cd1355578852db83954dc36e4f58f299646916da976147c20cf6892ba5dc43", size = 2171184, upload-time = "2026-04-15T14:52:34.854Z" }, - { url = "https://files.pythonhosted.org/packages/a2/57/c9a855527fe56c2072070640221f53095b0b19eaf651f3c77643c9cabbe3/pydantic_core-2.46.1-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:a170fefdb068279a473cc9d34848b85e61d68bfcc2668415b172c5dfc6f213bf", size = 2316573, upload-time = "2026-04-15T14:52:12.871Z" }, - { url = "https://files.pythonhosted.org/packages/37/b3/14c39ffc7399819c5448007c7bcb4e6da5669850cfb7dcbb727594290b48/pydantic_core-2.46.1-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:556a63ff1006934dba4eed7ea31b58274c227e29298ec398e4275eda4b905e95", size = 2378340, upload-time = "2026-04-15T14:51:02.619Z" }, - { url = "https://files.pythonhosted.org/packages/01/55/a37461fbb29c053ea4e62cfc5c2d56425cb5efbef8316e63f6d84ae45718/pydantic_core-2.46.1-cp314-cp314t-win32.whl", hash = "sha256:3b146d8336a995f7d7da6d36e4a779b7e7dff2719ac00a1eb8bd3ded00bec87b", size = 1960843, upload-time = "2026-04-15T14:52:06.103Z" }, - { url = "https://files.pythonhosted.org/packages/22/d7/97e1221197d17a27f768363f87ec061519eeeed15bbd315d2e9d1429ff03/pydantic_core-2.46.1-cp314-cp314t-win_amd64.whl", hash = "sha256:f1bc856c958e6fe9ec071e210afe6feb695f2e2e81fd8d2b102f558d364c4c17", size = 2048696, upload-time = "2026-04-15T14:52:52.154Z" }, - { url = "https://files.pythonhosted.org/packages/19/d5/4eac95255c7d35094b46a32ec1e4d80eac94729c694726ee1d69948bd5f0/pydantic_core-2.46.1-cp314-cp314t-win_arm64.whl", hash = "sha256:21a5bfd8a1aa4de60494cdf66b0c912b1495f26a8899896040021fbd6038d989", size = 2022343, upload-time = "2026-04-15T14:49:49.036Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, + { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, + { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, + { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, ] [[package]] name = "pydantic-settings" -version = "2.13.1" +version = "2.14.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "python-dotenv" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/52/6d/fffca34caecc4a3f97bda81b2098da5e8ab7efc9a66e819074a11955d87e/pydantic_settings-2.13.1.tar.gz", hash = "sha256:b4c11847b15237fb0171e1462bf540e294affb9b86db4d9aa5c01730bdbe4025", size = 223826, upload-time = "2026-02-19T13:45:08.055Z" } +sdist = { url = "https://files.pythonhosted.org/packages/07/60/1d1e59c9c90d54591469ada7d268251f71c24bdb765f1a8a832cee8c6653/pydantic_settings-2.14.1.tar.gz", hash = "sha256:e874d3bec7e787b0c9958277956ed9b4dd5de6a80e162188fdaff7c5e26fd5fa", size = 235551, upload-time = "2026-05-08T13:40:06.542Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/00/4b/ccc026168948fec4f7555b9164c724cf4125eac006e176541483d2c959be/pydantic_settings-2.13.1-py3-none-any.whl", hash = "sha256:d56fd801823dbeae7f0975e1f8c8e25c258eb75d278ea7abb5d9cebb01b56237", size = 58929, upload-time = "2026-02-19T13:45:06.034Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8d/f1af3832f5e6eb13ba94ee809e72b8ecb5eef226d27ee0bef7d963d943c7/pydantic_settings-2.14.1-py3-none-any.whl", hash = "sha256:6e3c7edfd8277687cdc598f56e5cff0e9bfff0910a3749deaa8d4401c3a2b9de", size = 60964, upload-time = "2026-05-08T13:40:04.958Z" }, ] [[package]] @@ -2641,6 +2963,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] +[[package]] +name = "pyjwt" +version = "2.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + [[package]] name = "pyspark" version = "4.1.1" @@ -2663,15 +2999,15 @@ connect = [ [[package]] name = "pystow" -version = "0.8.3" +version = "0.8.14" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/02/74/6d3ca88e7f10f5d45ca54cdca41e2c6ebf312dd6a3b970e98da19bbf09e2/pystow-0.8.3.tar.gz", hash = "sha256:90d927cfaa6172aaa0677005763b32f98ee5d9e0c4f243fc02943a90c08f2b98", size = 51566, upload-time = "2026-03-27T20:06:29.829Z" } +sdist = { url = "https://files.pythonhosted.org/packages/40/43/40dd4fa249d388fcb3044e617e3b45c0457602672d6a9fc0b51aab191a62/pystow-0.8.14.tar.gz", hash = "sha256:09b2b2aa8f3177cc79689ccc40dac96724b03a614335a6100928f5ddb163f2b9", size = 53751, upload-time = "2026-05-20T14:53:59.694Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/32/cb/25fc0914ad183958131f6f549a200b892225b3fad9dcb251062493204e42/pystow-0.8.3-py3-none-any.whl", hash = "sha256:022cc481b77d02b2df746f778a6d5014f3ed7b3abed96aacf46bc4333d61d2b6", size = 58588, upload-time = "2026-03-27T20:06:28.29Z" }, + { url = "https://files.pythonhosted.org/packages/db/cb/4a2424c2736b4d7752f18324e44434b87f2158e42832a020a4b4e24501b3/pystow-0.8.14-py3-none-any.whl", hash = "sha256:8f087aaff834f4300017ea604ab633b1b1e77919e27107add430288be5ca57f9", size = 61041, upload-time = "2026-05-20T14:54:01.36Z" }, ] [[package]] @@ -2763,6 +3099,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, ] +[[package]] +name = "python-multipart" +version = "0.0.29" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/fe/70bd71a6738b09a0bdf6480ca6436b167469ca4578b2a0efbe390b4b0e70/python_multipart-0.0.29.tar.gz", hash = "sha256:643e93849196645e2dbdd81a0f8829a23123ad7f797a84a364c6fb3563f18904", size = 45678, upload-time = "2026-05-17T17:29:47.654Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/cb/769cfc37177252872a45a71f3fbdde9d51b471a3f3c14bfe95dde3407386/python_multipart-0.0.29-py3-none-any.whl", hash = "sha256:2ddcc971cef266225f54f552d8fa10bcfbb1f14446caec199060daac59ff2d69", size = 29640, upload-time = "2026-05-17T17:29:45.69Z" }, +] + [[package]] name = "python-slugify" version = "8.0.4" @@ -2777,11 +3122,11 @@ wheels = [ [[package]] name = "pytz" -version = "2026.1.post1" +version = "2026.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/56/db/b8721d71d945e6a8ac63c0fc900b2067181dbb50805958d4d4661cf7d277/pytz-2026.1.post1.tar.gz", hash = "sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1", size = 321088, upload-time = "2026-03-03T07:47:50.683Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/46/dd499ec9038423421951e4fad73051febaa13d2df82b4064f87af8b8c0c3/pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a", size = 320861, upload-time = "2026-05-04T01:35:29.667Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/10/99/781fe0c827be2742bcc775efefccb3b048a3a9c6ce9aec0cbf4a101677e5/pytz-2026.1.post1-py2.py3-none-any.whl", hash = "sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a", size = 510489, upload-time = "2026-03-03T07:47:49.167Z" }, + { url = "https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126", size = 510141, upload-time = "2026-05-04T01:35:27.408Z" }, ] [[package]] @@ -2833,6 +3178,49 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "pyzmq" +version = "27.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "implementation_name == 'pypy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/0b/3c9baedbdf613ecaa7aa07027780b8867f57b6293b6ee50de316c9f3222b/pyzmq-27.1.0.tar.gz", hash = "sha256:ac0765e3d44455adb6ddbf4417dcce460fc40a05978c08efdf2948072f6db540", size = 281750, upload-time = "2025-09-08T23:10:18.157Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/e7/038aab64a946d535901103da16b953c8c9cc9c961dadcbf3609ed6428d23/pyzmq-27.1.0-cp312-abi3-macosx_10_15_universal2.whl", hash = "sha256:452631b640340c928fa343801b0d07eb0c3789a5ffa843f6e1a9cee0ba4eb4fc", size = 1306279, upload-time = "2025-09-08T23:08:03.807Z" }, + { url = "https://files.pythonhosted.org/packages/e8/5e/c3c49fdd0f535ef45eefcc16934648e9e59dace4a37ee88fc53f6cd8e641/pyzmq-27.1.0-cp312-abi3-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1c179799b118e554b66da67d88ed66cd37a169f1f23b5d9f0a231b4e8d44a113", size = 895645, upload-time = "2025-09-08T23:08:05.301Z" }, + { url = "https://files.pythonhosted.org/packages/f8/e5/b0b2504cb4e903a74dcf1ebae157f9e20ebb6ea76095f6cfffea28c42ecd/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3837439b7f99e60312f0c926a6ad437b067356dc2bc2ec96eb395fd0fe804233", size = 652574, upload-time = "2025-09-08T23:08:06.828Z" }, + { url = "https://files.pythonhosted.org/packages/f8/9b/c108cdb55560eaf253f0cbdb61b29971e9fb34d9c3499b0e96e4e60ed8a5/pyzmq-27.1.0-cp312-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43ad9a73e3da1fab5b0e7e13402f0b2fb934ae1c876c51d0afff0e7c052eca31", size = 840995, upload-time = "2025-09-08T23:08:08.396Z" }, + { url = "https://files.pythonhosted.org/packages/c2/bb/b79798ca177b9eb0825b4c9998c6af8cd2a7f15a6a1a4272c1d1a21d382f/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0de3028d69d4cdc475bfe47a6128eb38d8bc0e8f4d69646adfbcd840facbac28", size = 1642070, upload-time = "2025-09-08T23:08:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/9c/80/2df2e7977c4ede24c79ae39dcef3899bfc5f34d1ca7a5b24f182c9b7a9ca/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_i686.whl", hash = "sha256:cf44a7763aea9298c0aa7dbf859f87ed7012de8bda0f3977b6fb1d96745df856", size = 2021121, upload-time = "2025-09-08T23:08:11.907Z" }, + { url = "https://files.pythonhosted.org/packages/46/bd/2d45ad24f5f5ae7e8d01525eb76786fa7557136555cac7d929880519e33a/pyzmq-27.1.0-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f30f395a9e6fbca195400ce833c731e7b64c3919aa481af4d88c3759e0cb7496", size = 1878550, upload-time = "2025-09-08T23:08:13.513Z" }, + { url = "https://files.pythonhosted.org/packages/e6/2f/104c0a3c778d7c2ab8190e9db4f62f0b6957b53c9d87db77c284b69f33ea/pyzmq-27.1.0-cp312-abi3-win32.whl", hash = "sha256:250e5436a4ba13885494412b3da5d518cd0d3a278a1ae640e113c073a5f88edd", size = 559184, upload-time = "2025-09-08T23:08:15.163Z" }, + { url = "https://files.pythonhosted.org/packages/fc/7f/a21b20d577e4100c6a41795842028235998a643b1ad406a6d4163ea8f53e/pyzmq-27.1.0-cp312-abi3-win_amd64.whl", hash = "sha256:9ce490cf1d2ca2ad84733aa1d69ce6855372cb5ce9223802450c9b2a7cba0ccf", size = 619480, upload-time = "2025-09-08T23:08:17.192Z" }, + { url = "https://files.pythonhosted.org/packages/78/c2/c012beae5f76b72f007a9e91ee9401cb88c51d0f83c6257a03e785c81cc2/pyzmq-27.1.0-cp312-abi3-win_arm64.whl", hash = "sha256:75a2f36223f0d535a0c919e23615fc85a1e23b71f40c7eb43d7b1dedb4d8f15f", size = 552993, upload-time = "2025-09-08T23:08:18.926Z" }, + { url = "https://files.pythonhosted.org/packages/60/cb/84a13459c51da6cec1b7b1dc1a47e6db6da50b77ad7fd9c145842750a011/pyzmq-27.1.0-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:93ad4b0855a664229559e45c8d23797ceac03183c7b6f5b4428152a6b06684a5", size = 1122436, upload-time = "2025-09-08T23:08:20.801Z" }, + { url = "https://files.pythonhosted.org/packages/dc/b6/94414759a69a26c3dd674570a81813c46a078767d931a6c70ad29fc585cb/pyzmq-27.1.0-cp313-cp313-android_24_x86_64.whl", hash = "sha256:fbb4f2400bfda24f12f009cba62ad5734148569ff4949b1b6ec3b519444342e6", size = 1156301, upload-time = "2025-09-08T23:08:22.47Z" }, + { url = "https://files.pythonhosted.org/packages/a5/ad/15906493fd40c316377fd8a8f6b1f93104f97a752667763c9b9c1b71d42d/pyzmq-27.1.0-cp313-cp313t-macosx_10_15_universal2.whl", hash = "sha256:e343d067f7b151cfe4eb3bb796a7752c9d369eed007b91231e817071d2c2fec7", size = 1341197, upload-time = "2025-09-08T23:08:24.286Z" }, + { url = "https://files.pythonhosted.org/packages/14/1d/d343f3ce13db53a54cb8946594e567410b2125394dafcc0268d8dda027e0/pyzmq-27.1.0-cp313-cp313t-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:08363b2011dec81c354d694bdecaef4770e0ae96b9afea70b3f47b973655cc05", size = 897275, upload-time = "2025-09-08T23:08:26.063Z" }, + { url = "https://files.pythonhosted.org/packages/69/2d/d83dd6d7ca929a2fc67d2c3005415cdf322af7751d773524809f9e585129/pyzmq-27.1.0-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d54530c8c8b5b8ddb3318f481297441af102517602b569146185fa10b63f4fa9", size = 660469, upload-time = "2025-09-08T23:08:27.623Z" }, + { url = "https://files.pythonhosted.org/packages/3e/cd/9822a7af117f4bc0f1952dbe9ef8358eb50a24928efd5edf54210b850259/pyzmq-27.1.0-cp313-cp313t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f3afa12c392f0a44a2414056d730eebc33ec0926aae92b5ad5cf26ebb6cc128", size = 847961, upload-time = "2025-09-08T23:08:29.672Z" }, + { url = "https://files.pythonhosted.org/packages/9a/12/f003e824a19ed73be15542f172fd0ec4ad0b60cf37436652c93b9df7c585/pyzmq-27.1.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c65047adafe573ff023b3187bb93faa583151627bc9c51fc4fb2c561ed689d39", size = 1650282, upload-time = "2025-09-08T23:08:31.349Z" }, + { url = "https://files.pythonhosted.org/packages/d5/4a/e82d788ed58e9a23995cee70dbc20c9aded3d13a92d30d57ec2291f1e8a3/pyzmq-27.1.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:90e6e9441c946a8b0a667356f7078d96411391a3b8f80980315455574177ec97", size = 2024468, upload-time = "2025-09-08T23:08:33.543Z" }, + { url = "https://files.pythonhosted.org/packages/d9/94/2da0a60841f757481e402b34bf4c8bf57fa54a5466b965de791b1e6f747d/pyzmq-27.1.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:add071b2d25f84e8189aaf0882d39a285b42fa3853016ebab234a5e78c7a43db", size = 1885394, upload-time = "2025-09-08T23:08:35.51Z" }, + { url = "https://files.pythonhosted.org/packages/4f/6f/55c10e2e49ad52d080dc24e37adb215e5b0d64990b57598abc2e3f01725b/pyzmq-27.1.0-cp313-cp313t-win32.whl", hash = "sha256:7ccc0700cfdf7bd487bea8d850ec38f204478681ea02a582a8da8171b7f90a1c", size = 574964, upload-time = "2025-09-08T23:08:37.178Z" }, + { url = "https://files.pythonhosted.org/packages/87/4d/2534970ba63dd7c522d8ca80fb92777f362c0f321900667c615e2067cb29/pyzmq-27.1.0-cp313-cp313t-win_amd64.whl", hash = "sha256:8085a9fba668216b9b4323be338ee5437a235fe275b9d1610e422ccc279733e2", size = 641029, upload-time = "2025-09-08T23:08:40.595Z" }, + { url = "https://files.pythonhosted.org/packages/f6/fa/f8aea7a28b0641f31d40dea42d7ef003fded31e184ef47db696bc74cd610/pyzmq-27.1.0-cp313-cp313t-win_arm64.whl", hash = "sha256:6bb54ca21bcfe361e445256c15eedf083f153811c37be87e0514934d6913061e", size = 561541, upload-time = "2025-09-08T23:08:42.668Z" }, + { url = "https://files.pythonhosted.org/packages/87/45/19efbb3000956e82d0331bafca5d9ac19ea2857722fa2caacefb6042f39d/pyzmq-27.1.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:ce980af330231615756acd5154f29813d553ea555485ae712c491cd483df6b7a", size = 1341197, upload-time = "2025-09-08T23:08:44.973Z" }, + { url = "https://files.pythonhosted.org/packages/48/43/d72ccdbf0d73d1343936296665826350cb1e825f92f2db9db3e61c2162a2/pyzmq-27.1.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1779be8c549e54a1c38f805e56d2a2e5c009d26de10921d7d51cfd1c8d4632ea", size = 897175, upload-time = "2025-09-08T23:08:46.601Z" }, + { url = "https://files.pythonhosted.org/packages/2f/2e/a483f73a10b65a9ef0161e817321d39a770b2acf8bcf3004a28d90d14a94/pyzmq-27.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7200bb0f03345515df50d99d3db206a0a6bee1955fbb8c453c76f5bf0e08fb96", size = 660427, upload-time = "2025-09-08T23:08:48.187Z" }, + { url = "https://files.pythonhosted.org/packages/f5/d2/5f36552c2d3e5685abe60dfa56f91169f7a2d99bbaf67c5271022ab40863/pyzmq-27.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01c0e07d558b06a60773744ea6251f769cd79a41a97d11b8bf4ab8f034b0424d", size = 847929, upload-time = "2025-09-08T23:08:49.76Z" }, + { url = "https://files.pythonhosted.org/packages/c4/2a/404b331f2b7bf3198e9945f75c4c521f0c6a3a23b51f7a4a401b94a13833/pyzmq-27.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:80d834abee71f65253c91540445d37c4c561e293ba6e741b992f20a105d69146", size = 1650193, upload-time = "2025-09-08T23:08:51.7Z" }, + { url = "https://files.pythonhosted.org/packages/1c/0b/f4107e33f62a5acf60e3ded67ed33d79b4ce18de432625ce2fc5093d6388/pyzmq-27.1.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:544b4e3b7198dde4a62b8ff6685e9802a9a1ebf47e77478a5eb88eca2a82f2fd", size = 2024388, upload-time = "2025-09-08T23:08:53.393Z" }, + { url = "https://files.pythonhosted.org/packages/0d/01/add31fe76512642fd6e40e3a3bd21f4b47e242c8ba33efb6809e37076d9b/pyzmq-27.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cedc4c68178e59a4046f97eca31b148ddcf51e88677de1ef4e78cf06c5376c9a", size = 1885316, upload-time = "2025-09-08T23:08:55.702Z" }, + { url = "https://files.pythonhosted.org/packages/c4/59/a5f38970f9bf07cee96128de79590bb354917914a9be11272cfc7ff26af0/pyzmq-27.1.0-cp314-cp314t-win32.whl", hash = "sha256:1f0b2a577fd770aa6f053211a55d1c47901f4d537389a034c690291485e5fe92", size = 587472, upload-time = "2025-09-08T23:08:58.18Z" }, + { url = "https://files.pythonhosted.org/packages/70/d8/78b1bad170f93fcf5e3536e70e8fadac55030002275c9a29e8f5719185de/pyzmq-27.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:19c9468ae0437f8074af379e986c5d3d7d7bfe033506af442e8c879732bedbe0", size = 661401, upload-time = "2025-09-08T23:08:59.802Z" }, + { url = "https://files.pythonhosted.org/packages/81/d6/4bfbb40c9a0b42fc53c7cf442f6385db70b40f74a783130c5d0a5aa62228/pyzmq-27.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:dc5dbf68a7857b59473f7df42650c621d7e8923fb03fa74a526890f4d33cc4d7", size = 575170, upload-time = "2025-09-08T23:09:01.418Z" }, +] + [[package]] name = "referencing" version = "0.37.0" @@ -2848,74 +3236,74 @@ wheels = [ [[package]] name = "regex" -version = "2026.4.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cb/0e/3a246dbf05666918bd3664d9d787f84a9108f6f43cc953a077e4a7dfdb7e/regex-2026.4.4.tar.gz", hash = "sha256:e08270659717f6973523ce3afbafa53515c4dc5dcad637dc215b6fd50f689423", size = 416000, upload-time = "2026-04-03T20:56:28.155Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/83/c4373bc5f31f2cf4b66f9b7c31005bd87fe66f0dce17701f7db4ee79ee29/regex-2026.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:62f5519042c101762509b1d717b45a69c0139d60414b3c604b81328c01bd1943", size = 490273, upload-time = "2026-04-03T20:54:11.202Z" }, - { url = "https://files.pythonhosted.org/packages/46/f8/fe62afbcc3cf4ad4ac9adeaafd98aa747869ae12d3e8e2ac293d0593c435/regex-2026.4.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3790ba9fb5dd76715a7afe34dbe603ba03f8820764b1dc929dd08106214ed031", size = 291954, upload-time = "2026-04-03T20:54:13.412Z" }, - { url = "https://files.pythonhosted.org/packages/5a/92/4712b9fe6a33d232eeb1c189484b80c6c4b8422b90e766e1195d6e758207/regex-2026.4.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8fae3c6e795d7678963f2170152b0d892cf6aee9ee8afc8c45e6be38d5107fe7", size = 289487, upload-time = "2026-04-03T20:54:15.824Z" }, - { url = "https://files.pythonhosted.org/packages/88/2c/f83b93f85e01168f1070f045a42d4c937b69fdb8dd7ae82d307253f7e36e/regex-2026.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:298c3ec2d53225b3bf91142eb9691025bab610e0c0c51592dde149db679b3d17", size = 796646, upload-time = "2026-04-03T20:54:18.229Z" }, - { url = "https://files.pythonhosted.org/packages/df/55/61a2e17bf0c4dc57e11caf8dd11771280d8aaa361785f9e3bc40d653f4a7/regex-2026.4.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e9638791082eaf5b3ac112c587518ee78e083a11c4b28012d8fe2a0f536dfb17", size = 865904, upload-time = "2026-04-03T20:54:20.019Z" }, - { url = "https://files.pythonhosted.org/packages/45/32/1ac8ed1b5a346b5993a3d256abe0a0f03b0b73c8cc88d928537368ac65b6/regex-2026.4.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ae3e764bd4c5ff55035dc82a8d49acceb42a5298edf6eb2fc4d328ee5dd7afae", size = 912304, upload-time = "2026-04-03T20:54:22.403Z" }, - { url = "https://files.pythonhosted.org/packages/26/47/2ee5c613ab546f0eddebf9905d23e07beb933416b1246c2d8791d01979b4/regex-2026.4.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ffa81f81b80047ba89a3c69ae6a0f78d06f4a42ce5126b0eb2a0a10ad44e0b2e", size = 801126, upload-time = "2026-04-03T20:54:24.308Z" }, - { url = "https://files.pythonhosted.org/packages/75/cd/41dacd129ca9fd20bd7d02f83e0fad83e034ac8a084ec369c90f55ef37e2/regex-2026.4.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f56ebf9d70305307a707911b88469213630aba821e77de7d603f9d2f0730687d", size = 776772, upload-time = "2026-04-03T20:54:26.319Z" }, - { url = "https://files.pythonhosted.org/packages/89/6d/5af0b588174cb5f46041fa7dd64d3fd5cd2fe51f18766703d1edc387f324/regex-2026.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:773d1dfd652bbffb09336abf890bfd64785c7463716bf766d0eb3bc19c8b7f27", size = 785228, upload-time = "2026-04-03T20:54:28.387Z" }, - { url = "https://files.pythonhosted.org/packages/b7/3b/f5a72b7045bd59575fc33bf1345f156fcfd5a8484aea6ad84b12c5a82114/regex-2026.4.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d51d20befd5275d092cdffba57ded05f3c436317ee56466c8928ac32d960edaf", size = 860032, upload-time = "2026-04-03T20:54:30.641Z" }, - { url = "https://files.pythonhosted.org/packages/39/a4/72a317003d6fcd7a573584a85f59f525dfe8f67e355ca74eb6b53d66a5e2/regex-2026.4.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:0a51cdb3c1e9161154f976cb2bef9894bc063ac82f31b733087ffb8e880137d0", size = 765714, upload-time = "2026-04-03T20:54:32.789Z" }, - { url = "https://files.pythonhosted.org/packages/25/1e/5672e16f34dbbcb2560cc7e6a2fbb26dfa8b270711e730101da4423d3973/regex-2026.4.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ae5266a82596114e41fb5302140e9630204c1b5f325c770bec654b95dd54b0aa", size = 852078, upload-time = "2026-04-03T20:54:34.546Z" }, - { url = "https://files.pythonhosted.org/packages/f7/0d/c813f0af7c6cc7ed7b9558bac2e5120b60ad0fa48f813e4d4bd55446f214/regex-2026.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c882cd92ec68585e9c1cf36c447ec846c0d94edd706fe59e0c198e65822fd23b", size = 789181, upload-time = "2026-04-03T20:54:36.642Z" }, - { url = "https://files.pythonhosted.org/packages/ea/6d/a344608d1adbd2a95090ddd906cec09a11be0e6517e878d02a5123e0917f/regex-2026.4.4-cp313-cp313-win32.whl", hash = "sha256:05568c4fbf3cb4fa9e28e3af198c40d3237cf6041608a9022285fe567ec3ad62", size = 266690, upload-time = "2026-04-03T20:54:38.343Z" }, - { url = "https://files.pythonhosted.org/packages/31/07/54049f89b46235ca6f45cd6c88668a7050e77d4a15555e47dd40fde75263/regex-2026.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:3384df51ed52db0bea967e21458ab0a414f67cdddfd94401688274e55147bb81", size = 277733, upload-time = "2026-04-03T20:54:40.11Z" }, - { url = "https://files.pythonhosted.org/packages/0e/21/61366a8e20f4d43fb597708cac7f0e2baadb491ecc9549b4980b2be27d16/regex-2026.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:acd38177bd2c8e69a411d6521760806042e244d0ef94e2dd03ecdaa8a3c99427", size = 270565, upload-time = "2026-04-03T20:54:41.883Z" }, - { url = "https://files.pythonhosted.org/packages/f1/1e/3a2b9672433bef02f5d39aa1143ca2c08f311c1d041c464a42be9ae648dc/regex-2026.4.4-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:f94a11a9d05afcfcfa640e096319720a19cc0c9f7768e1a61fceee6a3afc6c7c", size = 494126, upload-time = "2026-04-03T20:54:43.602Z" }, - { url = "https://files.pythonhosted.org/packages/4e/4b/c132a4f4fe18ad3340d89fcb56235132b69559136036b845be3c073142ed/regex-2026.4.4-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:36bcb9d6d1307ab629edc553775baada2aefa5c50ccc0215fbfd2afcfff43141", size = 293882, upload-time = "2026-04-03T20:54:45.41Z" }, - { url = "https://files.pythonhosted.org/packages/f4/5f/eaa38092ce7a023656280f2341dbbd4ad5f05d780a70abba7bb4f4bea54c/regex-2026.4.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261c015b3e2ed0919157046d768774ecde57f03d8fa4ba78d29793447f70e717", size = 292334, upload-time = "2026-04-03T20:54:47.051Z" }, - { url = "https://files.pythonhosted.org/packages/5f/f6/dd38146af1392dac33db7074ab331cec23cced3759167735c42c5460a243/regex-2026.4.4-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c228cf65b4a54583763645dcd73819b3b381ca8b4bb1b349dee1c135f4112c07", size = 811691, upload-time = "2026-04-03T20:54:49.074Z" }, - { url = "https://files.pythonhosted.org/packages/7a/f0/dc54c2e69f5eeec50601054998ec3690d5344277e782bd717e49867c1d29/regex-2026.4.4-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:dd2630faeb6876fb0c287f664d93ddce4d50cd46c6e88e60378c05c9047e08ca", size = 871227, upload-time = "2026-04-03T20:54:51.035Z" }, - { url = "https://files.pythonhosted.org/packages/a1/af/cb16bd5dc61621e27df919a4449bbb7e5a1034c34d307e0a706e9cc0f3e3/regex-2026.4.4-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6a50ab11b7779b849472337191f3a043e27e17f71555f98d0092fa6d73364520", size = 917435, upload-time = "2026-04-03T20:54:52.994Z" }, - { url = "https://files.pythonhosted.org/packages/5c/71/8b260897f22996b666edd9402861668f45a2ca259f665ac029e6104a2d7d/regex-2026.4.4-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0734f63afe785138549fbe822a8cfeaccd1bae814c5057cc0ed5b9f2de4fc883", size = 816358, upload-time = "2026-04-03T20:54:54.884Z" }, - { url = "https://files.pythonhosted.org/packages/1c/60/775f7f72a510ef238254906c2f3d737fc80b16ca85f07d20e318d2eea894/regex-2026.4.4-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c4ee50606cb1967db7e523224e05f32089101945f859928e65657a2cbb3d278b", size = 785549, upload-time = "2026-04-03T20:54:57.01Z" }, - { url = "https://files.pythonhosted.org/packages/58/42/34d289b3627c03cf381e44da534a0021664188fa49ba41513da0b4ec6776/regex-2026.4.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6c1818f37be3ca02dcb76d63f2c7aaba4b0dc171b579796c6fbe00148dfec6b1", size = 801364, upload-time = "2026-04-03T20:54:58.981Z" }, - { url = "https://files.pythonhosted.org/packages/fc/20/f6ecf319b382a8f1ab529e898b222c3f30600fcede7834733c26279e7465/regex-2026.4.4-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:f5bfc2741d150d0be3e4a0401a5c22b06e60acb9aa4daa46d9e79a6dcd0f135b", size = 866221, upload-time = "2026-04-03T20:55:00.88Z" }, - { url = "https://files.pythonhosted.org/packages/92/6a/9f16d3609d549bd96d7a0b2aee1625d7512ba6a03efc01652149ef88e74d/regex-2026.4.4-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:504ffa8a03609a087cad81277a629b6ce884b51a24bd388a7980ad61748618ff", size = 772530, upload-time = "2026-04-03T20:55:03.213Z" }, - { url = "https://files.pythonhosted.org/packages/fa/f6/aa9768bc96a4c361ac96419fbaf2dcdc33970bb813df3ba9b09d5d7b6d96/regex-2026.4.4-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:70aadc6ff12e4b444586e57fc30771f86253f9f0045b29016b9605b4be5f7dfb", size = 856989, upload-time = "2026-04-03T20:55:05.087Z" }, - { url = "https://files.pythonhosted.org/packages/4d/b4/c671db3556be2473ae3e4bb7a297c518d281452871501221251ea4ecba57/regex-2026.4.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f4f83781191007b6ef43b03debc35435f10cad9b96e16d147efe84a1d48bdde4", size = 803241, upload-time = "2026-04-03T20:55:07.162Z" }, - { url = "https://files.pythonhosted.org/packages/2a/5c/83e3b1d89fa4f6e5a1bc97b4abd4a9a97b3c1ac7854164f694f5f0ba98a0/regex-2026.4.4-cp313-cp313t-win32.whl", hash = "sha256:e014a797de43d1847df957c0a2a8e861d1c17547ee08467d1db2c370b7568baa", size = 269921, upload-time = "2026-04-03T20:55:09.62Z" }, - { url = "https://files.pythonhosted.org/packages/28/07/077c387121f42cdb4d92b1301133c0d93b5709d096d1669ab847dda9fe2e/regex-2026.4.4-cp313-cp313t-win_amd64.whl", hash = "sha256:b15b88b0d52b179712632832c1d6e58e5774f93717849a41096880442da41ab0", size = 281240, upload-time = "2026-04-03T20:55:11.521Z" }, - { url = "https://files.pythonhosted.org/packages/9d/22/ead4a4abc7c59a4d882662aa292ca02c8b617f30b6e163bc1728879e9353/regex-2026.4.4-cp313-cp313t-win_arm64.whl", hash = "sha256:586b89cdadf7d67bf86ae3342a4dcd2b8d70a832d90c18a0ae955105caf34dbe", size = 272440, upload-time = "2026-04-03T20:55:13.365Z" }, - { url = "https://files.pythonhosted.org/packages/f0/f5/ed97c2dc47b5fbd4b73c0d7d75f9ebc8eca139f2bbef476bba35f28c0a77/regex-2026.4.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:2da82d643fa698e5e5210e54af90181603d5853cf469f5eedf9bfc8f59b4b8c7", size = 490343, upload-time = "2026-04-03T20:55:15.241Z" }, - { url = "https://files.pythonhosted.org/packages/80/e9/de4828a7385ec166d673a5790ad06ac48cdaa98bc0960108dd4b9cc1aef7/regex-2026.4.4-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:54a1189ad9d9357760557c91103d5e421f0a2dabe68a5cdf9103d0dcf4e00752", size = 291909, upload-time = "2026-04-03T20:55:17.558Z" }, - { url = "https://files.pythonhosted.org/packages/b4/d6/5cfbfc97f3201a4d24b596a77957e092030dcc4205894bc035cedcfce62f/regex-2026.4.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:76d67d5afb1fe402d10a6403bae668d000441e2ab115191a804287d53b772951", size = 289692, upload-time = "2026-04-03T20:55:20.561Z" }, - { url = "https://files.pythonhosted.org/packages/8e/ac/f2212d9fd56fe897e36d0110ba30ba2d247bd6410c5bd98499c7e5a1e1f2/regex-2026.4.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e7cd3e4ee8d80447a83bbc9ab0c8459781fa77087f856c3e740d7763be0df27f", size = 796979, upload-time = "2026-04-03T20:55:22.56Z" }, - { url = "https://files.pythonhosted.org/packages/c9/e3/a016c12675fbac988a60c7e1c16e67823ff0bc016beb27bd7a001dbdabc6/regex-2026.4.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e19e18c568d2866d8b6a6dfad823db86193503f90823a8f66689315ba28fbe8", size = 866744, upload-time = "2026-04-03T20:55:24.646Z" }, - { url = "https://files.pythonhosted.org/packages/af/a4/0b90ca4cf17adc3cb43de80ec71018c37c88ad64987e8d0d481a95ca60b5/regex-2026.4.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7698a6f38730fd1385d390d1ed07bb13dce39aa616aca6a6d89bea178464b9a4", size = 911613, upload-time = "2026-04-03T20:55:27.033Z" }, - { url = "https://files.pythonhosted.org/packages/8e/3b/2b3dac0b82d41ab43aa87c6ecde63d71189d03fe8854b8ca455a315edac3/regex-2026.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:173a66f3651cdb761018078e2d9487f4cf971232c990035ec0eb1cdc6bf929a9", size = 800551, upload-time = "2026-04-03T20:55:29.532Z" }, - { url = "https://files.pythonhosted.org/packages/25/fe/5365eb7aa0e753c4b5957815c321519ecab033c279c60e1b1ae2367fa810/regex-2026.4.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fa7922bbb2cc84fa062d37723f199d4c0cd200245ce269c05db82d904db66b83", size = 776911, upload-time = "2026-04-03T20:55:31.526Z" }, - { url = "https://files.pythonhosted.org/packages/aa/b3/7fb0072156bba065e3b778a7bc7b0a6328212be5dd6a86fd207e0c4f2dab/regex-2026.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:59f67cd0a0acaf0e564c20bbd7f767286f23e91e2572c5703bf3e56ea7557edb", size = 785751, upload-time = "2026-04-03T20:55:33.797Z" }, - { url = "https://files.pythonhosted.org/packages/02/1a/9f83677eb699273e56e858f7bd95acdbee376d42f59e8bfca2fd80d79df3/regex-2026.4.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:475e50f3f73f73614f7cba5524d6de49dee269df00272a1b85e3d19f6d498465", size = 860484, upload-time = "2026-04-03T20:55:35.745Z" }, - { url = "https://files.pythonhosted.org/packages/3b/7a/93937507b61cfcff8b4c5857f1b452852b09f741daa9acae15c971d8554e/regex-2026.4.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a1c0c7d67b64d85ac2e1879923bad2f08a08f3004055f2f406ef73c850114bd4", size = 765939, upload-time = "2026-04-03T20:55:37.972Z" }, - { url = "https://files.pythonhosted.org/packages/86/ea/81a7f968a351c6552b1670ead861e2a385be730ee28402233020c67f9e0f/regex-2026.4.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:1371c2ccbb744d66ee63631cc9ca12aa233d5749972626b68fe1a649dd98e566", size = 851417, upload-time = "2026-04-03T20:55:39.92Z" }, - { url = "https://files.pythonhosted.org/packages/4c/7e/323c18ce4b5b8f44517a36342961a0306e931e499febbd876bb149d900f0/regex-2026.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:59968142787042db793348a3f5b918cf24ced1f23247328530e063f89c128a95", size = 789056, upload-time = "2026-04-03T20:55:42.303Z" }, - { url = "https://files.pythonhosted.org/packages/c0/af/e7510f9b11b1913b0cd44eddb784b2d650b2af6515bfce4cffcc5bfd1d38/regex-2026.4.4-cp314-cp314-win32.whl", hash = "sha256:59efe72d37fd5a91e373e5146f187f921f365f4abc1249a5ab446a60f30dd5f8", size = 272130, upload-time = "2026-04-03T20:55:44.995Z" }, - { url = "https://files.pythonhosted.org/packages/9a/51/57dae534c915e2d3a21490e88836fa2ae79dde3b66255ecc0c0a155d2c10/regex-2026.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:e0aab3ff447845049d676827d2ff714aab4f73f340e155b7de7458cf53baa5a4", size = 280992, upload-time = "2026-04-03T20:55:47.316Z" }, - { url = "https://files.pythonhosted.org/packages/0a/5e/abaf9f4c3792e34edb1434f06717fae2b07888d85cb5cec29f9204931bf8/regex-2026.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:a7a5bb6aa0cf62208bb4fa079b0c756734f8ad0e333b425732e8609bd51ee22f", size = 273563, upload-time = "2026-04-03T20:55:49.273Z" }, - { url = "https://files.pythonhosted.org/packages/ff/06/35da85f9f217b9538b99cbb170738993bcc3b23784322decb77619f11502/regex-2026.4.4-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:97850d0638391bdc7d35dc1c1039974dcb921eaafa8cc935ae4d7f272b1d60b3", size = 494191, upload-time = "2026-04-03T20:55:51.258Z" }, - { url = "https://files.pythonhosted.org/packages/54/5b/1bc35f479eef8285c4baf88d8c002023efdeebb7b44a8735b36195486ae7/regex-2026.4.4-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:ee7337f88f2a580679f7bbfe69dc86c043954f9f9c541012f49abc554a962f2e", size = 293877, upload-time = "2026-04-03T20:55:53.214Z" }, - { url = "https://files.pythonhosted.org/packages/39/5b/f53b9ad17480b3ddd14c90da04bfb55ac6894b129e5dea87bcaf7d00e336/regex-2026.4.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7429f4e6192c11d659900c0648ba8776243bf396ab95558b8c51a345afeddde6", size = 292410, upload-time = "2026-04-03T20:55:55.736Z" }, - { url = "https://files.pythonhosted.org/packages/bb/56/52377f59f60a7c51aa4161eecf0b6032c20b461805aca051250da435ffc9/regex-2026.4.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc4f10fbd5dd13dcf4265b4cc07d69ca70280742870c97ae10093e3d66000359", size = 811831, upload-time = "2026-04-03T20:55:57.802Z" }, - { url = "https://files.pythonhosted.org/packages/dd/63/8026310bf066f702a9c361f83a8c9658f3fe4edb349f9c1e5d5273b7c40c/regex-2026.4.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a152560af4f9742b96f3827090f866eeec5becd4765c8e0d3473d9d280e76a5a", size = 871199, upload-time = "2026-04-03T20:56:00.333Z" }, - { url = "https://files.pythonhosted.org/packages/20/9f/a514bbb00a466dbb506d43f187a04047f7be1505f10a9a15615ead5080ee/regex-2026.4.4-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54170b3e95339f415d54651f97df3bff7434a663912f9358237941bbf9143f55", size = 917649, upload-time = "2026-04-03T20:56:02.445Z" }, - { url = "https://files.pythonhosted.org/packages/cb/6b/8399f68dd41a2030218839b9b18360d79b86d22b9fab5ef477c7f23ca67c/regex-2026.4.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:07f190d65f5a72dcb9cf7106bfc3d21e7a49dd2879eda2207b683f32165e4d99", size = 816388, upload-time = "2026-04-03T20:56:04.595Z" }, - { url = "https://files.pythonhosted.org/packages/1e/9c/103963f47c24339a483b05edd568594c2be486188f688c0170fd504b2948/regex-2026.4.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9a2741ce5a29d3c84b0b94261ba630ab459a1b847a0d6beca7d62d188175c790", size = 785746, upload-time = "2026-04-03T20:56:07.13Z" }, - { url = "https://files.pythonhosted.org/packages/fa/ee/7f6054c0dec0cee3463c304405e4ff42e27cff05bf36fcb34be549ab17bd/regex-2026.4.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:b26c30df3a28fd9793113dac7385a4deb7294a06c0f760dd2b008bd49a9139bc", size = 801483, upload-time = "2026-04-03T20:56:09.365Z" }, - { url = "https://files.pythonhosted.org/packages/30/c2/51d3d941cf6070dc00c3338ecf138615fc3cce0421c3df6abe97a08af61a/regex-2026.4.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:421439d1bee44b19f4583ccf42670ca464ffb90e9fdc38d37f39d1ddd1e44f1f", size = 866331, upload-time = "2026-04-03T20:56:12.039Z" }, - { url = "https://files.pythonhosted.org/packages/16/e8/76d50dcc122ac33927d939f350eebcfe3dbcbda96913e03433fc36de5e63/regex-2026.4.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b40379b53ecbc747fd9bdf4a0ea14eb8188ca1bd0f54f78893a39024b28f4863", size = 772673, upload-time = "2026-04-03T20:56:14.558Z" }, - { url = "https://files.pythonhosted.org/packages/a5/6e/5f6bf75e20ea6873d05ba4ec78378c375cbe08cdec571c83fbb01606e563/regex-2026.4.4-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:08c55c13d2eef54f73eeadc33146fb0baaa49e7335eb1aff6ae1324bf0ddbe4a", size = 857146, upload-time = "2026-04-03T20:56:16.663Z" }, - { url = "https://files.pythonhosted.org/packages/0b/33/3c76d9962949e487ebba353a18e89399f292287204ac8f2f4cfc3a51c233/regex-2026.4.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9776b85f510062f5a75ef112afe5f494ef1635607bf1cc220c1391e9ac2f5e81", size = 803463, upload-time = "2026-04-03T20:56:18.923Z" }, - { url = "https://files.pythonhosted.org/packages/19/eb/ef32dcd2cb69b69bc0c3e55205bce94a7def48d495358946bc42186dcccc/regex-2026.4.4-cp314-cp314t-win32.whl", hash = "sha256:385edaebde5db5be103577afc8699fea73a0e36a734ba24870be7ffa61119d74", size = 275709, upload-time = "2026-04-03T20:56:20.996Z" }, - { url = "https://files.pythonhosted.org/packages/a0/86/c291bf740945acbf35ed7dbebf8e2eea2f3f78041f6bd7cdab80cb274dc0/regex-2026.4.4-cp314-cp314t-win_amd64.whl", hash = "sha256:5d354b18839328927832e2fa5f7c95b7a3ccc39e7a681529e1685898e6436d45", size = 285622, upload-time = "2026-04-03T20:56:23.641Z" }, - { url = "https://files.pythonhosted.org/packages/d5/e7/ec846d560ae6a597115153c02ca6138a7877a1748b2072d9521c10a93e58/regex-2026.4.4-cp314-cp314t-win_arm64.whl", hash = "sha256:af0384cb01a33600c49505c27c6c57ab0b27bf84a74e28524c92ca897ebdac9d", size = 275773, upload-time = "2026-04-03T20:56:26.07Z" }, +version = "2026.5.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/0e/49aee608ad09480e7fd276898c99ec6192985fa331abe4eb3a986094490b/regex-2026.5.9.tar.gz", hash = "sha256:a8234aa23ec39894bfe4a3f1b85616a7032481964a13ac6fc9f10de4f6fca270", size = 416074, upload-time = "2026-05-09T23:15:19.37Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/da/797e91ecec6f84135da778ddce78c20e0af5d2a15c26f87a81bc3eadb6db/regex-2026.5.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d626b84406444b165fc0ba981604edea39f0588ff1f92baa23fe50799ea9afdb", size = 490303, upload-time = "2026-05-09T23:13:04.382Z" }, + { url = "https://files.pythonhosted.org/packages/44/da/bf30abaaa737b58f4a4b8c4a03659e02fd92092c822e0197ed9e0daab917/regex-2026.5.9-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d7bdc0ab8f3dd7e1b4f9ab88634e13374669db86bb3c72e8292f07ae313f539f", size = 292019, upload-time = "2026-05-09T23:13:06.022Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e7/d0eaf5713828417b9e5648cf81fa9bacd4961f6ab98c380c2034f8716e35/regex-2026.5.9-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a8820737949116ffff55fe18f9fc644530063ba6ebfcb8314239416e78f1347c", size = 289468, upload-time = "2026-05-09T23:13:08.214Z" }, + { url = "https://files.pythonhosted.org/packages/d3/9b/b3fdd62b003baa1a9b593cd8c8699c9651c2e80cc21a5c715707983c42d7/regex-2026.5.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa0fbdbac82cb3e4450d0ccde7d7a35607f4cb2dd9fba4b8b69bfaf8c9fa6aed", size = 796749, upload-time = "2026-05-09T23:13:10.573Z" }, + { url = "https://files.pythonhosted.org/packages/d4/30/66ab84588765f5b4b271a9ca09ef7ce2b87caa95176ec3d2ad65d7bc4902/regex-2026.5.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:57e8915c7986aa33d25e4d3629cef711cd2863f2961b10409f0c04cb8b7d9020", size = 865445, upload-time = "2026-05-09T23:13:12.523Z" }, + { url = "https://files.pythonhosted.org/packages/1a/89/f05169e8588aac365f35ffc7f3bc3184f095ef4cfded7cfaa3c7fd5dbd89/regex-2026.5.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:508f56a89ba9cb26e4168cbc37dbd60a28d82430a9e18ad1d25fe0883c314ca2", size = 912322, upload-time = "2026-05-09T23:13:14.281Z" }, + { url = "https://files.pythonhosted.org/packages/30/e1/c93444052cf41581f3c884ab3fb5823daf0992f11cd4388d4275ca610558/regex-2026.5.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b6d189041f15691cfa2b6c4290448ec221244d225b3f5fe9e7771b34ffcdf6e2", size = 801269, upload-time = "2026-05-09T23:13:16.569Z" }, + { url = "https://files.pythonhosted.org/packages/50/fe/0cf96b882f540e62e8b9956599798203d599c44cf4c77917ca27400ff69b/regex-2026.5.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e82db382b44d0111b22601c509c89f64434816c9e0eef9d1989cda8cc6ff1c04", size = 777085, upload-time = "2026-05-09T23:13:18.675Z" }, + { url = "https://files.pythonhosted.org/packages/23/5c/d78d4924e7fc875557b9e9b768423925fdfaac5549d06da7810019a9bd26/regex-2026.5.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2acfb48634f64996b57f90f39afa692ff362162722581921fe92239a59960f3c", size = 785153, upload-time = "2026-05-09T23:13:20.525Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e0/5214774090e7b4524dcea3e3c4aa74141d43043f8beb49c1599db1c8b53a/regex-2026.5.9-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d29eebfc9525db68cad3c97eedd7f754fa265aa5cd0cf4f863b2421e1b48fc9f", size = 860164, upload-time = "2026-05-09T23:13:22.263Z" }, + { url = "https://files.pythonhosted.org/packages/6e/e1/4a57a83350319b1271f0d7a249b8672513ed928b237a741631270de6caea/regex-2026.5.9-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:debb893095e944091c16e641a6e33c1b0f4cb61ab945ec5afbf53ce7068834d8", size = 765731, upload-time = "2026-05-09T23:13:24.277Z" }, + { url = "https://files.pythonhosted.org/packages/12/f4/499e74a20c156fc75836ee04a72a38d1a063978f600937f9760467beb1b0/regex-2026.5.9-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d659eee77986549c9ea45b861c7567e44d6287c3dc9a4565478853f7b9fe2ff6", size = 852062, upload-time = "2026-05-09T23:13:26.125Z" }, + { url = "https://files.pythonhosted.org/packages/5b/92/7eebc0d0a01e78629695f342ba17e0deaff8fb45e79cc0d7b98287da6e3e/regex-2026.5.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2efa205e6d98b24d1f3ab395c11aa15cdf10935bca283d0285e0499c284fba21", size = 789577, upload-time = "2026-05-09T23:13:27.814Z" }, + { url = "https://files.pythonhosted.org/packages/05/a4/018e71f7d2ad48c1ebe6d3ae0026f9b7cb4802fd15c7cc02fdf724355102/regex-2026.5.9-cp313-cp313-win32.whl", hash = "sha256:f3844f134e834076677dd369976e9f5068679fcb8e50102fdf6b7ac96a3ec127", size = 266691, upload-time = "2026-05-09T23:13:29.549Z" }, + { url = "https://files.pythonhosted.org/packages/e6/1d/861a93719fb9ee7dbfc3761b3797b7a3e112a5d42c6129459d2d741be9b5/regex-2026.5.9-cp313-cp313-win_amd64.whl", hash = "sha256:3527bb4942d2c14552155406cdedd906567456821848aed1cb4933a391bf5eca", size = 277747, upload-time = "2026-05-09T23:13:31.859Z" }, + { url = "https://files.pythonhosted.org/packages/d9/c6/0a2436ae4da1ba76e51cb98943c6838a9a721faa40ebe2dce07694ae34e3/regex-2026.5.9-cp313-cp313-win_arm64.whl", hash = "sha256:56a33f191f17d8c417f99945ebdc1e691d3af9605d86ec68c7e54a57e3e17af6", size = 270500, upload-time = "2026-05-09T23:13:33.525Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e9/d21346f7b60ed58789371358ed66b09d00f832e1bd7c06e55d9da5679882/regex-2026.5.9-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:01f28d868834624c934b8d2e0aa1c8341337e37831f4a012f18a5afcba4cbaf3", size = 494172, upload-time = "2026-05-09T23:13:35.935Z" }, + { url = "https://files.pythonhosted.org/packages/c4/43/fd1177a2032037c681baecdb3422ee4e1424aec4e4f470ef47793d325274/regex-2026.5.9-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:48036f6374aaa79eb3b754ec29c61d1c6b1606749d705a13f8854fa2539671f6", size = 293952, upload-time = "2026-05-09T23:13:38.307Z" }, + { url = "https://files.pythonhosted.org/packages/f2/7d/9fbf919768368d3f8a4f6c692cf2aa61e482b2b81ec6a298ace4cbf02480/regex-2026.5.9-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b96350aa424e79d4fd6b567b344dcbe2b2d6bfc48dfe7717587e1fa6d43da6ff", size = 292314, upload-time = "2026-05-09T23:13:40.353Z" }, + { url = "https://files.pythonhosted.org/packages/e2/6c/e41bfeecb589716843e7c4df09ba46ff2a42961457afece19059d85caeef/regex-2026.5.9-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f3af7a4903c5c04a11a196a5aa75cdd7dd3f8508132f9fb3259d9f5908e3b88", size = 811681, upload-time = "2026-05-09T23:13:42.543Z" }, + { url = "https://files.pythonhosted.org/packages/87/83/a5c1c525fba0aa656e88ad0face0b1829788ef4c2fb6b26df58aa1151b84/regex-2026.5.9-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7e87577720152d2caae19fe2baaf1f8d5ca12091e9e229f03915c37d1e4b9178", size = 871135, upload-time = "2026-05-09T23:13:44.326Z" }, + { url = "https://files.pythonhosted.org/packages/18/d4/80882e799e440dd878b0979cbebf8fa4d54624a332c83037c7a701649e3f/regex-2026.5.9-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c8b9b9d294cfea3cd19c718ade7cc93492b2c4991abd9a68d0b3477ae6d8e100", size = 917265, upload-time = "2026-05-09T23:13:47.295Z" }, + { url = "https://files.pythonhosted.org/packages/ae/ff/8db60211e2286e396aad7dc7725356c502bff0901ea05bd6cdc2e1a042b9/regex-2026.5.9-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:728d8bfd28a8845c8b6bc5dc7ce010453d206396786c0765c2740cb65f37791e", size = 816311, upload-time = "2026-05-09T23:13:49.885Z" }, + { url = "https://files.pythonhosted.org/packages/4c/47/742ef579c61730f8d268e5cf1f9ce0e37e2ea041ad0f5644724f2378e463/regex-2026.5.9-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7e30b874d341fac767d7df5a0870540541c2c054b80cfaac116e8d367a8a7ff2", size = 785498, upload-time = "2026-05-09T23:13:52.25Z" }, + { url = "https://files.pythonhosted.org/packages/7f/ab/cb0999802dcb0fb95b1ab005e8d4163d8afdd67efc2cb6b6630ac13f8cb1/regex-2026.5.9-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:fd190e88a895a8901325fad284a3f74ea52b1da8525b76cc811fa9b1edf0ce2b", size = 801348, upload-time = "2026-05-09T23:13:54.127Z" }, + { url = "https://files.pythonhosted.org/packages/7d/62/8ca59a24c55bc34d166eefaf3717bd77772f329fdbf984d86581e0a3571c/regex-2026.5.9-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:8e76e8161ad00694cfce6767d5dea860c6391ac5b83e5c3a39661e696f11fc7e", size = 866493, upload-time = "2026-05-09T23:13:56.067Z" }, + { url = "https://files.pythonhosted.org/packages/8d/3d/30f2ae62cef3278bb5bb821f467277a55fb73f01032cf85997e15e8289a8/regex-2026.5.9-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ddda5340e6c01a293027dd46232fa79eaff1b48058ce7a98f572b6445b088041", size = 772811, upload-time = "2026-05-09T23:13:57.867Z" }, + { url = "https://files.pythonhosted.org/packages/d8/ae/7d2089bcd78ad0c0161bc684339df50032acb438a7bd3305e7ddb1193cec/regex-2026.5.9-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:205109e96b3cf5adf8f4cd62bedde9487feb282b9497a3535451e5a24cd706a0", size = 856584, upload-time = "2026-05-09T23:13:59.679Z" }, + { url = "https://files.pythonhosted.org/packages/a9/29/92ff47f75990131ea4f24ba17819e5a9d141e10819807e09addd73409af6/regex-2026.5.9-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dfbe4579b9f08036aa7d101d1835437a20783574ac66327e6b29b4018a138081", size = 803453, upload-time = "2026-05-09T23:14:01.978Z" }, + { url = "https://files.pythonhosted.org/packages/04/99/eff29f1037dcab36702c9ee5d6858cf1ce2336ea8ea2987f64245b99ea5e/regex-2026.5.9-cp313-cp313t-win32.whl", hash = "sha256:ed2c9e8068b614c574d8d30e543d617cf5379b0535d46f97ef00e904745a08b5", size = 269951, upload-time = "2026-05-09T23:14:03.661Z" }, + { url = "https://files.pythonhosted.org/packages/0e/9d/8870b8981d27b22cda77bb26a5ac7ebfa9c7d9e0dea195a834a82380e748/regex-2026.5.9-cp313-cp313t-win_amd64.whl", hash = "sha256:b46b0f094dc1d3b90356c85a0bd2c9bafc4a6a190b9d6f8ddd5a033b6e088ed4", size = 281240, upload-time = "2026-05-09T23:14:05.56Z" }, + { url = "https://files.pythonhosted.org/packages/72/b1/3379415e8f135c13ac551353397cc4fe97b4978f3cac73c5fcbcded548b8/regex-2026.5.9-cp313-cp313t-win_arm64.whl", hash = "sha256:872acc074bd29ffc9913ecdfedf6ea77502312ca44a4aa0d3779089c6069d8de", size = 272383, upload-time = "2026-05-09T23:14:07.843Z" }, + { url = "https://files.pythonhosted.org/packages/13/3e/9c3cd292d8808b3645a2ce517e200179b6d0e903f176300bd8b542e14de5/regex-2026.5.9-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:1bd7587a2948b4085195d5a3374eaf4a425dc3e55784c038175355ecf3bbbf8a", size = 490376, upload-time = "2026-05-09T23:14:09.64Z" }, + { url = "https://files.pythonhosted.org/packages/60/70/d43ee8a2ca0a8b68d167f21658b85520ac0574617c7f320367c5047f7556/regex-2026.5.9-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:dea2e88e1cce4522496cce630e11e67b98b7076620bc4336c3f674bc21a375f4", size = 291964, upload-time = "2026-05-09T23:14:11.424Z" }, + { url = "https://files.pythonhosted.org/packages/21/91/9d50b433828d8e74196904e168a43abf1e6e88b2a15d47ed742456720c37/regex-2026.5.9-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:2099f7e7ff7b6aa3192312650a56e91cc091e49d50b04e4f6f8b6e28b3b27f1c", size = 289682, upload-time = "2026-05-09T23:14:13.123Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d2/b835e3cafbb9d977736912436259ff551d60919f7d7b3d37d46659c63564/regex-2026.5.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecd353045824e4477562a2ac718c25799cdaaa41f7aa925a806a8a3e6848a5b9", size = 796996, upload-time = "2026-05-09T23:14:14.923Z" }, + { url = "https://files.pythonhosted.org/packages/2c/a6/9f992d00019166b9de01c546dd4549bc679f2a68df11b877740b0760b7c2/regex-2026.5.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:65c8c8c37377794bd5b2f3ebe51919042bf17aec802e23c833d89782ed0c78af", size = 866089, upload-time = "2026-05-09T23:14:17.757Z" }, + { url = "https://files.pythonhosted.org/packages/e0/08/4d32af657e049b19cb62b02e46e38fe1518797bfb2203ee93a510b21b0dc/regex-2026.5.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5b73ab8afcf66c622db143d1c6fda4e58e4d537ee4f125229ad47b1ab80f34c0", size = 911530, upload-time = "2026-05-09T23:14:20.353Z" }, + { url = "https://files.pythonhosted.org/packages/d9/27/2af43dd1dc201d1fecefda64a45f4ad0995855b92724f795a777b402ee69/regex-2026.5.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0de5cf193997384ed2ca6f1cd4f78055b255d93d82d5a8cd6ba0d11c10b167e4", size = 800643, upload-time = "2026-05-09T23:14:22.265Z" }, + { url = "https://files.pythonhosted.org/packages/a4/dd/23a249047013b5321d4a60c4d2437462086f601b061776a525e5fba2a59f/regex-2026.5.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d641a8c9a61618047796d572a39a79b26167b0411d2c3031937b2fe2d081e2cf", size = 777223, upload-time = "2026-05-09T23:14:24.179Z" }, + { url = "https://files.pythonhosted.org/packages/94/6a/e85ed9538cd19586d0465076a4578a12e093ce776d15f3f8ce92733a8dd6/regex-2026.5.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:24b2355ef5cc9aa5b8f07d17704face1c166fdcc2290fa7bd6e6c925655a8346", size = 785760, upload-time = "2026-05-09T23:14:26.065Z" }, + { url = "https://files.pythonhosted.org/packages/2a/c4/f25473209438638e947c55f9156fd8f236f74169229028cc99116380868e/regex-2026.5.9-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:a24852d3c29ad9e47593593d8a247c44ccc3d0548ef12c822d6ed0810affe676", size = 860891, upload-time = "2026-05-09T23:14:28.17Z" }, + { url = "https://files.pythonhosted.org/packages/f9/f7/f4f86e3c74419c37370e91f150ae0c2ef7d34b2e0e4cdd5da046a02e4022/regex-2026.5.9-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:916714069da19329ef7de197dcbc77bb3104145c7c2c864dbfbe318f46b88b14", size = 765891, upload-time = "2026-05-09T23:14:30.06Z" }, + { url = "https://files.pythonhosted.org/packages/26/70/704d8e13765939146b1cd0ef4e2feb71d7929727d2290f026eed10095955/regex-2026.5.9-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:fa411799ca8da32a8d38d020a88faa5b6f91657d284761352940ecf9f7c3bbdd", size = 851380, upload-time = "2026-05-09T23:14:32.123Z" }, + { url = "https://files.pythonhosted.org/packages/26/29/1a13582a8460038edc38e49f64ceb0dd7c60f5caba77571f4bf6601965d9/regex-2026.5.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1e6da47d679b7010ef27556b6e0f99771b744936db1792a10ceac6547ae1503e", size = 789350, upload-time = "2026-05-09T23:14:34.799Z" }, + { url = "https://files.pythonhosted.org/packages/73/56/3dcafe34fc72e271d62ad9a291801e88a1457bb251c132f15fcc2e5aad1a/regex-2026.5.9-cp314-cp314-win32.whl", hash = "sha256:98bd73080e8756255137e1bd3f3f00295bbc5aa383c0e0f973920e9134d7c4ad", size = 272130, upload-time = "2026-05-09T23:14:36.729Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9c/02eebf0be95efe416c664db7fb8b6b05b7a0b06a7544f2884f2558b0526f/regex-2026.5.9-cp314-cp314-win_amd64.whl", hash = "sha256:ff8d372ac2acdc048d1c19916f27ee61bc5722728458ba6ca5052f2c72d51763", size = 280999, upload-time = "2026-05-09T23:14:39.126Z" }, + { url = "https://files.pythonhosted.org/packages/70/5a/1dd1abee76cb7a846a0bcf42fdc87e5720c3c33c24f3e37814310a513d9f/regex-2026.5.9-cp314-cp314-win_arm64.whl", hash = "sha256:e1d93bf647916292e8edcec150c07ddf3dc50179ccaf770c04a7f9e452155372", size = 273500, upload-time = "2026-05-09T23:14:41.059Z" }, + { url = "https://files.pythonhosted.org/packages/86/c1/c5f619b0057a7965cb78ec559c1d7a45ce8c99a35bea95483d64959a93d9/regex-2026.5.9-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:83d0ee4a57d1c87cb549e195ec300b8f0ec3a82eba66d835e4e2ed8634fe4499", size = 494269, upload-time = "2026-05-09T23:14:42.869Z" }, + { url = "https://files.pythonhosted.org/packages/05/2c/5d01f1aee33de4bbe60c8452945bfc8477ca7c5ae4450f6bfe711036cb36/regex-2026.5.9-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:d3d7eb5c9a7f6df82ed3cfac9beb93882a5cbcb5b8b157b56cb2b3b276574ac1", size = 293954, upload-time = "2026-05-09T23:14:44.822Z" }, + { url = "https://files.pythonhosted.org/packages/7a/fe/e8988b2ae2108c6ef71bd4aa8d87fbe257976dd0810e826cd75f701c68b6/regex-2026.5.9-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:075160bf16658e16d35233300b8453aac25de4cbea808d22348b6979668e924d", size = 292405, upload-time = "2026-05-09T23:14:47.211Z" }, + { url = "https://files.pythonhosted.org/packages/79/34/d2b0937faa7859263f7f0a3c6b103a1296306be6952dc173d0154e9a2f49/regex-2026.5.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45375819235558a4ff1c4971dc32881f022613abdb180128f5cb4768c1765a1c", size = 811855, upload-time = "2026-05-09T23:14:49.21Z" }, + { url = "https://files.pythonhosted.org/packages/80/fe/daf53a47457a8486db66c66c01ceb9c2303eecee3f87197f1e77eb1a736d/regex-2026.5.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ead4b163ac30a29574510cd4b3e2e985ac5290c05fc7095557d6a5f403fc31b5", size = 871189, upload-time = "2026-05-09T23:14:51.555Z" }, + { url = "https://files.pythonhosted.org/packages/1c/75/058fc4470cbfbf57d800aff1a0022b929a3f9fa553ee10a0cdf2070eb31f/regex-2026.5.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8c6e4218fbdfbcd4f6c19efca40930d24a621bf4b48cb76bc6640543bd28ef20", size = 917485, upload-time = "2026-05-09T23:14:53.633Z" }, + { url = "https://files.pythonhosted.org/packages/88/e7/179cfda3a28bc843b5c6cfe7f79f23489c791ed95f151083803660878432/regex-2026.5.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6351571c8a42b505eb555c0dc47d740d0fb66977dc142919eea6f4325b7c56a0", size = 816369, upload-time = "2026-05-09T23:14:56.198Z" }, + { url = "https://files.pythonhosted.org/packages/41/90/6f0cc422071688266d344fca8462d787cba0a2c144acb25721f9a61ec265/regex-2026.5.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:002205cafd2a9e78c6290c7d1df277bf3277b3b7a30e0b4bb0dac2e2e3f7cb2d", size = 785869, upload-time = "2026-05-09T23:14:58.602Z" }, + { url = "https://files.pythonhosted.org/packages/02/67/a31f1760f09c27b251ef39e9beb541f462cf977381d067faa764c2c0e393/regex-2026.5.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8abd33fef90b2a9efac5557d6033ca82d1195ed3a15fea5af15ba7b463c6a63b", size = 801427, upload-time = "2026-05-09T23:15:00.642Z" }, + { url = "https://files.pythonhosted.org/packages/e3/c4/1a80654597b6bc1e1ea0494824c31200e8a956abe290afae9b19a166a148/regex-2026.5.9-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:31037c82eccb44b7ea2e9e221d7c01429430e989a1f4b91ea5a855f6017b509a", size = 866482, upload-time = "2026-05-09T23:15:03.384Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/960724e06482c08466ff5611e242e86f80062949cdf6b4b9cc317b9dd93d/regex-2026.5.9-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5604dfd046dc37eca90250fc3be938b076c8059fa772ac0ed6f499b0f0fb0415", size = 773022, upload-time = "2026-05-09T23:15:05.625Z" }, + { url = "https://files.pythonhosted.org/packages/50/a8/a9979c3e7918280e93159ebcab5ef1a65116dd4f3bd6091be0eae4a126e8/regex-2026.5.9-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:0e1b1b4e496afbb24f4a62aba855ee4f88f25578927697b340702e48c9ee6bc2", size = 856642, upload-time = "2026-05-09T23:15:07.966Z" }, + { url = "https://files.pythonhosted.org/packages/fe/d4/a9b732f2f0072c0ab12227483abb24fffcb9f73f8a2b203df0a6d0434735/regex-2026.5.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:be3372b9df6ddecff6486d37e19095a7b4973137caf5512407a89f4455361f41", size = 803552, upload-time = "2026-05-09T23:15:10.215Z" }, + { url = "https://files.pythonhosted.org/packages/d5/fe/1b3113817447a1d4155e4ac76d2e072f42c0bcba2f43fa8a0e756ea2cd91/regex-2026.5.9-cp314-cp314t-win32.whl", hash = "sha256:3ddd90103f9e5c471c49c7852ecc1fe27c7e45eb99e977aefe7caa4e779f4f58", size = 275746, upload-time = "2026-05-09T23:15:12.609Z" }, + { url = "https://files.pythonhosted.org/packages/92/73/93d42045302636c91f2e5ef588b65b84b01428f28ec77de256b1dfdfbe5c/regex-2026.5.9-cp314-cp314t-win_amd64.whl", hash = "sha256:ca518ed29c46eecba6010b15f1b9a479314d2de409536e71b6a13aa04e3b8a77", size = 285685, upload-time = "2026-05-09T23:15:15.086Z" }, + { url = "https://files.pythonhosted.org/packages/da/80/35b4c33c804a165a7f55289afda3ea9e3eb6d15800341a2d66455c0f1f30/regex-2026.5.9-cp314-cp314t-win_arm64.whl", hash = "sha256:5e41809d2683fcde7d5a8c87a6567ba1fb1ce0de9f31bff578de00a4b2d76daa", size = 275713, upload-time = "2026-05-09T23:15:16.98Z" }, ] [[package]] @@ -2995,14 +3383,14 @@ wheels = [ [[package]] name = "rich-argparse" -version = "1.7.2" +version = "1.8.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "rich" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4c/f7/1c65e0245d4c7009a87ac92908294a66e7e7635eccf76a68550f40c6df80/rich_argparse-1.7.2.tar.gz", hash = "sha256:64fd2e948fc96e8a1a06e0e72c111c2ce7f3af74126d75c0f5f63926e7289cd1", size = 38500, upload-time = "2025-11-01T10:35:44.232Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/e5/1064c43203a357d668cd42435f7a15fe6af51512d85b2104fecb937aa861/rich_argparse-1.8.0.tar.gz", hash = "sha256:679df3d832fa94ad6e4bdb07ded088cd7ea2dddc58ae9b2b46346a40b06cbc0c", size = 38940, upload-time = "2026-05-01T15:18:43.604Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/04/80/97b6f357ac458d9ad9872cc3183ca09ef7439ac89e030ea43053ba1294b6/rich_argparse-1.7.2-py3-none-any.whl", hash = "sha256:0559b1f47a19bbeb82bf15f95a057f99bcbbc98385532f57937f9fc57acc501a", size = 25476, upload-time = "2025-11-01T10:35:42.681Z" }, + { url = "https://files.pythonhosted.org/packages/0b/35/1cceccc5fcb50fa2ed53e2aa278cd032f3902682a73e763fb1ac3be8e6fa/rich_argparse-1.8.0-py3-none-any.whl", hash = "sha256:d2a3ce7854654e2253c578763ab0a32f05016f23a55fadba7b9a91b6c0e92142", size = 25616, upload-time = "2026-05-01T15:18:42.395Z" }, ] [[package]] @@ -3098,16 +3486,16 @@ wheels = [ [[package]] name = "s3fs" -version = "2026.3.0" +version = "2026.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiobotocore" }, { name = "aiohttp" }, { name = "fsspec" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0b/93/093972862fb9c2fdc24ecf8d6d2212853df1945eddf26ba2625e8eaeee66/s3fs-2026.3.0.tar.gz", hash = "sha256:ce8b30a9dc5e01c5127c96cb7377290243a689a251ef9257336ac29d72d7b0d8", size = 85986, upload-time = "2026-03-27T19:28:20.963Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/d8/76f3dc1558bdf4494b117a9f7a9cc0a5d9d34edadc9e5d7ceabc5a6a7c37/s3fs-2026.4.0.tar.gz", hash = "sha256:5bdce0abb00b0435ee150807a45fea727451dbc22de4cbc116464f8504ab9d37", size = 85986, upload-time = "2026-04-29T20:52:51.748Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/52/5ccdc01f7a8a61357d15a66b5d8a6580aa8529cb33f32e6cbb71c52622c5/s3fs-2026.3.0-py3-none-any.whl", hash = "sha256:2fa40a64c03003cfa5ae0e352788d97aa78ae8f9e25ea98b28ce9d21ba10c1b8", size = 32399, upload-time = "2026-03-27T19:28:19.702Z" }, + { url = "https://files.pythonhosted.org/packages/5d/a4/9d1ea10ebc9e028a289a72fec84da170689549a8102c8aacfcad26bc5035/s3fs-2026.4.0-py3-none-any.whl", hash = "sha256:de0d2a1f33cdf03831fd2382d278c6e4e31fe57c3bf2f703c61f8aec6b703e2a", size = 32392, upload-time = "2026-04-29T20:52:50.295Z" }, ] [[package]] @@ -3172,24 +3560,44 @@ wheels = [ [[package]] name = "simplejson" -version = "3.20.2" +version = "4.1.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/41/f4/a1ac5ed32f7ed9a088d62a59d410d4c204b3b3815722e2ccfb491fa8251b/simplejson-3.20.2.tar.gz", hash = "sha256:5fe7a6ce14d1c300d80d08695b7f7e633de6cd72c80644021874d985b3393649", size = 85784, upload-time = "2025-09-26T16:29:36.64Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/9e/f326d43f6bf47f4e7704a4426c36e044c6bedfd24e072fb8e27589a373a5/simplejson-3.20.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:90d311ba8fcd733a3677e0be21804827226a57144130ba01c3c6a325e887dd86", size = 93530, upload-time = "2025-09-26T16:28:18.07Z" }, - { url = "https://files.pythonhosted.org/packages/35/28/5a4b8f3483fbfb68f3f460bc002cef3a5735ef30950e7c4adce9c8da15c7/simplejson-3.20.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:feed6806f614bdf7f5cb6d0123cb0c1c5f40407ef103aa935cffaa694e2e0c74", size = 75846, upload-time = "2025-09-26T16:28:19.12Z" }, - { url = "https://files.pythonhosted.org/packages/7a/4d/30dfef83b9ac48afae1cf1ab19c2867e27b8d22b5d9f8ca7ce5a0a157d8c/simplejson-3.20.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6b1d8d7c3e1a205c49e1aee6ba907dcb8ccea83651e6c3e2cb2062f1e52b0726", size = 75661, upload-time = "2025-09-26T16:28:20.219Z" }, - { url = "https://files.pythonhosted.org/packages/09/1d/171009bd35c7099d72ef6afd4bb13527bab469965c968a17d69a203d62a6/simplejson-3.20.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:552f55745044a24c3cb7ec67e54234be56d5d6d0e054f2e4cf4fb3e297429be5", size = 150579, upload-time = "2025-09-26T16:28:21.337Z" }, - { url = "https://files.pythonhosted.org/packages/61/ae/229bbcf90a702adc6bfa476e9f0a37e21d8c58e1059043038797cbe75b8c/simplejson-3.20.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2da97ac65165d66b0570c9e545786f0ac7b5de5854d3711a16cacbcaa8c472d", size = 158797, upload-time = "2025-09-26T16:28:22.53Z" }, - { url = "https://files.pythonhosted.org/packages/90/c5/fefc0ac6b86b9108e302e0af1cf57518f46da0baedd60a12170791d56959/simplejson-3.20.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f59a12966daa356bf68927fca5a67bebac0033cd18b96de9c2d426cd11756cd0", size = 148851, upload-time = "2025-09-26T16:28:23.733Z" }, - { url = "https://files.pythonhosted.org/packages/43/f1/b392952200f3393bb06fbc4dd975fc63a6843261705839355560b7264eb2/simplejson-3.20.2-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133ae2098a8e162c71da97cdab1f383afdd91373b7ff5fe65169b04167da976b", size = 152598, upload-time = "2025-09-26T16:28:24.962Z" }, - { url = "https://files.pythonhosted.org/packages/f4/b4/d6b7279e52a3e9c0fa8c032ce6164e593e8d9cf390698ee981ed0864291b/simplejson-3.20.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7977640af7b7d5e6a852d26622057d428706a550f7f5083e7c4dd010a84d941f", size = 150498, upload-time = "2025-09-26T16:28:26.114Z" }, - { url = "https://files.pythonhosted.org/packages/62/22/ec2490dd859224326d10c2fac1353e8ad5c84121be4837a6dd6638ba4345/simplejson-3.20.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b530ad6d55e71fa9e93e1109cf8182f427a6355848a4ffa09f69cc44e1512522", size = 152129, upload-time = "2025-09-26T16:28:27.552Z" }, - { url = "https://files.pythonhosted.org/packages/33/ce/b60214d013e93dd9e5a705dcb2b88b6c72bada442a97f79828332217f3eb/simplejson-3.20.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:bd96a7d981bf64f0e42345584768da4435c05b24fd3c364663f5fbc8fabf82e3", size = 159359, upload-time = "2025-09-26T16:28:28.667Z" }, - { url = "https://files.pythonhosted.org/packages/99/21/603709455827cdf5b9d83abe726343f542491ca8dc6a2528eb08de0cf034/simplejson-3.20.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f28ee755fadb426ba2e464d6fcf25d3f152a05eb6b38e0b4f790352f5540c769", size = 154717, upload-time = "2025-09-26T16:28:30.288Z" }, - { url = "https://files.pythonhosted.org/packages/3c/f9/dc7f7a4bac16cf7eb55a4df03ad93190e11826d2a8950052949d3dfc11e2/simplejson-3.20.2-cp313-cp313-win32.whl", hash = "sha256:472785b52e48e3eed9b78b95e26a256f59bb1ee38339be3075dad799e2e1e661", size = 74289, upload-time = "2025-09-26T16:28:31.809Z" }, - { url = "https://files.pythonhosted.org/packages/87/10/d42ad61230436735c68af1120622b28a782877146a83d714da7b6a2a1c4e/simplejson-3.20.2-cp313-cp313-win_amd64.whl", hash = "sha256:a1a85013eb33e4820286139540accbe2c98d2da894b2dcefd280209db508e608", size = 75972, upload-time = "2025-09-26T16:28:32.883Z" }, - { url = "https://files.pythonhosted.org/packages/05/5b/83e1ff87eb60ca706972f7e02e15c0b33396e7bdbd080069a5d1b53cf0d8/simplejson-3.20.2-py3-none-any.whl", hash = "sha256:3b6bb7fb96efd673eac2e4235200bfffdc2353ad12c54117e1e4e2fc485ac017", size = 57309, upload-time = "2025-09-26T16:29:35.312Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/0e/2a/54837395a3487c725669428d513293612a48d82b95a0642c936932e5d898/simplejson-4.1.1.tar.gz", hash = "sha256:c08eb9f7a90f77ae470e19a07472e9a79ebc0d1c2315d86a72767665bd5ba79f", size = 118860, upload-time = "2026-04-24T19:24:59.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/a9/47b445eeb559c9593453a0648e0fd6d08e8adff64dd5e5ced66726da8a09/simplejson-4.1.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:dff52fc7af272e84fc21cc5a06c927c823ca6ae00af14f3b0d7707b42775ed98", size = 113160, upload-time = "2026-04-24T19:23:26.033Z" }, + { url = "https://files.pythonhosted.org/packages/4c/65/cb72db31523c164dea5dc55b02dad065a40c478856bc7534b279d2b51906/simplejson-4.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:971aed0647ad6e840a3943bec812fcda5f2d26a5497a4981d1fb49aa4f9a396c", size = 91521, upload-time = "2026-04-24T19:23:27.572Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e5/54cb7c50ad5fdc1e0a86b7df4b135c2cbd5c4623605aa94466659098e8da/simplejson-4.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:249e2e220aa6d9b9d936bde84eb7bf79d5b6c5a8273c6e411f8b1635a9073f2d", size = 91407, upload-time = "2026-04-24T19:23:28.991Z" }, + { url = "https://files.pythonhosted.org/packages/38/2e/21a3ede87f0bf82d6c7bcb90480d50a6490eb974c6ab20881188e440957c/simplejson-4.1.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8e5cdd6a5d52299f345c15ab5678cc4249e24f383f361d986afbc3c7072a6b6b", size = 192451, upload-time = "2026-04-24T19:23:30.56Z" }, + { url = "https://files.pythonhosted.org/packages/59/df/9903edd3102bf0b5984edfcb90c88612330996efa3b4fbf8a971d6e17839/simplejson-4.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:642cec364e0676e2d5a73fa4d31d0c7c55886997caa2fde24e8292ca44d32728", size = 189015, upload-time = "2026-04-24T19:23:32.647Z" }, + { url = "https://files.pythonhosted.org/packages/98/cd/33230927a780e1398b857e3944abb914556994d252b1d765ae40d112cb25/simplejson-4.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:76fe296ca1df23d290033f10aaacf534fd1b3e3007e7f9ff8aa68b21413aaa78", size = 196658, upload-time = "2026-04-24T19:23:34.563Z" }, + { url = "https://files.pythonhosted.org/packages/cd/84/2c5a7444eb53e9a86d3738299bffddd9f53aeed799ded2f45368221fdb19/simplejson-4.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8f0ad25b7dc4e0fb23858355819f2e994f1a5badcdcde8737eac7921c2f1ed2a", size = 185967, upload-time = "2026-04-24T19:23:36.191Z" }, + { url = "https://files.pythonhosted.org/packages/d3/68/454378e06d059cd412a7ed5d87fb6d29fd5b60f13a4d89fc1f764ff434df/simplejson-4.1.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a59ebd0533f03fd06ff0c42ba0f02d93cbcdd7944922bf3b93911327a95b901f", size = 193940, upload-time = "2026-04-24T19:23:38.151Z" }, + { url = "https://files.pythonhosted.org/packages/d5/d5/a15bf915f623a2c5a079d6e3be8256fdb8ef06f110669493a09b9d6933e0/simplejson-4.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bccbf4419676b517939852e5aeff2af6aee4dc046881c67a1581fa6f1cb01abd", size = 189795, upload-time = "2026-04-24T19:23:40.139Z" }, + { url = "https://files.pythonhosted.org/packages/d2/c9/37212ae7dc4b607f0978c408e8633f05c810884e054c33113184c6c2c8a2/simplejson-4.1.1-cp313-cp313-win32.whl", hash = "sha256:6c845363eb5fd166fb7c72243da38f4fcfde666ede7fdf2cc6fd7762894626f7", size = 88773, upload-time = "2026-04-24T19:23:41.754Z" }, + { url = "https://files.pythonhosted.org/packages/fe/a5/c7a0a47883a9015b54c9d8a4b62f2aba17bd4335b1787b9b8a0fc2fa6d52/simplejson-4.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:104d8324c34f25b4b90800bc5fa363780cbc3d8496aef061cba7ce1af9162270", size = 90888, upload-time = "2026-04-24T19:23:43.11Z" }, + { url = "https://files.pythonhosted.org/packages/d3/18/4a118a6a92eb33bb08c8e2fe7ec85cb96f0673491bb2b829930831ee4fbe/simplejson-4.1.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:ed7473602b6625de793b6acba49aa949f144a475f538792067e4cf2fda2071f5", size = 110492, upload-time = "2026-04-24T19:23:44.957Z" }, + { url = "https://files.pythonhosted.org/packages/07/f4/84d160e9fa8cada1e0a9381cae4fa81eecd573577a5b34366d8ced59bdf7/simplejson-4.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:225c9caa324c5b554d009fb9cac22aee7711e71bd96f487938c659af467e828e", size = 90152, upload-time = "2026-04-24T19:23:46.355Z" }, + { url = "https://files.pythonhosted.org/packages/68/31/9a5432c433a7671107182cdc9a20ea78a70f99c4e5334aa54b6d4d0d79ed/simplejson-4.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:95407269340c7f22f09776ea7b717a52cf56cfcf119b5e45f66faa4a26445bea", size = 90115, upload-time = "2026-04-24T19:23:47.743Z" }, + { url = "https://files.pythonhosted.org/packages/78/91/3635cdb13318cb0a328abaa69e2b91251caad39d6779aa308098f341f6cb/simplejson-4.1.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3851658d642c1184d2023f0e6c9ce44a21eb1629e74e7c84ef956b128841fe12", size = 184036, upload-time = "2026-04-24T19:23:49.472Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ba/149b6ec5393f6849d98c59cadba888b710a8ef4b805ab91e11a566960d40/simplejson-4.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:95a3bb0f78e85f4937f99092239f2011ce06f0f2d803df5c299cc05abbeae008", size = 180543, upload-time = "2026-04-24T19:23:51.023Z" }, + { url = "https://files.pythonhosted.org/packages/df/7c/a5d968d0b527a748b667e62bea94309ccbcb1e2b108e8f0cf8547efaa12b/simplejson-4.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bbfdaa7c0603f75b7b14b211b7f2be44696d4e26833ad2d91d5c87bf5fb9a920", size = 188725, upload-time = "2026-04-24T19:23:52.995Z" }, + { url = "https://files.pythonhosted.org/packages/db/e3/6a8d11181d587ef00e2db9112357e6832111e56dd56b01b5c11758a1965d/simplejson-4.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:39e3c584071dced8c21b4689f0254303521daeb9b5bc1f4289755d71fa3cb0d3", size = 177492, upload-time = "2026-04-24T19:23:54.581Z" }, + { url = "https://files.pythonhosted.org/packages/67/e3/8b0eb8b06e8198cfbd1270487da163d0093df05cc4f557350cd65e2f7e79/simplejson-4.1.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:036a27bd0469b9d79557cbddb392969f876cd7f278cfbd0fba81534927a06575", size = 185281, upload-time = "2026-04-24T19:23:56.13Z" }, + { url = "https://files.pythonhosted.org/packages/dc/5f/64990f07ec9e2cb1a814c674e2e21b5693207f74ac70eb72151b847ea4e6/simplejson-4.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b70bfd2f67f3351baba08aa3ae9233c83f21fd95ae5e6b3d0ecb8c647929112f", size = 181848, upload-time = "2026-04-24T19:23:57.92Z" }, + { url = "https://files.pythonhosted.org/packages/61/a5/bbc1bc0447f339f79f99ab8c37f7f037cb2f1f93af75d6a4d553096bb0c3/simplejson-4.1.1-cp314-cp314-win32.whl", hash = "sha256:37233c72ce88d06acb92747347742b3c07871eba6789f060c179c9302dde8efe", size = 88761, upload-time = "2026-04-24T19:23:59.397Z" }, + { url = "https://files.pythonhosted.org/packages/18/72/ec1b5cbdcb140c132e6c7bdf99bd73e4f675439e77126c88f472fcffa09c/simplejson-4.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:cc0442dea71cd9cbf30a0b8b9929ab5aa6c02c0443a3d977351e6ec5bada4388", size = 91018, upload-time = "2026-04-24T19:24:00.85Z" }, + { url = "https://files.pythonhosted.org/packages/3d/97/4fa437f68ff72219bac3bf3d050de9c6265691f3a170e16954bd69d7cddd/simplejson-4.1.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:c996a4d38290c515af347740659ce095b425449c164a5c9fa3977caa6eff5dbe", size = 113919, upload-time = "2026-04-24T19:24:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/c2/83/59de041d09eb4a9577f7015d7263c32095dfb7fde49717dff62145d89809/simplejson-4.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c65c763fb20d7ca113c1c14dce2fc04a0fc3a57aceff533d6fdac707c7bffb40", size = 91904, upload-time = "2026-04-24T19:24:03.812Z" }, + { url = "https://files.pythonhosted.org/packages/03/8e/46bb345d540f6eb31427d984a4e518cdb182d0621814fee4fee045e8815b/simplejson-4.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0da5c9f57206ee7ef280ff7f1d924937b0a64f9a271a5ef371a2ecdbebba7421", size = 91752, upload-time = "2026-04-24T19:24:05.622Z" }, + { url = "https://files.pythonhosted.org/packages/83/e2/1b2ce97f068835eb3d253c116a4df7a3f436b7bf2fb5ff1ba29287e8b0ec/simplejson-4.1.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ea3426e786425d10e9e82f8a6eda74a7d6eb10d99165ac3d0d3bbcb65c0ea343", size = 214021, upload-time = "2026-04-24T19:24:07.447Z" }, + { url = "https://files.pythonhosted.org/packages/48/70/d93e556df6a0786298644a7c08304fcbeddc248325f23f38acbebeb21165/simplejson-4.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d75cea7a1025edd7e439b2966b3d977c45b5b899e2adaf422811b3ac702ed9fb", size = 213530, upload-time = "2026-04-24T19:24:09.289Z" }, + { url = "https://files.pythonhosted.org/packages/1b/a5/c93bf305b9f00d7259e09e713d60e75bd0f7f53da970f716ab90491770e7/simplejson-4.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63c2ada8e58f266491f19eed2eeeb7c25c6141e52f8f9e820f6bb94156cf8dbc", size = 218282, upload-time = "2026-04-24T19:24:10.991Z" }, + { url = "https://files.pythonhosted.org/packages/0c/20/a9b5d2e27ec44b069ee251bd55544fc76929a067107b1050001566ba86f3/simplejson-4.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d1fffb56305c5b475ee746cf9e04f97423ba5aaacd292dc1255bd75b1d3b124b", size = 209249, upload-time = "2026-04-24T19:24:12.662Z" }, + { url = "https://files.pythonhosted.org/packages/97/e4/e06ee682ed5df67592181f5ecb062e35878967e27f5b6e087237d4548d95/simplejson-4.1.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:a6525ec733f43d0541206cffa64fd2aad5a7ae3eb76566aff49cd4db6382209a", size = 213963, upload-time = "2026-04-24T19:24:14.302Z" }, + { url = "https://files.pythonhosted.org/packages/9c/9f/1e160e4cd8cdbf062bf6a454cdf814dc7a48eb47e566fdb8f80ccb202605/simplejson-4.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:861e393260508efa64d8805a8e49c416c3484907e3f146ce966c69552b49b9a3", size = 210474, upload-time = "2026-04-24T19:24:15.917Z" }, + { url = "https://files.pythonhosted.org/packages/7a/e6/cecd913df322df5bbe7ebb8ba39e0708e505a165553900da8a7761026d6f/simplejson-4.1.1-cp314-cp314t-win32.whl", hash = "sha256:d083b89d30948a751d3d97476c2ed91e4caaa24a1a1459bdbadb8876242c71fe", size = 91134, upload-time = "2026-04-24T19:24:17.635Z" }, + { url = "https://files.pythonhosted.org/packages/97/73/f540dde99cc1d393bd062ab3b5735b777561a5d8f8a5f2e241164444d77a/simplejson-4.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:4cbb299d0528ec0447fe366d8c9641860e28f997a62730690fef905f1f41046e", size = 94467, upload-time = "2026-04-24T19:24:19.109Z" }, + { url = "https://files.pythonhosted.org/packages/ce/6a/8b74c52ffd33dbbde00fe7251fee6a0acdc8cea33f7a43805aed258fb79b/simplejson-4.1.1-py3-none-any.whl", hash = "sha256:2ce92b3748f02423e26d2bfb636fb9d7a8f67c8f5854dcae69d350d123b2eee2", size = 69195, upload-time = "2026-04-24T19:24:57.962Z" }, ] [[package]] @@ -3221,55 +3629,63 @@ wheels = [ [[package]] name = "sqlalchemy" -version = "2.0.49" +version = "2.0.50" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/09/45/461788f35e0364a8da7bda51a1fe1b09762d0c32f12f63727998d85a873b/sqlalchemy-2.0.49.tar.gz", hash = "sha256:d15950a57a210e36dd4cec1aac22787e2a4d57ba9318233e2ef8b2daf9ff2d5f", size = 9898221, upload-time = "2026-04-03T16:38:11.704Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/81/81755f50eb2478eaf2049728491d4ea4f416c1eb013338682173259efa09/sqlalchemy-2.0.49-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:df2d441bacf97022e81ad047e1597552eb3f83ca8a8f1a1fdd43cd7fe3898120", size = 2154547, upload-time = "2026-04-03T16:53:08.64Z" }, - { url = "https://files.pythonhosted.org/packages/a2/bc/3494270da80811d08bcfa247404292428c4fe16294932bce5593f215cad9/sqlalchemy-2.0.49-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8e20e511dc15265fb433571391ba313e10dd8ea7e509d51686a51313b4ac01a2", size = 3280782, upload-time = "2026-04-03T17:07:43.508Z" }, - { url = "https://files.pythonhosted.org/packages/cd/f5/038741f5e747a5f6ea3e72487211579d8cbea5eb9827a9cbd61d0108c4bd/sqlalchemy-2.0.49-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47604cb2159f8bbd5a1ab48a714557156320f20871ee64d550d8bf2683d980d3", size = 3297156, upload-time = "2026-04-03T17:12:27.697Z" }, - { url = "https://files.pythonhosted.org/packages/88/50/a6af0ff9dc954b43a65ca9b5367334e45d99684c90a3d3413fc19a02d43c/sqlalchemy-2.0.49-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:22d8798819f86720bc646ab015baff5ea4c971d68121cb36e2ebc2ee43ead2b7", size = 3228832, upload-time = "2026-04-03T17:07:45.38Z" }, - { url = "https://files.pythonhosted.org/packages/bc/d1/5f6bdad8de0bf546fc74370939621396515e0cdb9067402d6ba1b8afbe9a/sqlalchemy-2.0.49-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9b1c058c171b739e7c330760044803099c7fff11511e3ab3573e5327116a9c33", size = 3267000, upload-time = "2026-04-03T17:12:29.657Z" }, - { url = "https://files.pythonhosted.org/packages/f7/30/ad62227b4a9819a5e1c6abff77c0f614fa7c9326e5a3bdbee90f7139382b/sqlalchemy-2.0.49-cp313-cp313-win32.whl", hash = "sha256:a143af2ea6672f2af3f44ed8f9cd020e9cc34c56f0e8db12019d5d9ecf41cb3b", size = 2115641, upload-time = "2026-04-03T17:05:43.989Z" }, - { url = "https://files.pythonhosted.org/packages/17/3a/7215b1b7d6d49dc9a87211be44562077f5f04f9bb5a59552c1c8e2d98173/sqlalchemy-2.0.49-cp313-cp313-win_amd64.whl", hash = "sha256:12b04d1db2663b421fe072d638a138460a51d5a862403295671c4f3987fb9148", size = 2141498, upload-time = "2026-04-03T17:05:45.7Z" }, - { url = "https://files.pythonhosted.org/packages/28/4b/52a0cb2687a9cd1648252bb257be5a1ba2c2ded20ba695c65756a55a15a4/sqlalchemy-2.0.49-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:24bd94bb301ec672d8f0623eba9226cc90d775d25a0c92b5f8e4965d7f3a1518", size = 3560807, upload-time = "2026-04-03T16:58:31.666Z" }, - { url = "https://files.pythonhosted.org/packages/8c/d8/fda95459204877eed0458550d6c7c64c98cc50c2d8d618026737de9ed41a/sqlalchemy-2.0.49-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a51d3db74ba489266ef55c7a4534eb0b8db9a326553df481c11e5d7660c8364d", size = 3527481, upload-time = "2026-04-03T17:06:00.155Z" }, - { url = "https://files.pythonhosted.org/packages/ff/0a/2aac8b78ac6487240cf7afef8f203ca783e8796002dc0cf65c4ee99ff8bb/sqlalchemy-2.0.49-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:55250fe61d6ebfd6934a272ee16ef1244e0f16b7af6cd18ab5b1fc9f08631db0", size = 3468565, upload-time = "2026-04-03T16:58:33.414Z" }, - { url = "https://files.pythonhosted.org/packages/a5/3d/ce71cfa82c50a373fd2148b3c870be05027155ce791dc9a5dcf439790b8b/sqlalchemy-2.0.49-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:46796877b47034b559a593d7e4b549aba151dae73f9e78212a3478161c12ab08", size = 3477769, upload-time = "2026-04-03T17:06:02.787Z" }, - { url = "https://files.pythonhosted.org/packages/d5/e8/0a9f5c1f7c6f9ca480319bf57c2d7423f08d31445974167a27d14483c948/sqlalchemy-2.0.49-cp313-cp313t-win32.whl", hash = "sha256:9c4969a86e41454f2858256c39bdfb966a20961e9b58bf8749b65abf447e9a8d", size = 2143319, upload-time = "2026-04-03T17:02:04.328Z" }, - { url = "https://files.pythonhosted.org/packages/0e/51/fb5240729fbec73006e137c4f7a7918ffd583ab08921e6ff81a999d6517a/sqlalchemy-2.0.49-cp313-cp313t-win_amd64.whl", hash = "sha256:b9870d15ef00e4d0559ae10ee5bc71b654d1f20076dbe8bc7ed19b4c0625ceba", size = 2175104, upload-time = "2026-04-03T17:02:05.989Z" }, - { url = "https://files.pythonhosted.org/packages/55/33/bf28f618c0a9597d14e0b9ee7d1e0622faff738d44fe986ee287cdf1b8d0/sqlalchemy-2.0.49-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:233088b4b99ebcbc5258c755a097aa52fbf90727a03a5a80781c4b9c54347a2e", size = 2156356, upload-time = "2026-04-03T16:53:09.914Z" }, - { url = "https://files.pythonhosted.org/packages/d1/a7/5f476227576cb8644650eff68cc35fa837d3802b997465c96b8340ced1e2/sqlalchemy-2.0.49-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57ca426a48eb2c682dae8204cd89ea8ab7031e2675120a47924fabc7caacbc2a", size = 3276486, upload-time = "2026-04-03T17:07:46.9Z" }, - { url = "https://files.pythonhosted.org/packages/2e/84/efc7c0bf3a1c5eef81d397f6fddac855becdbb11cb38ff957888603014a7/sqlalchemy-2.0.49-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:685e93e9c8f399b0c96a624799820176312f5ceef958c0f88215af4013d29066", size = 3281479, upload-time = "2026-04-03T17:12:32.226Z" }, - { url = "https://files.pythonhosted.org/packages/91/68/bb406fa4257099c67bd75f3f2261b129c63204b9155de0d450b37f004698/sqlalchemy-2.0.49-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9e0400fa22f79acc334d9a6b185dc00a44a8e6578aa7e12d0ddcd8434152b187", size = 3226269, upload-time = "2026-04-03T17:07:48.678Z" }, - { url = "https://files.pythonhosted.org/packages/67/84/acb56c00cca9f251f437cb49e718e14f7687505749ea9255d7bd8158a6df/sqlalchemy-2.0.49-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a05977bffe9bffd2229f477fa75eabe3192b1b05f408961d1bebff8d1cd4d401", size = 3248260, upload-time = "2026-04-03T17:12:34.381Z" }, - { url = "https://files.pythonhosted.org/packages/56/19/6a20ea25606d1efd7bd1862149bb2a22d1451c3f851d23d887969201633f/sqlalchemy-2.0.49-cp314-cp314-win32.whl", hash = "sha256:0f2fa354ba106eafff2c14b0cc51f22801d1e8b2e4149342023bd6f0955de5f5", size = 2118463, upload-time = "2026-04-03T17:05:47.093Z" }, - { url = "https://files.pythonhosted.org/packages/cf/4f/8297e4ed88e80baa1f5aa3c484a0ee29ef3c69c7582f206c916973b75057/sqlalchemy-2.0.49-cp314-cp314-win_amd64.whl", hash = "sha256:77641d299179c37b89cf2343ca9972c88bb6eef0d5fc504a2f86afd15cd5adf5", size = 2144204, upload-time = "2026-04-03T17:05:48.694Z" }, - { url = "https://files.pythonhosted.org/packages/1f/33/95e7216df810c706e0cd3655a778604bbd319ed4f43333127d465a46862d/sqlalchemy-2.0.49-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c1dc3368794d522f43914e03312202523cc89692f5389c32bea0233924f8d977", size = 3565474, upload-time = "2026-04-03T16:58:35.128Z" }, - { url = "https://files.pythonhosted.org/packages/0c/a4/ed7b18d8ccf7f954a83af6bb73866f5bc6f5636f44c7731fbb741f72cc4f/sqlalchemy-2.0.49-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7c821c47ecfe05cc32140dcf8dc6fd5d21971c86dbd56eabfe5ba07a64910c01", size = 3530567, upload-time = "2026-04-03T17:06:04.587Z" }, - { url = "https://files.pythonhosted.org/packages/73/a3/20faa869c7e21a827c4a2a42b41353a54b0f9f5e96df5087629c306df71e/sqlalchemy-2.0.49-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9c04bff9a5335eb95c6ecf1c117576a0aa560def274876fd156cfe5510fccc61", size = 3474282, upload-time = "2026-04-03T16:58:37.131Z" }, - { url = "https://files.pythonhosted.org/packages/b7/50/276b9a007aa0764304ad467eceb70b04822dc32092492ee5f322d559a4dc/sqlalchemy-2.0.49-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7f605a456948c35260e7b2a39f8952a26f077fd25653c37740ed186b90aaa68a", size = 3480406, upload-time = "2026-04-03T17:06:07.176Z" }, - { url = "https://files.pythonhosted.org/packages/e5/c3/c80fcdb41905a2df650c2a3e0337198b6848876e63d66fe9188ef9003d24/sqlalchemy-2.0.49-cp314-cp314t-win32.whl", hash = "sha256:6270d717b11c5476b0cbb21eedc8d4dbb7d1a956fd6c15a23e96f197a6193158", size = 2149151, upload-time = "2026-04-03T17:02:07.281Z" }, - { url = "https://files.pythonhosted.org/packages/05/52/9f1a62feab6ed368aff068524ff414f26a6daebc7361861035ae00b05530/sqlalchemy-2.0.49-cp314-cp314t-win_amd64.whl", hash = "sha256:275424295f4256fd301744b8f335cff367825d270f155d522b30c7bf49903ee7", size = 2184178, upload-time = "2026-04-03T17:02:08.623Z" }, - { url = "https://files.pythonhosted.org/packages/e5/30/8519fdde58a7bdf155b714359791ad1dc018b47d60269d5d160d311fdc36/sqlalchemy-2.0.49-py3-none-any.whl", hash = "sha256:ec44cfa7ef1a728e88ad41674de50f6db8cfdb3e2af84af86e0041aaf02d43d0", size = 1942158, upload-time = "2026-04-03T16:53:44.135Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/57/da/6fbf010c8ebb347679d0d100b22fe9ba5e13fd04046c5df7280d2f0bf706/sqlalchemy-2.0.50.tar.gz", hash = "sha256:af5607d11ef90fd6a5c0549fe0045dce1663d427426bcfb506dcb5346a85a3b9", size = 9907424, upload-time = "2026-05-24T19:20:04.018Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/c4/c42356b527296e9862f67990efce31ef78b4cf69cd3f80873a528a060320/sqlalchemy-2.0.50-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:06a9210bdc5f4298cff0781087e2ff45683922252dacc452846373a58761f093", size = 2156697, upload-time = "2026-05-24T19:27:54.764Z" }, + { url = "https://files.pythonhosted.org/packages/60/a1/b1a70e3c4365ac7fe9e347f3710f19b562c866fb96d45e3c891588789a7b/sqlalchemy-2.0.50-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b53784972ade4f8174b9aa661f31a06f8a936d2cfdd602913ff3c6dd40ae873", size = 3284260, upload-time = "2026-05-24T20:09:34.195Z" }, + { url = "https://files.pythonhosted.org/packages/3f/4a/f3ac3caa19f263d57b0a47f8c91bbf56583dc2d3fc63acfbf644abb24fe0/sqlalchemy-2.0.50-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:31648fa14460537e768a7303b078e4344d208e0d23e06867c1f376a227ed82db", size = 3302280, upload-time = "2026-05-24T20:17:17.825Z" }, + { url = "https://files.pythonhosted.org/packages/66/55/ccada3e3d62254587819749a0bc69f41173eb48a6e385d10e66d32a9c88e/sqlalchemy-2.0.50-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:03f4323c980ad0e918cc9e5369b015f759f4e534db5bbaf4dc36832c10d05064", size = 3231580, upload-time = "2026-05-24T20:09:36.406Z" }, + { url = "https://files.pythonhosted.org/packages/05/f6/6809349130a2de0e109e7f00fd7d431da9565b9b2868b32ee684754f672b/sqlalchemy-2.0.50-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2b9dcc43afef8ac157cd92fce96985d6b8b0cfbd3df4d666f66b4d55a75d202f", size = 3269375, upload-time = "2026-05-24T20:17:20.34Z" }, + { url = "https://files.pythonhosted.org/packages/48/84/278a811ef4e07be9c89dc5cdd7be833268509a66a68c4897cf585e67428f/sqlalchemy-2.0.50-cp313-cp313-win32.whl", hash = "sha256:60922d6599065ddca2c6f376b9aa2f41a6b85a271725e0909490bbc50b1998a5", size = 2117229, upload-time = "2026-05-24T19:50:08.215Z" }, + { url = "https://files.pythonhosted.org/packages/f6/1c/067cc6187ed32d2ec222fe6d2643acc1659a6d0659f8a7cbc5ad3ae83280/sqlalchemy-2.0.50-cp313-cp313-win_amd64.whl", hash = "sha256:287086e67275a212c4582d166a6fb03a65ccc5551d80866270ce0dd9f34eccd3", size = 2143126, upload-time = "2026-05-24T19:50:09.691Z" }, + { url = "https://files.pythonhosted.org/packages/df/32/10ac51b4be7cdecd7e93d069251c86dfbf70b7adbd7c67b48ccea6c49e1c/sqlalchemy-2.0.50-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c966932507a4d7d0a37314927dbfcd89720e3f37d2a1e3352e7ae7939fa8e8a0", size = 2158519, upload-time = "2026-05-24T19:27:56.472Z" }, + { url = "https://files.pythonhosted.org/packages/5a/76/e703d2f7681d7d66c4c891af3f07c7ccf4c76ad7f18351de035b5eda007a/sqlalchemy-2.0.50-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:faffef4bcc20a1892e65e155293d99d60855bbbc79250ab712819cfd56a8e6bb", size = 3282063, upload-time = "2026-05-24T20:09:38.57Z" }, + { url = "https://files.pythonhosted.org/packages/31/26/ef168b184a25701f9995e8fb7e503fafd7a99c1c77cda1bc1a26ea2ed486/sqlalchemy-2.0.50-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c206aec519a2e7bd08abbfb33436e325fd22c632d9c21a9047e376ce241646e", size = 3287069, upload-time = "2026-05-24T20:17:21.942Z" }, + { url = "https://files.pythonhosted.org/packages/c2/15/765acc2bc693bccc43ca4a95d5b69750da8aaf6db1b5c616536e087f8920/sqlalchemy-2.0.50-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bef4ac756363227ef6402a75fee025a4bc690f92328e825868939b3b3a446a6d", size = 3230453, upload-time = "2026-05-24T20:09:40.398Z" }, + { url = "https://files.pythonhosted.org/packages/63/61/08e03c3adbf5db0087a0b6816746fec8f3032fb2f7fc899a9bb9b2a48ce4/sqlalchemy-2.0.50-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:96fbee6b19c19cd1556c8bf9419447cf2ec149ffcab7ab64348c23e54ef8547f", size = 3252413, upload-time = "2026-05-24T20:17:24.067Z" }, + { url = "https://files.pythonhosted.org/packages/03/0c/370a1f2db38436c615e10134c8a37de3688e74084792380695f3f5083860/sqlalchemy-2.0.50-cp314-cp314-win32.whl", hash = "sha256:8f00e3eb43ba30eb1b238ee03a8a62309486d1321eda3328bb611e0340033ad8", size = 2120063, upload-time = "2026-05-24T19:50:11.08Z" }, + { url = "https://files.pythonhosted.org/packages/7f/a0/fe92bb9817863bc13ba093bda931979a26cc2ca69f8e8f26d07add3d7c6f/sqlalchemy-2.0.50-cp314-cp314-win_amd64.whl", hash = "sha256:15708c613cd5005b7dffe1f66ee6a63ee8f5e46799f71c70ebad74178c676a39", size = 2145830, upload-time = "2026-05-24T19:50:12.452Z" }, + { url = "https://files.pythonhosted.org/packages/cc/ff/e5640a98a0b2f491eb8fde10fb6c773621a2e44340de231fafcc9370f4a9/sqlalchemy-2.0.50-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3699dac4be410e97049a1658e9480da9cde956594aa0f3aebc60b88f21c5ba70", size = 2178435, upload-time = "2026-05-24T19:42:58.889Z" }, + { url = "https://files.pythonhosted.org/packages/b7/85/337116e186f1236375b5fb70c21cfac98e8e8ab0d3a47be838dc47a59e08/sqlalchemy-2.0.50-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f96233858e3df43932ac11589e22520da6e8aeb624b03fedfeebb0e8ea213086", size = 3566059, upload-time = "2026-05-24T20:01:20.848Z" }, + { url = "https://files.pythonhosted.org/packages/96/34/bb0e190e161c3c2c24314a65add57218be14a4a9486886b7f5047c1ff7c8/sqlalchemy-2.0.50-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c4e70c46fad30c3bcc6a4708bc0130a3173e11a5b25f0ea4a9d8911b450f1f52", size = 3535366, upload-time = "2026-05-24T20:03:56.768Z" }, + { url = "https://files.pythonhosted.org/packages/df/5a/a7f759f97e4fd499c5d4e4488c760d5a7fbecf3028b465a04274fcd52384/sqlalchemy-2.0.50-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1918a3cf564d16d95bca7301005f41ab2ad50b07cd3b9da50d3ed986db148d6a", size = 3474879, upload-time = "2026-05-24T20:01:23.058Z" }, + { url = "https://files.pythonhosted.org/packages/9d/d9/2907ea38eb60687d297bf9c39e5ee58053c87b57fe8a9cae97090cecbf10/sqlalchemy-2.0.50-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b00098cdbdbd38c7be3d568b0c9c3122b8c0ec62b911b57cd5e6e0254d60a76d", size = 3486117, upload-time = "2026-05-24T20:03:59.052Z" }, + { url = "https://files.pythonhosted.org/packages/f2/e3/5aa06f167559f8c0bdae487e297d23ba548150ab016a3418265d617a4985/sqlalchemy-2.0.50-cp314-cp314t-win32.whl", hash = "sha256:1fbd55a969d7ac44a98e3dec75016074f809fa08f871585ace58dde110d1bf3e", size = 2150823, upload-time = "2026-05-24T20:08:58.644Z" }, + { url = "https://files.pythonhosted.org/packages/65/9b/112fb8f977582d7489d036e409e3723948bcf5320b3ac465f3c481bbe8f9/sqlalchemy-2.0.50-cp314-cp314t-win_amd64.whl", hash = "sha256:c5c3cdb753a9004183e1ccb634b41611654c989e61bc68617ce878e46d6f1e51", size = 2185794, upload-time = "2026-05-24T20:09:00.319Z" }, + { url = "https://files.pythonhosted.org/packages/d0/10/f7220e9b784d295d241c86ed99aeb537f92afcd469a64861f2717e9bb077/sqlalchemy-2.0.50-py3-none-any.whl", hash = "sha256:92064363517a3ff8212b5a93b8c62876579d8dfd1ca5b561335f30152d884fa9", size = 1943861, upload-time = "2026-05-24T19:59:01.119Z" }, ] [[package]] name = "sqlglot" -version = "30.4.3" +version = "30.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/64/89299aefc6ebdf4fc899f5dc14c7fcb7eb9da9290a2b4d615ae7ab884b17/sqlglot-30.8.0.tar.gz", hash = "sha256:1c5f93fb742dd9aaa75eee6bb33a637794a858b9a86375fac23a2dc0f7bc127e", size = 5869750, upload-time = "2026-05-13T09:04:38.923Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/4e/80705091aaf9c95e125d243f0aa871bc9f3670b4c9d963e6bad3b3dce8ff/sqlglot-30.8.0-py3-none-any.whl", hash = "sha256:af903378c331d5b72277a1b41118f07bc3e50cf4478e2d47eed12c96ee6a22a4", size = 687831, upload-time = "2026-05-13T09:04:36.336Z" }, +] + +[[package]] +name = "sse-starlette" +version = "3.4.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f8/9b/af8fcaca1b0821349ef4f88e2775e059bcd7640900bca6533832f1fb845d/sqlglot-30.4.3.tar.gz", hash = "sha256:3a4e9a1e1dd47f8e536ba822d77cb784681704da5e4a3e1a07d2ef86b6067826", size = 5827662, upload-time = "2026-04-13T17:05:15.456Z" } +dependencies = [ + { name = "anyio" }, + { name = "starlette" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f7/2b/58abc2d1fd397e7dde08e947e05c884d8ef2f78d5e2588c17a12d42d6994/sse_starlette-3.4.4.tar.gz", hash = "sha256:07e0fa0460138baf25cdd5fb28683472c3995dc1642225191b3832d62526bcb0", size = 31819, upload-time = "2026-05-12T17:37:17.019Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ee/19/7df8b292accba3bc0de92c611c1e89423b25c08c82c18b14ca1fdbcf6e44/sqlglot-30.4.3-py3-none-any.whl", hash = "sha256:58ea8e723444569da5cec91e4c8f16e385bce3f0ce0374b8c722c3088e1c1c7a", size = 670965, upload-time = "2026-04-13T17:05:13.128Z" }, + { url = "https://files.pythonhosted.org/packages/dc/67/805710444ea8cc75fbf70b920ed431a560c4bf9c57f7d5a3117213189399/sse_starlette-3.4.4-py3-none-any.whl", hash = "sha256:3f4dd50d8aed2771a091f3a83000323fc3844541c16b4fe585ae2420cc6df973", size = 16514, upload-time = "2026-05-12T17:37:15.601Z" }, ] [[package]] name = "sssom-pydantic" -version = "0.5.1" +version = "0.5.9" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "curies" }, @@ -3278,9 +3694,9 @@ dependencies = [ { name = "pyyaml" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f6/62/93e5afcf102f5f30981f32bc663ee7662031d27ced22f725c052cbffb592/sssom_pydantic-0.5.1.tar.gz", hash = "sha256:7efa265c9e9d142cfb4ad9aace68691149fb737cb42e45e4bd1e2d889a0eb360", size = 54262, upload-time = "2026-04-14T19:58:24.927Z" } +sdist = { url = "https://files.pythonhosted.org/packages/85/4a/86451b0a6410089b1734fa612f8989392aec5856453686d251ba99bbf465/sssom_pydantic-0.5.9.tar.gz", hash = "sha256:64870feaa5ef8f5b41fcac4028a5c51b0fd8b74ea7ee1c55ed23687119cc3b29", size = 61842, upload-time = "2026-05-26T20:04:21.393Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/72/7c16e5130388312d2b78ad9e4a2bb490d1acf69a44fc08db80ce4acdc890/sssom_pydantic-0.5.1-py3-none-any.whl", hash = "sha256:a1a1f5daefa77fd5227ef3105159fb49b683006ccc4ccac29d18eb4f1288b7a4", size = 67360, upload-time = "2026-04-14T19:58:23.701Z" }, + { url = "https://files.pythonhosted.org/packages/80/04/2a079709862c79484dec82a91feaab09dbd7442a4f2b529214927f79e0a9/sssom_pydantic-0.5.9-py3-none-any.whl", hash = "sha256:251cb1d7c09ff400f30487aa04d7dbdc8eaa5547ae7e84ae93a4569206b174e8", size = 75361, upload-time = "2026-05-26T20:04:19.33Z" }, ] [[package]] @@ -3297,6 +3713,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" }, ] +[[package]] +name = "starlette" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/66/4d20cdf39a8d6a51e663b7038e3b828ff211d3891a43a713fe7e4643f3a8/starlette-1.1.0.tar.gz", hash = "sha256:e83c7fe0ddecd8719c5b840080325aec0260acec86e9832899e377b91d65e90f", size = 2660060, upload-time = "2026-05-23T16:55:41.376Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/79/920b8e0a8b20f793e8d64855095cb8febabf6175b8550b6f7a547d813891/starlette-1.1.0-py3-none-any.whl", hash = "sha256:7f0dfd38e428aad5cb6f9f667f0ca1d2d8ca3f3385dccac8305f79ec98458382", size = 72899, upload-time = "2026-05-23T16:55:39.201Z" }, +] + [[package]] name = "tabulate" version = "0.10.0" @@ -3326,57 +3754,57 @@ wheels = [ [[package]] name = "thrift" -version = "0.22.0" +version = "0.23.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b2/c2/db648cc10dd7d15560f2eafd92a27cd280811924696e0b4a87175fb28c94/thrift-0.22.0.tar.gz", hash = "sha256:42e8276afbd5f54fe1d364858b6877bc5e5a4a5ed69f6a005b94ca4918fe1466", size = 62303, upload-time = "2025-05-23T20:49:33.309Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d2/fd/6766a2a1a2532dbcb0fa8e31fd9a22de78b7338c7a07b5e5a6ef4e8913f1/thrift-0.23.0.tar.gz", hash = "sha256:5f43448a92c36ed6a450048355d10e231a1787e4c28965f08fabac0eb978914c", size = 66440, upload-time = "2026-05-14T08:35:49.196Z" } [[package]] name = "tiktoken" -version = "0.12.0" +version = "0.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "regex" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7d/ab/4d017d0f76ec3171d469d80fc03dfbb4e48a4bcaddaa831b31d526f05edc/tiktoken-0.12.0.tar.gz", hash = "sha256:b18ba7ee2b093863978fcb14f74b3707cdc8d4d4d3836853ce7ec60772139931", size = 37806, upload-time = "2025-10-06T20:22:45.419Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/00/61/441588ee21e6b5cdf59d6870f86beb9789e532ee9718c251b391b70c68d6/tiktoken-0.12.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:775c2c55de2310cc1bc9a3ad8826761cbdc87770e586fd7b6da7d4589e13dab3", size = 1050802, upload-time = "2025-10-06T20:22:00.96Z" }, - { url = "https://files.pythonhosted.org/packages/1f/05/dcf94486d5c5c8d34496abe271ac76c5b785507c8eae71b3708f1ad9b45a/tiktoken-0.12.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a01b12f69052fbe4b080a2cfb867c4de12c704b56178edf1d1d7b273561db160", size = 993995, upload-time = "2025-10-06T20:22:02.788Z" }, - { url = "https://files.pythonhosted.org/packages/a0/70/5163fe5359b943f8db9946b62f19be2305de8c3d78a16f629d4165e2f40e/tiktoken-0.12.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:01d99484dc93b129cd0964f9d34eee953f2737301f18b3c7257bf368d7615baa", size = 1128948, upload-time = "2025-10-06T20:22:03.814Z" }, - { url = "https://files.pythonhosted.org/packages/0c/da/c028aa0babf77315e1cef357d4d768800c5f8a6de04d0eac0f377cb619fa/tiktoken-0.12.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:4a1a4fcd021f022bfc81904a911d3df0f6543b9e7627b51411da75ff2fe7a1be", size = 1151986, upload-time = "2025-10-06T20:22:05.173Z" }, - { url = "https://files.pythonhosted.org/packages/a0/5a/886b108b766aa53e295f7216b509be95eb7d60b166049ce2c58416b25f2a/tiktoken-0.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:981a81e39812d57031efdc9ec59fa32b2a5a5524d20d4776574c4b4bd2e9014a", size = 1194222, upload-time = "2025-10-06T20:22:06.265Z" }, - { url = "https://files.pythonhosted.org/packages/f4/f8/4db272048397636ac7a078d22773dd2795b1becee7bc4922fe6207288d57/tiktoken-0.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9baf52f84a3f42eef3ff4e754a0db79a13a27921b457ca9832cf944c6be4f8f3", size = 1255097, upload-time = "2025-10-06T20:22:07.403Z" }, - { url = "https://files.pythonhosted.org/packages/8e/32/45d02e2e0ea2be3a9ed22afc47d93741247e75018aac967b713b2941f8ea/tiktoken-0.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:b8a0cd0c789a61f31bf44851defbd609e8dd1e2c8589c614cc1060940ef1f697", size = 879117, upload-time = "2025-10-06T20:22:08.418Z" }, - { url = "https://files.pythonhosted.org/packages/ce/76/994fc868f88e016e6d05b0da5ac24582a14c47893f4474c3e9744283f1d5/tiktoken-0.12.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d5f89ea5680066b68bcb797ae85219c72916c922ef0fcdd3480c7d2315ffff16", size = 1050309, upload-time = "2025-10-06T20:22:10.939Z" }, - { url = "https://files.pythonhosted.org/packages/f6/b8/57ef1456504c43a849821920d582a738a461b76a047f352f18c0b26c6516/tiktoken-0.12.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b4e7ed1c6a7a8a60a3230965bdedba8cc58f68926b835e519341413370e0399a", size = 993712, upload-time = "2025-10-06T20:22:12.115Z" }, - { url = "https://files.pythonhosted.org/packages/72/90/13da56f664286ffbae9dbcfadcc625439142675845baa62715e49b87b68b/tiktoken-0.12.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:fc530a28591a2d74bce821d10b418b26a094bf33839e69042a6e86ddb7a7fb27", size = 1128725, upload-time = "2025-10-06T20:22:13.541Z" }, - { url = "https://files.pythonhosted.org/packages/05/df/4f80030d44682235bdaecd7346c90f67ae87ec8f3df4a3442cb53834f7e4/tiktoken-0.12.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:06a9f4f49884139013b138920a4c393aa6556b2f8f536345f11819389c703ebb", size = 1151875, upload-time = "2025-10-06T20:22:14.559Z" }, - { url = "https://files.pythonhosted.org/packages/22/1f/ae535223a8c4ef4c0c1192e3f9b82da660be9eb66b9279e95c99288e9dab/tiktoken-0.12.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:04f0e6a985d95913cabc96a741c5ffec525a2c72e9df086ff17ebe35985c800e", size = 1194451, upload-time = "2025-10-06T20:22:15.545Z" }, - { url = "https://files.pythonhosted.org/packages/78/a7/f8ead382fce0243cb625c4f266e66c27f65ae65ee9e77f59ea1653b6d730/tiktoken-0.12.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0ee8f9ae00c41770b5f9b0bb1235474768884ae157de3beb5439ca0fd70f3e25", size = 1253794, upload-time = "2025-10-06T20:22:16.624Z" }, - { url = "https://files.pythonhosted.org/packages/93/e0/6cc82a562bc6365785a3ff0af27a2a092d57c47d7a81d9e2295d8c36f011/tiktoken-0.12.0-cp313-cp313t-win_amd64.whl", hash = "sha256:dc2dd125a62cb2b3d858484d6c614d136b5b848976794edfb63688d539b8b93f", size = 878777, upload-time = "2025-10-06T20:22:18.036Z" }, - { url = "https://files.pythonhosted.org/packages/72/05/3abc1db5d2c9aadc4d2c76fa5640134e475e58d9fbb82b5c535dc0de9b01/tiktoken-0.12.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:a90388128df3b3abeb2bfd1895b0681412a8d7dc644142519e6f0a97c2111646", size = 1050188, upload-time = "2025-10-06T20:22:19.563Z" }, - { url = "https://files.pythonhosted.org/packages/e3/7b/50c2f060412202d6c95f32b20755c7a6273543b125c0985d6fa9465105af/tiktoken-0.12.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:da900aa0ad52247d8794e307d6446bd3cdea8e192769b56276695d34d2c9aa88", size = 993978, upload-time = "2025-10-06T20:22:20.702Z" }, - { url = "https://files.pythonhosted.org/packages/14/27/bf795595a2b897e271771cd31cb847d479073497344c637966bdf2853da1/tiktoken-0.12.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:285ba9d73ea0d6171e7f9407039a290ca77efcdb026be7769dccc01d2c8d7fff", size = 1129271, upload-time = "2025-10-06T20:22:22.06Z" }, - { url = "https://files.pythonhosted.org/packages/f5/de/9341a6d7a8f1b448573bbf3425fa57669ac58258a667eb48a25dfe916d70/tiktoken-0.12.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:d186a5c60c6a0213f04a7a802264083dea1bbde92a2d4c7069e1a56630aef830", size = 1151216, upload-time = "2025-10-06T20:22:23.085Z" }, - { url = "https://files.pythonhosted.org/packages/75/0d/881866647b8d1be4d67cb24e50d0c26f9f807f994aa1510cb9ba2fe5f612/tiktoken-0.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:604831189bd05480f2b885ecd2d1986dc7686f609de48208ebbbddeea071fc0b", size = 1194860, upload-time = "2025-10-06T20:22:24.602Z" }, - { url = "https://files.pythonhosted.org/packages/b3/1e/b651ec3059474dab649b8d5b69f5c65cd8fcd8918568c1935bd4136c9392/tiktoken-0.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8f317e8530bb3a222547b85a58583238c8f74fd7a7408305f9f63246d1a0958b", size = 1254567, upload-time = "2025-10-06T20:22:25.671Z" }, - { url = "https://files.pythonhosted.org/packages/80/57/ce64fd16ac390fafde001268c364d559447ba09b509181b2808622420eec/tiktoken-0.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:399c3dd672a6406719d84442299a490420b458c44d3ae65516302a99675888f3", size = 921067, upload-time = "2025-10-06T20:22:26.753Z" }, - { url = "https://files.pythonhosted.org/packages/ac/a4/72eed53e8976a099539cdd5eb36f241987212c29629d0a52c305173e0a68/tiktoken-0.12.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:c2c714c72bc00a38ca969dae79e8266ddec999c7ceccd603cc4f0d04ccd76365", size = 1050473, upload-time = "2025-10-06T20:22:27.775Z" }, - { url = "https://files.pythonhosted.org/packages/e6/d7/0110b8f54c008466b19672c615f2168896b83706a6611ba6e47313dbc6e9/tiktoken-0.12.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:cbb9a3ba275165a2cb0f9a83f5d7025afe6b9d0ab01a22b50f0e74fee2ad253e", size = 993855, upload-time = "2025-10-06T20:22:28.799Z" }, - { url = "https://files.pythonhosted.org/packages/5f/77/4f268c41a3957c418b084dd576ea2fad2e95da0d8e1ab705372892c2ca22/tiktoken-0.12.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:dfdfaa5ffff8993a3af94d1125870b1d27aed7cb97aa7eb8c1cefdbc87dbee63", size = 1129022, upload-time = "2025-10-06T20:22:29.981Z" }, - { url = "https://files.pythonhosted.org/packages/4e/2b/fc46c90fe5028bd094cd6ee25a7db321cb91d45dc87531e2bdbb26b4867a/tiktoken-0.12.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:584c3ad3d0c74f5269906eb8a659c8bfc6144a52895d9261cdaf90a0ae5f4de0", size = 1150736, upload-time = "2025-10-06T20:22:30.996Z" }, - { url = "https://files.pythonhosted.org/packages/28/c0/3c7a39ff68022ddfd7d93f3337ad90389a342f761c4d71de99a3ccc57857/tiktoken-0.12.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:54c891b416a0e36b8e2045b12b33dd66fb34a4fe7965565f1b482da50da3e86a", size = 1194908, upload-time = "2025-10-06T20:22:32.073Z" }, - { url = "https://files.pythonhosted.org/packages/ab/0d/c1ad6f4016a3968c048545f5d9b8ffebf577774b2ede3e2e352553b685fe/tiktoken-0.12.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5edb8743b88d5be814b1a8a8854494719080c28faaa1ccbef02e87354fe71ef0", size = 1253706, upload-time = "2025-10-06T20:22:33.385Z" }, - { url = "https://files.pythonhosted.org/packages/af/df/c7891ef9d2712ad774777271d39fdef63941ffba0a9d59b7ad1fd2765e57/tiktoken-0.12.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f61c0aea5565ac82e2ec50a05e02a6c44734e91b51c10510b084ea1b8e633a71", size = 920667, upload-time = "2025-10-06T20:22:34.444Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/e4/e5/5f3cb2159769d0f4324c0e9e87f9de3c4b1cd45848a96b2eb3566ad5ca77/tiktoken-0.13.0.tar.gz", hash = "sha256:c9435714c3a84c2319499de9a300c0e604449dd0799ff246458b3bb6a7f433c1", size = 38986, upload-time = "2026-05-15T04:51:27.153Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/83/b096c859c2a47c11731bf2f5885f4028b809dfe2396582883eed9cae372f/tiktoken-0.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5df5d1507bd245f1ccad4a074698240021239e455eb0bb4ced4e3d7181872154", size = 1034228, upload-time = "2026-05-15T04:50:40.988Z" }, + { url = "https://files.pythonhosted.org/packages/53/61/c68e123b6d753e3fc2751e9b18e732c9d8bf1e1926762e736eee935d931c/tiktoken-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8fe806a50664e83a6ffd56cbd1e4f5dcc6cd32a3e7538f70dc38b1a271384545", size = 982978, upload-time = "2026-05-15T04:50:42.195Z" }, + { url = "https://files.pythonhosted.org/packages/ef/8b/96cc178cc584e65d363134500f297790b06cd48cdeb1e8fcf7bbe60f4715/tiktoken-0.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:125bc05005e747f993a83dc67934249932d6e4209854452cd4c0b1d53fba3ba2", size = 1116355, upload-time = "2026-05-15T04:50:43.564Z" }, + { url = "https://files.pythonhosted.org/packages/86/f5/bab735d2c72ea55404b295d02d092644eb5f7cc6205e34d35eb9abfb9ab2/tiktoken-0.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5e6358911cab4adee6712da27d65573496a4f68cf8a2b5fca6a4ad10fc5748cf", size = 1135772, upload-time = "2026-05-15T04:50:44.782Z" }, + { url = "https://files.pythonhosted.org/packages/4e/b9/6de04ebdf904edfaad87788011b3735087a0c9ea671b9027e1e4e965e8c8/tiktoken-0.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:975cbd78d085d75d26b59660e262736dcaed1e35f8f142cd6291025c01d25486", size = 1182415, upload-time = "2026-05-15T04:50:46.422Z" }, + { url = "https://files.pythonhosted.org/packages/0d/9c/470a05f3b1caf038f44880e334d47ab674e0c80d514c66b375d14d5afa10/tiktoken-0.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:75ab9bc99fa020a4c283424590ecd7f3afd70c1c281cb3fa3192a6c3af9f9615", size = 1239879, upload-time = "2026-05-15T04:50:48.052Z" }, + { url = "https://files.pythonhosted.org/packages/42/a6/c1936d16055436cb32e6c6128d68629622e00f4768562f55653752d34768/tiktoken-0.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:6b1615f0ff71953d19729ceb18865429c185b0a23c5353f1bbca34a394bf60f7", size = 874829, upload-time = "2026-05-15T04:50:49.202Z" }, + { url = "https://files.pythonhosted.org/packages/d6/07/acb5992c3772b5a36284f742cfb7a5895aa4471d1848ac31464ad50d7fdf/tiktoken-0.13.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6eb4a5bfbc6426938026b1a334e898ac53541360d62d8c689870160cc80abd67", size = 1033600, upload-time = "2026-05-15T04:50:50.4Z" }, + { url = "https://files.pythonhosted.org/packages/14/e9/742e9aec30f59b9f161f7ff7cd072e02ea836c9e1c0854a8076dfcd40d5c/tiktoken-0.13.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:43cee3e5400573b2046fbf092cc7a5bc30164f9e4c95ce20714da929df48737a", size = 982516, upload-time = "2026-05-15T04:50:52.03Z" }, + { url = "https://files.pythonhosted.org/packages/72/74/ca1541b053e7648254d2e4b42a253e1bb4359f2c91a0a8d49228c794e1a0/tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:7de52e3f566d19b3b11bd37eea552c6c305ad74081f736882bd44d148ed4c48d", size = 1115518, upload-time = "2026-05-15T04:50:53.543Z" }, + { url = "https://files.pythonhosted.org/packages/46/e3/93825eaf5a4a504795b787e5d5dea07fbeb3dabf97aa7b450be8bde59c89/tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:51384448aa508e4df84c0f7c1dc3211c7f7b8096325660ee5fc82f3e11b381ce", size = 1136867, upload-time = "2026-05-15T04:50:55.191Z" }, + { url = "https://files.pythonhosted.org/packages/8c/46/002b68de6827091d5ae90b048f326e8aad8d953520950e5ce1508879414f/tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e28157350f7ebf35008dd8e9e0fdb621f976e4230c881099c85e8cf07eaa50e2", size = 1181826, upload-time = "2026-05-15T04:50:56.296Z" }, + { url = "https://files.pythonhosted.org/packages/db/c6/d393e3185a276505182f7abd93fe714f3c444a2be9180798fa052347504e/tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:165cf1820ea4a354985c2490a5205d4cc74661c934aca79dd0368232fff94e0f", size = 1239489, upload-time = "2026-05-15T04:50:57.918Z" }, + { url = "https://files.pythonhosted.org/packages/b7/4d/bc07d1f1635d4897a202acc0ae11c2886eaa7325c359ba4741b47bf8e225/tiktoken-0.13.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6c43a675ca14f6f2749ba7f12075d37456015a24b859f2517b9beb4ef30807ec", size = 873820, upload-time = "2026-05-15T04:50:59.528Z" }, + { url = "https://files.pythonhosted.org/packages/8c/93/0dd6adca026a616c3a92974566b43381eea4b475ce1f36c062b8271a9ac5/tiktoken-0.13.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaaaef47c2406277181d2086484c317bf7fc433e2d5d03ff94f56b0dcec87471", size = 1034977, upload-time = "2026-05-15T04:51:00.957Z" }, + { url = "https://files.pythonhosted.org/packages/d9/77/5ec6e6bc5b30bed6d93f7f2162d8f6b32437b3ba27cb527cfe004f6109c9/tiktoken-0.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ca8b310bd93b3772cb1b7922d915446864860f562bdfe4825c63a0aed3fb28cd", size = 983635, upload-time = "2026-05-15T04:51:02.629Z" }, + { url = "https://files.pythonhosted.org/packages/94/b0/c8ae9aff00d625c50659b4513e707a0462c4bf5d4d6cc1b802103225c02e/tiktoken-0.13.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:32e0c12305105002c047b3bb1070b0dd9a73b0cb3b2856a8972b810e7a4f5881", size = 1116036, upload-time = "2026-05-15T04:51:04.082Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ac/6a5dddd1d0a6018ecb389bd0353e6b4a515eb4d2286611bd0ace1937b9e1/tiktoken-0.13.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:5ba5fd62507a932d1241346179e3b39bc7bf7408f03c272652d93b3bedf5db24", size = 1135544, upload-time = "2026-05-15T04:51:05.229Z" }, + { url = "https://files.pythonhosted.org/packages/f4/b8/585032b4384b2f7dcdaddcb52865c83a701a420d09e3c2b4a2be1c450c57/tiktoken-0.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d108bc2d470fc53c8ecd24f2c0fd2b5f98c33e87cdb6aa2e9b8c5dced703d273", size = 1182217, upload-time = "2026-05-15T04:51:06.517Z" }, + { url = "https://files.pythonhosted.org/packages/cd/b6/993ff1ded3958215fd341a847b8e5ffeb5de473f435296870d314fc91ac4/tiktoken-0.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cb99cb5127449f58d0a2d5f5ccfb390d8dbdfd919c221246caaee29d8725ed51", size = 1239404, upload-time = "2026-05-15T04:51:07.843Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3d/fef7e06e3b33e7538db0ced734cf9fe23b6832d2ac4990c119c377aec55e/tiktoken-0.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:115c4f26ffa11caac8b54eea35c2ad38c612c20a48d35dd15d70a02ac6f51f58", size = 918686, upload-time = "2026-05-15T04:51:08.925Z" }, + { url = "https://files.pythonhosted.org/packages/c1/82/a7fc44582bc32ab00de988a2299bf77c077f59068b233109e34b7d6ca7e6/tiktoken-0.13.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:472527e9132952f2fbf77cd290658bacf003d4d5a3fabc18e5fbd407cbae4d9b", size = 1034454, upload-time = "2026-05-15T04:51:10.035Z" }, + { url = "https://files.pythonhosted.org/packages/37/d0/24d8a890c14f432a05cea669c17bebeaa99f96a7c79523b590f564246411/tiktoken-0.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4e2f67d27c9626cdd25fe33d9313c5cdb3d8d82da646b68d6eb8e7e9c20e6448", size = 982976, upload-time = "2026-05-15T04:51:11.23Z" }, + { url = "https://files.pythonhosted.org/packages/49/b7/2ab43f62788a9266187a9bfc1d3af99ad83e5eaa25fbef168a69cd5ad14f/tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:2b920b35805cd64585a37c3dc7ce65fba4d2d36016be01e1d7942482ca29093a", size = 1115526, upload-time = "2026-05-15T04:51:12.608Z" }, + { url = "https://files.pythonhosted.org/packages/64/39/1494321ed323ce7a14d88e3cd6cb9058625977df1c6961ddc492bd10a9f3/tiktoken-0.13.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:493af3aa28a4aaf2e3d2600a2ee717252c9bf5ab38fff94eb5a02db5ab77e5ad", size = 1136466, upload-time = "2026-05-15T04:51:13.926Z" }, + { url = "https://files.pythonhosted.org/packages/96/d9/dfd086aa2d918c563a140720e0ce296cada1634efd2783d5cf51e05f984e/tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6644c9c2b5cf3916f5a3641d7d12fdb3f006a7b3d9ff6acdaec44e29ab1ff91e", size = 1181863, upload-time = "2026-05-15T04:51:15.025Z" }, + { url = "https://files.pythonhosted.org/packages/2f/68/a18b4f307086954fdae32714cb4f85562e34f9d34ab206e61f1816aa6018/tiktoken-0.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cb65b60b9408563676d874a3a4ee573370066f0dc4e29d84e82e989c6517424", size = 1239218, upload-time = "2026-05-15T04:51:16.103Z" }, + { url = "https://files.pythonhosted.org/packages/16/5b/f2aa703a4fc5d2dff73460a7d46cc2f3f44aa0f3dd8eeb20d2a0ecf68862/tiktoken-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:85b78cc3a2c3d48723ca751fa981f1fedccd54194ca0471b957364353a898b07", size = 918110, upload-time = "2026-05-15T04:51:17.237Z" }, ] [[package]] name = "tomlkit" -version = "0.14.0" +version = "0.15.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/af/14b24e41977adb296d6bd1fb59402cf7d60ce364f90c890bd2ec65c43b5a/tomlkit-0.14.0.tar.gz", hash = "sha256:cf00efca415dbd57575befb1f6634c4f42d2d87dbba376128adb42c121b87064", size = 187167, upload-time = "2026-01-13T01:14:53.304Z" } +sdist = { url = "https://files.pythonhosted.org/packages/51/db/03eaf4331631ef6b27d6e3c9b68c54dc6f0d63d87201fed600cc409307fd/tomlkit-0.15.0.tar.gz", hash = "sha256:7d1a9ecba3086638211b13814ea79c90dd54dd11993564376f3aa92271f5c7a3", size = 161875, upload-time = "2026-05-10T07:38:22.245Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/11/87d6d29fb5d237229d67973a6c9e06e048f01cf4994dee194ab0ea841814/tomlkit-0.14.0-py3-none-any.whl", hash = "sha256:592064ed85b40fa213469f81ac584f67a4f2992509a7c3ea2d632208623a3680", size = 39310, upload-time = "2026-01-13T01:14:51.965Z" }, + { url = "https://files.pythonhosted.org/packages/6a/43/8bd850ee71a191bf072e31302c73a66be413fecdd98fdcd111ecbcce13ca/tomlkit-0.15.0-py3-none-any.whl", hash = "sha256:4dbc8f0fc024412b57ced8757ac7461305126a648ff8c2c807fcb8e133a78738", size = 41328, upload-time = "2026-05-10T07:38:23.517Z" }, ] [[package]] @@ -3388,6 +3816,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f6/17/57b444fd314d5e1593350b9a31d000e7411ba8e17ce12dc7ad54ca76b810/toposort-1.10-py3-none-any.whl", hash = "sha256:cbdbc0d0bee4d2695ab2ceec97fe0679e9c10eab4b2a87a9372b929e70563a87", size = 8500, upload-time = "2023-02-25T20:07:06.538Z" }, ] +[[package]] +name = "tornado" +version = "6.5.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/f1/3173dfa4a18db4a9b03e5d55325559dab51ee653763bb8745a75af491286/tornado-6.5.5.tar.gz", hash = "sha256:192b8f3ea91bd7f1f50c06955416ed76c6b72f96779b962f07f911b91e8d30e9", size = 516006, upload-time = "2026-03-10T21:31:02.067Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/8c/77f5097695f4dd8255ecbd08b2a1ed8ba8b953d337804dd7080f199e12bf/tornado-6.5.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:487dc9cc380e29f58c7ab88f9e27cdeef04b2140862e5076a66fb6bb68bb1bfa", size = 445983, upload-time = "2026-03-10T21:30:44.28Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5e/7625b76cd10f98f1516c36ce0346de62061156352353ef2da44e5c21523c/tornado-6.5.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:65a7f1d46d4bb41df1ac99f5fcb685fb25c7e61613742d5108b010975a9a6521", size = 444246, upload-time = "2026-03-10T21:30:46.571Z" }, + { url = "https://files.pythonhosted.org/packages/b2/04/7b5705d5b3c0fab088f434f9c83edac1573830ca49ccf29fb83bf7178eec/tornado-6.5.5-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e74c92e8e65086b338fd56333fb9a68b9f6f2fe7ad532645a290a464bcf46be5", size = 447229, upload-time = "2026-03-10T21:30:48.273Z" }, + { url = "https://files.pythonhosted.org/packages/34/01/74e034a30ef59afb4097ef8659515e96a39d910b712a89af76f5e4e1f93c/tornado-6.5.5-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:435319e9e340276428bbdb4e7fa732c2d399386d1de5686cb331ec8eee754f07", size = 448192, upload-time = "2026-03-10T21:30:51.22Z" }, + { url = "https://files.pythonhosted.org/packages/be/00/fe9e02c5a96429fce1a1d15a517f5d8444f9c412e0bb9eadfbe3b0fc55bf/tornado-6.5.5-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3f54aa540bdbfee7b9eb268ead60e7d199de5021facd276819c193c0fb28ea4e", size = 448039, upload-time = "2026-03-10T21:30:53.52Z" }, + { url = "https://files.pythonhosted.org/packages/82/9e/656ee4cec0398b1d18d0f1eb6372c41c6b889722641d84948351ae19556d/tornado-6.5.5-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:36abed1754faeb80fbd6e64db2758091e1320f6bba74a4cf8c09cd18ccce8aca", size = 447445, upload-time = "2026-03-10T21:30:55.541Z" }, + { url = "https://files.pythonhosted.org/packages/5a/76/4921c00511f88af86a33de770d64141170f1cfd9c00311aea689949e274e/tornado-6.5.5-cp39-abi3-win32.whl", hash = "sha256:dd3eafaaeec1c7f2f8fdcd5f964e8907ad788fe8a5a32c4426fbbdda621223b7", size = 448582, upload-time = "2026-03-10T21:30:57.142Z" }, + { url = "https://files.pythonhosted.org/packages/2c/23/f6c6112a04d28eed765e374435fb1a9198f73e1ec4b4024184f21faeb1ad/tornado-6.5.5-cp39-abi3-win_amd64.whl", hash = "sha256:6443a794ba961a9f619b1ae926a2e900ac20c34483eea67be4ed8f1e58d3ef7b", size = 448990, upload-time = "2026-03-10T21:30:58.857Z" }, + { url = "https://files.pythonhosted.org/packages/b7/c8/876602cbc96469911f0939f703453c1157b0c826ecb05bdd32e023397d4e/tornado-6.5.5-cp39-abi3-win_arm64.whl", hash = "sha256:2c9a876e094109333f888539ddb2de4361743e5d21eece20688e3e351e4990a6", size = 448016, upload-time = "2026-03-10T21:31:00.43Z" }, +] + [[package]] name = "tqdm" version = "4.67.3" @@ -3429,17 +3874,17 @@ wheels = [ [[package]] name = "typer" -version = "0.24.1" +version = "0.26.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-doc" }, - { name = "click" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "rich" }, { name = "shellingham" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f5/24/cb09efec5cc954f7f9b930bf8279447d24618bb6758d4f6adf2574c41780/typer-0.24.1.tar.gz", hash = "sha256:e39b4732d65fbdcde189ae76cf7cd48aeae72919dea1fdfc16593be016256b45", size = 118613, upload-time = "2026-02-21T16:54:40.609Z" } +sdist = { url = "https://files.pythonhosted.org/packages/08/26/8e9a4f2c98caefcf4ac25788d48939516a9dd4265fcf9bdd578a2a1b55dd/typer-0.26.1.tar.gz", hash = "sha256:537d27ae686d82967f6383382a952cb32ba4768898541effccb69ca75bbd5d23", size = 198884, upload-time = "2026-05-26T17:49:07.912Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4a/91/48db081e7a63bb37284f9fbcefda7c44c277b18b0e13fbc36ea2335b71e6/typer-0.24.1-py3-none-any.whl", hash = "sha256:112c1f0ce578bfb4cab9ffdabc68f031416ebcc216536611ba21f04e9aa84c9e", size = 56085, upload-time = "2026-02-21T16:54:41.616Z" }, + { url = "https://files.pythonhosted.org/packages/be/27/8a22d4833fe8aa0836ce7fa59096ad50d7e93b83be6d5383f11f9a140d54/typer-0.26.1-py3-none-any.whl", hash = "sha256:933e4f0083521f3c57d6a5aedf3b073271b2f95a19761b171b494dd6fdb21ff6", size = 123097, upload-time = "2026-05-26T17:49:09.065Z" }, ] [[package]] @@ -3451,19 +3896,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, ] -[[package]] -name = "typing-inspect" -version = "0.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mypy-extensions" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/dc/74/1789779d91f1961fa9438e9a8710cdae6bd138c80d7303996933d117264a/typing_inspect-0.9.0.tar.gz", hash = "sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78", size = 13825, upload-time = "2023-05-24T20:25:47.612Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f", size = 8827, upload-time = "2023-05-24T20:25:45.287Z" }, -] - [[package]] name = "typing-inspection" version = "0.4.2" @@ -3478,11 +3910,11 @@ wheels = [ [[package]] name = "tzdata" -version = "2026.1" +version = "2026.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/19/f5/cd531b2d15a671a40c0f66cf06bc3570a12cd56eef98960068ebbad1bf5a/tzdata-2026.1.tar.gz", hash = "sha256:67658a1903c75917309e753fdc349ac0efd8c27db7a0cb406a25be4840f87f98", size = 197639, upload-time = "2026-04-03T11:25:22.002Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/19/1b9b0e29f30c6d35cb345486df41110984ea67ae69dddbc0e8a100999493/tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10", size = 198254, upload-time = "2026-04-24T15:22:08.651Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/70/d460bd685a170790ec89317e9bd33047988e4bce507b831f5db771e142de/tzdata-2026.1-py2.py3-none-any.whl", hash = "sha256:4b1d2be7ac37ceafd7327b961aa3a54e467efbdb563a23655fbfe0d39cfc42a9", size = 348952, upload-time = "2026-04-03T11:25:20.313Z" }, + { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321, upload-time = "2026-04-24T15:22:05.876Z" }, ] [[package]] @@ -3517,24 +3949,79 @@ wheels = [ [[package]] name = "uuid-utils" -version = "0.14.1" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/a1/822ceef22d1c139cffebe4b1b660cfaa10253d5c770aa2598dc8e9497593/uuid_utils-0.16.0.tar.gz", hash = "sha256:d6902d4375dfba4c9902c736bb82d3c040417b67f7d0fa48910ddfdb1ac95de7", size = 42596, upload-time = "2026-05-19T07:44:23.28Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/9b/74c1f47a9b4f138a254e51528e5ffaeba6bf99ecead9f0c4b6fccccfbfcb/uuid_utils-0.16.0-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:d34cf9681e8892fad2a63e393068e544505408748cd8bf0c3517d753a01528d4", size = 563166, upload-time = "2026-05-19T07:44:10.494Z" }, + { url = "https://files.pythonhosted.org/packages/7c/1c/009e37b70f1f0ff17e7103a36bafde33d503d9ea7fe739761aa3e3c9fde6/uuid_utils-0.16.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:0681d1bdb7956e0c6d581e7601dabcfb2b08c25d2a65189f4e9b102c94f5ff46", size = 289529, upload-time = "2026-05-19T07:43:54.466Z" }, + { url = "https://files.pythonhosted.org/packages/5e/5e/e0323d54321166639eb2be5e8a464f5cb0fc04d72d91f3e78944bb6a1da8/uuid_utils-0.16.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed45fb8732d216426227096b55accbb87cba57febc86a044d90780b090eb99d0", size = 326328, upload-time = "2026-05-19T07:45:31.901Z" }, + { url = "https://files.pythonhosted.org/packages/f0/a3/046f6cb958467c3bf4a163a8a53b178b64a62e21ed8ad5b2c1dacb3a2cfc/uuid_utils-0.16.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b617a334bb01ef2ff8c22900f5a14125eb9063f602131494cc9dc59519beaa5b", size = 332322, upload-time = "2026-05-19T07:43:41.284Z" }, + { url = "https://files.pythonhosted.org/packages/67/80/01914e3949744db7acd0006885e5542fbebb6e39114857d007d29b3265c2/uuid_utils-0.16.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a750d8aeb8ae880aa9a2529606bde0e994bcc7448730c953107f357a28e6102e", size = 445787, upload-time = "2026-05-19T07:45:36.102Z" }, + { url = "https://files.pythonhosted.org/packages/14/ef/f6908f41279f205d70c8a0d5dcb25dd6802741d7f88e3f0123453c3584d3/uuid_utils-0.16.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a250e111903c4368745fce5ac2aa607bd477c62d3307e45347338fdb64b38e0", size = 324678, upload-time = "2026-05-19T07:45:12.77Z" }, + { url = "https://files.pythonhosted.org/packages/11/4a/bf841ba90f829c7779d82155e0f4b88ef6726ccc25507d064d50ac2cd329/uuid_utils-0.16.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:95b7f480010ea98a29ee809857a98aa923008c68129af1b39244adccff7377fb", size = 349704, upload-time = "2026-05-19T07:44:47.172Z" }, + { url = "https://files.pythonhosted.org/packages/e6/31/3b5c60172b8c57bf4ca485484b8e4edef550ca324f9287f1183be97422e2/uuid_utils-0.16.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:420aa3ca403cedb73490b6ea3aeefeea7e0455f5ce60bbf856390ee872ae3306", size = 502456, upload-time = "2026-05-19T07:45:00.821Z" }, + { url = "https://files.pythonhosted.org/packages/88/bf/3da8d497af80fd51d8bf85551c77ede67f07825924ec5987bf9b6031014a/uuid_utils-0.16.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:b8a9a7b1065a12d40f2cc25b7d705ab34954cc57095034367bca39ebcf4a876b", size = 607727, upload-time = "2026-05-19T07:44:30.058Z" }, + { url = "https://files.pythonhosted.org/packages/bd/4e/7c8cf03ec15cd6f40e4cbab81b2b4a625461327f68c7971e54723280ec3e/uuid_utils-0.16.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f235ac5827d74ac630cc87f29278cdaa5d2f273613a6e05bbd96df7aa4170776", size = 566204, upload-time = "2026-05-19T07:44:51.225Z" }, + { url = "https://files.pythonhosted.org/packages/f9/5f/af955feae69cce7fd2121ca3f790ff4b85ad2e17b2149546f50753e1a047/uuid_utils-0.16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c8083284488b84ad178e74add64cfd1e74e8be5e30821e5acbc5019281c658b0", size = 529986, upload-time = "2026-05-19T07:45:57.85Z" }, + { url = "https://files.pythonhosted.org/packages/10/cf/3fec757e51bef10eb41ae8075f5442c60e85ff456b42d16a3063f5dc6c80/uuid_utils-0.16.0-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:27a071a899ba46a551d6524dbbc5a98b88be176d0f55ddf72cf71c005326ac10", size = 98683, upload-time = "2026-05-19T07:44:16.369Z" }, + { url = "https://files.pythonhosted.org/packages/40/a7/cd1adbea7ef882a70db064c00cd93b12e11027b4cdd7ffd79e95c35fc3e3/uuid_utils-0.16.0-cp313-cp313-win32.whl", hash = "sha256:924a8de04460e4cf65998ad0b6568084f7c51740ebd3254d07a0bcde35a84af6", size = 168822, upload-time = "2026-05-19T07:44:24.09Z" }, + { url = "https://files.pythonhosted.org/packages/74/99/617ceb9e3a95b23837012740979baf71afad723b70daf34862da3f7c17a1/uuid_utils-0.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:5279bc7ab3c6683f1c67314695bee14d869015acbbc677bdb0015190fe753d16", size = 174967, upload-time = "2026-05-19T07:44:56.022Z" }, + { url = "https://files.pythonhosted.org/packages/d9/d8/148ae707bfc36d482e39db679c86b81bdce264d4feb9df5d40a03b7687e3/uuid_utils-0.16.0-cp313-cp313-win_arm64.whl", hash = "sha256:61a9c4c26ad12ac66fa4bfd0fdb8494724fe7a5b98a9fcd43e78e2b388663dbb", size = 173142, upload-time = "2026-05-19T07:43:50.171Z" }, + { url = "https://files.pythonhosted.org/packages/21/05/ca6d60705e71fdeaa3431dad94e279a8213c5573cb2925e1aabf3dc0330a/uuid_utils-0.16.0-cp313-cp313t-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:73486b6aa3f755a6c97000f5ea67e7ac78d6df89bf22980789a1e943e24b74f0", size = 564408, upload-time = "2026-05-19T07:44:38.351Z" }, + { url = "https://files.pythonhosted.org/packages/eb/8c/b9a0462c38535c1662acb1025768e2d626bee5ce9e1790bad6b5381162ea/uuid_utils-0.16.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:f1614572fd9345cdc3dde3f40c237345719fabca1aa87d2d87b321d523cfa34d", size = 289923, upload-time = "2026-05-19T07:45:19.611Z" }, + { url = "https://files.pythonhosted.org/packages/f2/33/a53afeef1a56051551a0f5a801e4bce411dd73c6a8c99bad16902651256d/uuid_utils-0.16.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9346ce6eb1fbd8b03a6b331d66016afcb4edcdff6eac708e21391600529a016a", size = 325762, upload-time = "2026-05-19T07:45:18.261Z" }, + { url = "https://files.pythonhosted.org/packages/72/ca/4462a4f36365d7ee72d41e05e6bcfe127e861b073ab37c25b2c8a518317c/uuid_utils-0.16.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a0fc6eb3fd821466fbab69cf356c6ec2b7327266bbbc740a2eb57c77c4bef965", size = 332359, upload-time = "2026-05-19T07:45:34.886Z" }, + { url = "https://files.pythonhosted.org/packages/c5/67/9d3373fa7c5a746fdecc64e30caf915c29eb632203508d87676f9243ed03/uuid_utils-0.16.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:13a797e5e8f0dadc18351a5aa013815ddac25dce6864072a539d510910c95f71", size = 445483, upload-time = "2026-05-19T07:44:49.598Z" }, + { url = "https://files.pythonhosted.org/packages/57/08/ce01aa6d897fc7f875844fe58cad0a542c8ebf089d9242b654b56260ecb8/uuid_utils-0.16.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:57c3583b1f1c00a94f59726a5e2b988fa209221143919a1af5c2fc24e318fc98", size = 326281, upload-time = "2026-05-19T07:44:59.677Z" }, + { url = "https://files.pythonhosted.org/packages/76/ef/2c719b2c26bb5b5e5061a1435c11ad2bd33ac3cd6d4cd0c7c3ac1d3396ed/uuid_utils-0.16.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:caac9c8b1d50e8fbddc76e93bfefbef472978eb45adbfdb6289d578816992953", size = 350809, upload-time = "2026-05-19T07:45:28.076Z" }, + { url = "https://files.pythonhosted.org/packages/e0/9b/c1ed447328b32229cca38ac4c62d309eab006e5e9c4020e2056a175bc607/uuid_utils-0.16.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:91db59bad97ed2b9d2c6ed25082fe9762b2c422e694fe06786b28cf4e776ac4c", size = 502088, upload-time = "2026-05-19T07:44:09.208Z" }, + { url = "https://files.pythonhosted.org/packages/c1/e0/8442f4efe7bde72f0b4ae5f675d0c7fbe209ad0b54718b8ddf43c46c6fae/uuid_utils-0.16.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:41985e342a30e76366a8becc60bbdb07d72cd1b86ec657b1f31654e9fb1baada", size = 607631, upload-time = "2026-05-19T07:44:19.384Z" }, + { url = "https://files.pythonhosted.org/packages/f1/1e/9a9fa261edf4c972f28ae83421377e3ab8dbd0bd7db58fd316e782d09a3b/uuid_utils-0.16.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:1b0dcedf9266bf34a54d5cbe78648eaa627e02352f2a6923ed647530aea2f661", size = 567618, upload-time = "2026-05-19T07:43:58.478Z" }, + { url = "https://files.pythonhosted.org/packages/cc/f7/1bcfdb9d539bd42736dd6076470a42fbb5db23f79712c0a06aa0a3752f7b/uuid_utils-0.16.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:26fe23ab60f05de4ad70aaa5b6a4c2a7bbd43055e3dd6f6b31efba0532ac9c71", size = 530971, upload-time = "2026-05-19T07:45:06.348Z" }, + { url = "https://files.pythonhosted.org/packages/24/0c/18945f417d6bb4d0dd2b7652fe36c58c4e83bcf593b9b326b83aa40b853a/uuid_utils-0.16.0-cp313-cp313t-win32.whl", hash = "sha256:7f8cf49c05d58523a0f977cb7f11afc05791a0fa164d7303b8365a34750638e7", size = 169369, upload-time = "2026-05-19T07:44:32.581Z" }, + { url = "https://files.pythonhosted.org/packages/cc/cc/c0eb0c3fab2ed80d706369b750029143b53126809b77b36bcbb77da66bab/uuid_utils-0.16.0-cp313-cp313t-win_amd64.whl", hash = "sha256:e99f9a8b2420b228faba23a637e96efaf5c6a678b2e225870f24431c82707f50", size = 175384, upload-time = "2026-05-19T07:45:56.623Z" }, + { url = "https://files.pythonhosted.org/packages/b7/77/50ac87b6e18b1c686f700aa38c9471a990683c6a955f71ac1a6677ed8145/uuid_utils-0.16.0-cp314-cp314-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:6853b627983aa1b4fd95aa52d9e87136eb94a7b3b7de0fbb1db8a498d457eeec", size = 564108, upload-time = "2026-05-19T07:43:55.609Z" }, + { url = "https://files.pythonhosted.org/packages/83/16/65046676de246bb5334d9f58aa96d2feb9fc347fda3556aaff7da1c2fc7a/uuid_utils-0.16.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:f44b65ae0c329843817d9c90e36a7a3c677b413bf407c99e67db874dac49dad3", size = 289967, upload-time = "2026-05-19T07:45:38.886Z" }, + { url = "https://files.pythonhosted.org/packages/91/d6/54fa988606a15dfd2028e925d8eb9c3ee6edbf1eb7692a67b37282880b56/uuid_utils-0.16.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de8a365795a76f347f5622621c2bee543cffa0c70949f3ee093bdefc9d926dcc", size = 325835, upload-time = "2026-05-19T07:44:42.02Z" }, + { url = "https://files.pythonhosted.org/packages/d5/1b/50622f967ceacea1f89fd065d9bfd395b51acb02cfb0a4ddc8fa9ff0c983/uuid_utils-0.16.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:426a8c9af90242d879706ccf29da56f0b0712e7739fb0bbe16baacabc75596e2", size = 332607, upload-time = "2026-05-19T07:43:42.42Z" }, + { url = "https://files.pythonhosted.org/packages/12/f5/4059706be6617e2787e375ea52994ce3c3fa3920b7d4a9c8ebf7895681a5/uuid_utils-0.16.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:833bc4b3c3fc24be541f67b01b4a75b6b9942a9b7137395b4eb35435948bd6da", size = 444287, upload-time = "2026-05-19T07:43:37.106Z" }, + { url = "https://files.pythonhosted.org/packages/65/d5/f44b2710563da687a368f0ce4dcbd462dfb6708bcd46439d831991d595c7/uuid_utils-0.16.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efb5252d7c00d586077f10e169d6e6d0b0d0f806d8a085073f0d19b4737aef4e", size = 324949, upload-time = "2026-05-19T07:45:33.175Z" }, + { url = "https://files.pythonhosted.org/packages/3a/a7/a69e859e37d26c5603f0bc0ae481860f691224f140e5a832f325b804770d/uuid_utils-0.16.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0b3377ce388fd7bf8d231ec9d1d4f58c8e87888ddea93581f60ed6f878a4f722", size = 349651, upload-time = "2026-05-19T07:43:59.998Z" }, + { url = "https://files.pythonhosted.org/packages/db/73/4139cd3ca7b81ea283c1c8769373e9b2008241c0744a8ffb25f0a1b31325/uuid_utils-0.16.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:12b6310beb38adc173ec5dc89e98812fd7e3d98f87f3ef01d2ea6ecb5d87994f", size = 502326, upload-time = "2026-05-19T07:45:40.292Z" }, + { url = "https://files.pythonhosted.org/packages/cb/8c/858101583fbad1b3fa04da88b1f7170836aa0f00b4cb712063325c44466d/uuid_utils-0.16.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a49b5a75497643479c919e2e537a4a36224ac3aaa0fada61b75d87024021ac3e", size = 607689, upload-time = "2026-05-19T07:44:48.355Z" }, + { url = "https://files.pythonhosted.org/packages/5e/bd/8f3d54a4763dd91ebd0f3d7b0c2ec434e4e0b1fc667b03a44d611a465ec6/uuid_utils-0.16.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:63bfdf00be51b6b3b79275d6767d034ea5c7a0caa067a35d72861284100cb60a", size = 566214, upload-time = "2026-05-19T07:44:53.519Z" }, + { url = "https://files.pythonhosted.org/packages/54/76/4c9a8d9baaa243c7902d84dbba4d51b1ab51c379c66d3fd6368ff6933ecf/uuid_utils-0.16.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7525bc59ac4579c32317d2493dd42cf134b9bb50cd0bc6a41dd9f77e4740dde6", size = 529989, upload-time = "2026-05-19T07:44:43.141Z" }, + { url = "https://files.pythonhosted.org/packages/6d/13/d32cea997f880cedde415730ce0e872ebfd7a040155ae0bbda70eccd208e/uuid_utils-0.16.0-cp314-cp314-win32.whl", hash = "sha256:fbcac6e6710aa2e4bfbb81762758e01470dc56d5048ba4253acc77c9833568ff", size = 169146, upload-time = "2026-05-19T07:45:46.655Z" }, + { url = "https://files.pythonhosted.org/packages/1c/19/9fc55172d8fe59e1f27a14d598b427fa508a7ebb35fa7b7b99c24fa0ef13/uuid_utils-0.16.0-cp314-cp314-win_amd64.whl", hash = "sha256:d23fcaf37368a1647319187ef6f8b741bf079f033065899bc2d00a44b0a1214a", size = 175364, upload-time = "2026-05-19T07:45:55.335Z" }, + { url = "https://files.pythonhosted.org/packages/89/5d/fcd9226b715c5aa0638fcdd6deaf0de6c6c3c451c692cd76bfca810c6512/uuid_utils-0.16.0-cp314-cp314-win_arm64.whl", hash = "sha256:ea3265f8e2b452a4870f3298cb1d183dc4e36a3682cbb264dbe46af31267e706", size = 173268, upload-time = "2026-05-19T07:44:31.19Z" }, + { url = "https://files.pythonhosted.org/packages/c1/64/97ec9af95e58b8187f2934008ffab26e1604d149e34fe01c388b0543a24f/uuid_utils-0.16.0-cp314-cp314t-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:99f8420c3ed59f89a086782ac197e257f4b1debb4545dffa90cf5db23f96c892", size = 564464, upload-time = "2026-05-19T07:44:40.856Z" }, + { url = "https://files.pythonhosted.org/packages/3e/6d/e4082f407484ac28923c0bf8e861e71d277118d8b7542d0a350340e45350/uuid_utils-0.16.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:259bab73c241743d684dcc3507feb76f484d720545e4e4805582aeff8e19700b", size = 290087, upload-time = "2026-05-19T07:44:01.084Z" }, + { url = "https://files.pythonhosted.org/packages/8c/43/c5c5f273c0ff889f20f10344784f9197dd00eb81ccc294330d4b949fea7e/uuid_utils-0.16.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:897e8ef0dc5e4ac0b17cf9cae84bb41e560d806280ec5b93db7475b504022105", size = 325532, upload-time = "2026-05-19T07:43:47.508Z" }, + { url = "https://files.pythonhosted.org/packages/13/7f/669aa899ab5378374d28a28231e6978f739921a1af394c7ebd6cc86e2639/uuid_utils-0.16.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c5af79cde16a7600dfccb7d431aec0afd3088ff170b6a09887bf3f7ab3cc7c81", size = 332209, upload-time = "2026-05-19T07:43:51.528Z" }, + { url = "https://files.pythonhosted.org/packages/2b/57/a2a32406d79a222794ef98a19254fd9a81a029a0f32d7740fba9873bff1f/uuid_utils-0.16.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bece1a6f677ca36047442c465d8166643eed9818b9e43e0bf42d3cf73e92dcff", size = 445507, upload-time = "2026-05-19T07:44:20.541Z" }, + { url = "https://files.pythonhosted.org/packages/26/6b/85459a35bfa7d73e79acbc4eab1cf6aa6e4d9d022c3260ed9dea539c7f0b/uuid_utils-0.16.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb3444498e7b099499c8a607d7771377020fa55f7274e46f54106af19f752d7", size = 326154, upload-time = "2026-05-19T07:45:23.587Z" }, + { url = "https://files.pythonhosted.org/packages/84/9e/e965efdbb503ed14d6e57aec1a22b98326ed24cc2fb48e750c4d192267a0/uuid_utils-0.16.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:542098f6cb6874aebeff98715f3ab7646fbe0f2ffb24509ca372828c68c4ed0e", size = 350905, upload-time = "2026-05-19T07:44:36.957Z" }, + { url = "https://files.pythonhosted.org/packages/23/ae/4321867888a783d03b7c053c0b68ca45d03974d86fcebf44d4ec268db397/uuid_utils-0.16.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7207b25fe534bcf4d57e0110f90670e61c1c38b6f4598ba855af69ab428fc118", size = 502098, upload-time = "2026-05-19T07:44:17.696Z" }, + { url = "https://files.pythonhosted.org/packages/9d/9a/914a47bf42479bff0ce3e1fa1cbe3585354708edc928e27687cf91de9c26/uuid_utils-0.16.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:16dc5c6e439f75b0456114e955983e2156c1f38887733e54d54205d3005223e4", size = 607032, upload-time = "2026-05-19T07:44:22.151Z" }, + { url = "https://files.pythonhosted.org/packages/85/4c/2abacd6badba61a047eaa39c8347656229d12843bd9bbe4906daa6dc752c/uuid_utils-0.16.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:a6d3ee32c57898d8415242b08d5dd086bc4f7bcbbb3fc102ef257f3d793eb294", size = 567664, upload-time = "2026-05-19T07:45:21.043Z" }, + { url = "https://files.pythonhosted.org/packages/53/1f/9d1a09521276424da19dc0d74456aed3311170fec181b28fa6acba45d963/uuid_utils-0.16.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7555f120a2282d1901c9a632c2398a614101af4fe3f7c8114aa0f1d8c1978855", size = 530996, upload-time = "2026-05-19T07:45:44.229Z" }, + { url = "https://files.pythonhosted.org/packages/b4/22/14dbedb6b61f492d5524077fd10bbfb137583b0f0aafa6cd870ccb43f39a/uuid_utils-0.16.0-cp314-cp314t-win32.whl", hash = "sha256:756575d082ea4cb7d2f923d5b640c0efe7c82573aab49220c4e09b62d13737ff", size = 169358, upload-time = "2026-05-19T07:45:05.146Z" }, + { url = "https://files.pythonhosted.org/packages/25/f4/a636806c98401a1108f2456e9cc3fa39a618145bfb1d0860c57203159cfe/uuid_utils-0.16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:aa50261a83991dbb570a00573741455bd8f3249444f7329e5bdcd494799d1504", size = 174813, upload-time = "2026-05-19T07:45:59.579Z" }, + { url = "https://files.pythonhosted.org/packages/75/12/3823742459d87a100deb24bb6b41692aa961b267abd130fa7739cdf7d409/uuid_utils-0.16.0-cp314-cp314t-win_arm64.whl", hash = "sha256:22a17e93a371d850ffce8fcdbacc2239f890efe73aa3262b6170c1febc08afe1", size = 171733, upload-time = "2026-05-19T07:45:29.283Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.48.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7b/d1/38a573f0c631c062cf42fa1f5d021d4dd3c31fb23e4376e4b56b0c9fbbed/uuid_utils-0.14.1.tar.gz", hash = "sha256:9bfc95f64af80ccf129c604fb6b8ca66c6f256451e32bc4570f760e4309c9b69", size = 22195, upload-time = "2026-02-20T22:50:38.833Z" } +dependencies = [ + { name = "click", marker = "sys_platform != 'emscripten'" }, + { name = "h11", marker = "sys_platform != 'emscripten'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e6/bf/f6544ba992ddb9a6077343a576f9844f7f8f06ab819aefd00206e9255f18/uvicorn-0.48.0.tar.gz", hash = "sha256:a5504207195d08c2511bf9125ede5ac4a4b71725d519e758d01dcf0bc2d31c37", size = 91074, upload-time = "2026-05-24T12:08:41.925Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/43/b7/add4363039a34506a58457d96d4aa2126061df3a143eb4d042aedd6a2e76/uuid_utils-0.14.1-cp39-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:93a3b5dc798a54a1feb693f2d1cb4cf08258c32ff05ae4929b5f0a2ca624a4f0", size = 604679, upload-time = "2026-02-20T22:50:27.469Z" }, - { url = "https://files.pythonhosted.org/packages/dd/84/d1d0bef50d9e66d31b2019997c741b42274d53dde2e001b7a83e9511c339/uuid_utils-0.14.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:ccd65a4b8e83af23eae5e56d88034b2fe7264f465d3e830845f10d1591b81741", size = 309346, upload-time = "2026-02-20T22:50:31.857Z" }, - { url = "https://files.pythonhosted.org/packages/ef/ed/b6d6fd52a6636d7c3eddf97d68da50910bf17cd5ac221992506fb56cf12e/uuid_utils-0.14.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b56b0cacd81583834820588378e432b0696186683b813058b707aedc1e16c4b1", size = 344714, upload-time = "2026-02-20T22:50:42.642Z" }, - { url = "https://files.pythonhosted.org/packages/a8/a7/a19a1719fb626fe0b31882db36056d44fe904dc0cf15b06fdf56b2679cf7/uuid_utils-0.14.1-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb3cf14de789097320a3c56bfdfdd51b1225d11d67298afbedee7e84e3837c96", size = 350914, upload-time = "2026-02-20T22:50:36.487Z" }, - { url = "https://files.pythonhosted.org/packages/1d/fc/f6690e667fdc3bb1a73f57951f97497771c56fe23e3d302d7404be394d4f/uuid_utils-0.14.1-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60e0854a90d67f4b0cc6e54773deb8be618f4c9bad98d3326f081423b5d14fae", size = 482609, upload-time = "2026-02-20T22:50:37.511Z" }, - { url = "https://files.pythonhosted.org/packages/54/6e/dcd3fa031320921a12ec7b4672dea3bd1dd90ddffa363a91831ba834d559/uuid_utils-0.14.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce6743ba194de3910b5feb1a62590cd2587e33a73ab6af8a01b642ceb5055862", size = 345699, upload-time = "2026-02-20T22:50:46.87Z" }, - { url = "https://files.pythonhosted.org/packages/04/28/e5220204b58b44ac0047226a9d016a113fde039280cc8732d9e6da43b39f/uuid_utils-0.14.1-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:043fb58fde6cf1620a6c066382f04f87a8e74feb0f95a585e4ed46f5d44af57b", size = 372205, upload-time = "2026-02-20T22:50:28.438Z" }, - { url = "https://files.pythonhosted.org/packages/c7/d9/3d2eb98af94b8dfffc82b6a33b4dfc87b0a5de2c68a28f6dde0db1f8681b/uuid_utils-0.14.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c915d53f22945e55fe0d3d3b0b87fd965a57f5fd15666fd92d6593a73b1dd297", size = 521836, upload-time = "2026-02-20T22:50:23.057Z" }, - { url = "https://files.pythonhosted.org/packages/a8/15/0eb106cc6fe182f7577bc0ab6e2f0a40be247f35c5e297dbf7bbc460bd02/uuid_utils-0.14.1-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:0972488e3f9b449e83f006ead5a0e0a33ad4a13e4462e865b7c286ab7d7566a3", size = 625260, upload-time = "2026-02-20T22:50:25.949Z" }, - { url = "https://files.pythonhosted.org/packages/3c/17/f539507091334b109e7496830af2f093d9fc8082411eafd3ece58af1f8ba/uuid_utils-0.14.1-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:1c238812ae0c8ffe77d8d447a32c6dfd058ea4631246b08b5a71df586ff08531", size = 587824, upload-time = "2026-02-20T22:50:35.225Z" }, - { url = "https://files.pythonhosted.org/packages/2e/c2/d37a7b2e41f153519367d4db01f0526e0d4b06f1a4a87f1c5dfca5d70a8b/uuid_utils-0.14.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:bec8f8ef627af86abf8298e7ec50926627e29b34fa907fcfbedb45aaa72bca43", size = 551407, upload-time = "2026-02-20T22:50:44.915Z" }, - { url = "https://files.pythonhosted.org/packages/65/36/2d24b2cbe78547c6532da33fb8613debd3126eccc33a6374ab788f5e46e9/uuid_utils-0.14.1-cp39-abi3-win32.whl", hash = "sha256:b54d6aa6252d96bac1fdbc80d26ba71bad9f220b2724d692ad2f2310c22ef523", size = 183476, upload-time = "2026-02-20T22:50:32.745Z" }, - { url = "https://files.pythonhosted.org/packages/83/92/2d7e90df8b1a69ec4cff33243ce02b7a62f926ef9e2f0eca5a026889cd73/uuid_utils-0.14.1-cp39-abi3-win_amd64.whl", hash = "sha256:fc27638c2ce267a0ce3e06828aff786f91367f093c80625ee21dad0208e0f5ba", size = 187147, upload-time = "2026-02-20T22:50:45.807Z" }, - { url = "https://files.pythonhosted.org/packages/d9/26/529f4beee17e5248e37e0bc17a2761d34c0fa3b1e5729c88adb2065bae6e/uuid_utils-0.14.1-cp39-abi3-win_arm64.whl", hash = "sha256:b04cb49b42afbc4ff8dbc60cf054930afc479d6f4dd7f1ec3bbe5dbfdde06b7a", size = 188132, upload-time = "2026-02-20T22:50:41.718Z" }, + { url = "https://files.pythonhosted.org/packages/01/be/72532be3da7acc5fdfbccdb95215cd04f995a0886532a5b423f929cda4cc/uvicorn-0.48.0-py3-none-any.whl", hash = "sha256:48097851328b87ec36117d3d575234519eb58c2b22d79666e9bbc6c49a761dad", size = 71410, upload-time = "2026-05-24T12:08:40.258Z" }, ] [[package]] @@ -3688,156 +4175,161 @@ lxml = [ [[package]] name = "xxhash" -version = "3.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/02/84/30869e01909fb37a6cc7e18688ee8bf1e42d57e7e0777636bd47524c43c7/xxhash-3.6.0.tar.gz", hash = "sha256:f0162a78b13a0d7617b2845b90c763339d1f1d82bb04a4b07f4ab535cc5e05d6", size = 85160, upload-time = "2025-10-02T14:37:08.097Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/33/76/35d05267ac82f53ae9b0e554da7c5e281ee61f3cad44c743f0fcd354f211/xxhash-3.6.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:599e64ba7f67472481ceb6ee80fa3bd828fd61ba59fb11475572cc5ee52b89ec", size = 32738, upload-time = "2025-10-02T14:34:55.839Z" }, - { url = "https://files.pythonhosted.org/packages/31/a8/3fbce1cd96534a95e35d5120637bf29b0d7f5d8fa2f6374e31b4156dd419/xxhash-3.6.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7d8b8aaa30fca4f16f0c84a5c8d7ddee0e25250ec2796c973775373257dde8f1", size = 30821, upload-time = "2025-10-02T14:34:57.219Z" }, - { url = "https://files.pythonhosted.org/packages/0c/ea/d387530ca7ecfa183cb358027f1833297c6ac6098223fd14f9782cd0015c/xxhash-3.6.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d597acf8506d6e7101a4a44a5e428977a51c0fadbbfd3c39650cca9253f6e5a6", size = 194127, upload-time = "2025-10-02T14:34:59.21Z" }, - { url = "https://files.pythonhosted.org/packages/ba/0c/71435dcb99874b09a43b8d7c54071e600a7481e42b3e3ce1eb5226a5711a/xxhash-3.6.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:858dc935963a33bc33490128edc1c12b0c14d9c7ebaa4e387a7869ecc4f3e263", size = 212975, upload-time = "2025-10-02T14:35:00.816Z" }, - { url = "https://files.pythonhosted.org/packages/84/7a/c2b3d071e4bb4a90b7057228a99b10d51744878f4a8a6dd643c8bd897620/xxhash-3.6.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ba284920194615cb8edf73bf52236ce2e1664ccd4a38fdb543506413529cc546", size = 212241, upload-time = "2025-10-02T14:35:02.207Z" }, - { url = "https://files.pythonhosted.org/packages/81/5f/640b6eac0128e215f177df99eadcd0f1b7c42c274ab6a394a05059694c5a/xxhash-3.6.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b54219177f6c6674d5378bd862c6aedf64725f70dd29c472eaae154df1a2e89", size = 445471, upload-time = "2025-10-02T14:35:03.61Z" }, - { url = "https://files.pythonhosted.org/packages/5e/1e/3c3d3ef071b051cc3abbe3721ffb8365033a172613c04af2da89d5548a87/xxhash-3.6.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42c36dd7dbad2f5238950c377fcbf6811b1cdb1c444fab447960030cea60504d", size = 193936, upload-time = "2025-10-02T14:35:05.013Z" }, - { url = "https://files.pythonhosted.org/packages/2c/bd/4a5f68381939219abfe1c22a9e3a5854a4f6f6f3c4983a87d255f21f2e5d/xxhash-3.6.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f22927652cba98c44639ffdc7aaf35828dccf679b10b31c4ad72a5b530a18eb7", size = 210440, upload-time = "2025-10-02T14:35:06.239Z" }, - { url = "https://files.pythonhosted.org/packages/eb/37/b80fe3d5cfb9faff01a02121a0f4d565eb7237e9e5fc66e73017e74dcd36/xxhash-3.6.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b45fad44d9c5c119e9c6fbf2e1c656a46dc68e280275007bbfd3d572b21426db", size = 197990, upload-time = "2025-10-02T14:35:07.735Z" }, - { url = "https://files.pythonhosted.org/packages/d7/fd/2c0a00c97b9e18f72e1f240ad4e8f8a90fd9d408289ba9c7c495ed7dc05c/xxhash-3.6.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:6f2580ffab1a8b68ef2b901cde7e55fa8da5e4be0977c68f78fc80f3c143de42", size = 210689, upload-time = "2025-10-02T14:35:09.438Z" }, - { url = "https://files.pythonhosted.org/packages/93/86/5dd8076a926b9a95db3206aba20d89a7fc14dd5aac16e5c4de4b56033140/xxhash-3.6.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40c391dd3cd041ebc3ffe6f2c862f402e306eb571422e0aa918d8070ba31da11", size = 414068, upload-time = "2025-10-02T14:35:11.162Z" }, - { url = "https://files.pythonhosted.org/packages/af/3c/0bb129170ee8f3650f08e993baee550a09593462a5cddd8e44d0011102b1/xxhash-3.6.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f205badabde7aafd1a31e8ca2a3e5a763107a71c397c4481d6a804eb5063d8bd", size = 191495, upload-time = "2025-10-02T14:35:12.971Z" }, - { url = "https://files.pythonhosted.org/packages/e9/3a/6797e0114c21d1725e2577508e24006fd7ff1d8c0c502d3b52e45c1771d8/xxhash-3.6.0-cp313-cp313-win32.whl", hash = "sha256:2577b276e060b73b73a53042ea5bd5203d3e6347ce0d09f98500f418a9fcf799", size = 30620, upload-time = "2025-10-02T14:35:14.129Z" }, - { url = "https://files.pythonhosted.org/packages/86/15/9bc32671e9a38b413a76d24722a2bf8784a132c043063a8f5152d390b0f9/xxhash-3.6.0-cp313-cp313-win_amd64.whl", hash = "sha256:757320d45d2fbcce8f30c42a6b2f47862967aea7bf458b9625b4bbe7ee390392", size = 31542, upload-time = "2025-10-02T14:35:15.21Z" }, - { url = "https://files.pythonhosted.org/packages/39/c5/cc01e4f6188656e56112d6a8e0dfe298a16934b8c47a247236549a3f7695/xxhash-3.6.0-cp313-cp313-win_arm64.whl", hash = "sha256:457b8f85dec5825eed7b69c11ae86834a018b8e3df5e77783c999663da2f96d6", size = 27880, upload-time = "2025-10-02T14:35:16.315Z" }, - { url = "https://files.pythonhosted.org/packages/f3/30/25e5321c8732759e930c555176d37e24ab84365482d257c3b16362235212/xxhash-3.6.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a42e633d75cdad6d625434e3468126c73f13f7584545a9cf34e883aa1710e702", size = 32956, upload-time = "2025-10-02T14:35:17.413Z" }, - { url = "https://files.pythonhosted.org/packages/9f/3c/0573299560d7d9f8ab1838f1efc021a280b5ae5ae2e849034ef3dee18810/xxhash-3.6.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:568a6d743219e717b07b4e03b0a828ce593833e498c3b64752e0f5df6bfe84db", size = 31072, upload-time = "2025-10-02T14:35:18.844Z" }, - { url = "https://files.pythonhosted.org/packages/7a/1c/52d83a06e417cd9d4137722693424885cc9878249beb3a7c829e74bf7ce9/xxhash-3.6.0-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bec91b562d8012dae276af8025a55811b875baace6af510412a5e58e3121bc54", size = 196409, upload-time = "2025-10-02T14:35:20.31Z" }, - { url = "https://files.pythonhosted.org/packages/e3/8e/c6d158d12a79bbd0b878f8355432075fc82759e356ab5a111463422a239b/xxhash-3.6.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78e7f2f4c521c30ad5e786fdd6bae89d47a32672a80195467b5de0480aa97b1f", size = 215736, upload-time = "2025-10-02T14:35:21.616Z" }, - { url = "https://files.pythonhosted.org/packages/bc/68/c4c80614716345d55071a396cf03d06e34b5f4917a467faf43083c995155/xxhash-3.6.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3ed0df1b11a79856df5ffcab572cbd6b9627034c1c748c5566fa79df9048a7c5", size = 214833, upload-time = "2025-10-02T14:35:23.32Z" }, - { url = "https://files.pythonhosted.org/packages/7e/e9/ae27c8ffec8b953efa84c7c4a6c6802c263d587b9fc0d6e7cea64e08c3af/xxhash-3.6.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0e4edbfc7d420925b0dd5e792478ed393d6e75ff8fc219a6546fb446b6a417b1", size = 448348, upload-time = "2025-10-02T14:35:25.111Z" }, - { url = "https://files.pythonhosted.org/packages/d7/6b/33e21afb1b5b3f46b74b6bd1913639066af218d704cc0941404ca717fc57/xxhash-3.6.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fba27a198363a7ef87f8c0f6b171ec36b674fe9053742c58dd7e3201c1ab30ee", size = 196070, upload-time = "2025-10-02T14:35:26.586Z" }, - { url = "https://files.pythonhosted.org/packages/96/b6/fcabd337bc5fa624e7203aa0fa7d0c49eed22f72e93229431752bddc83d9/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:794fe9145fe60191c6532fa95063765529770edcdd67b3d537793e8004cabbfd", size = 212907, upload-time = "2025-10-02T14:35:28.087Z" }, - { url = "https://files.pythonhosted.org/packages/4b/d3/9ee6160e644d660fcf176c5825e61411c7f62648728f69c79ba237250143/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:6105ef7e62b5ac73a837778efc331a591d8442f8ef5c7e102376506cb4ae2729", size = 200839, upload-time = "2025-10-02T14:35:29.857Z" }, - { url = "https://files.pythonhosted.org/packages/0d/98/e8de5baa5109394baf5118f5e72ab21a86387c4f89b0e77ef3e2f6b0327b/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:f01375c0e55395b814a679b3eea205db7919ac2af213f4a6682e01220e5fe292", size = 213304, upload-time = "2025-10-02T14:35:31.222Z" }, - { url = "https://files.pythonhosted.org/packages/7b/1d/71056535dec5c3177eeb53e38e3d367dd1d16e024e63b1cee208d572a033/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:d706dca2d24d834a4661619dcacf51a75c16d65985718d6a7d73c1eeeb903ddf", size = 416930, upload-time = "2025-10-02T14:35:32.517Z" }, - { url = "https://files.pythonhosted.org/packages/dc/6c/5cbde9de2cd967c322e651c65c543700b19e7ae3e0aae8ece3469bf9683d/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5f059d9faeacd49c0215d66f4056e1326c80503f51a1532ca336a385edadd033", size = 193787, upload-time = "2025-10-02T14:35:33.827Z" }, - { url = "https://files.pythonhosted.org/packages/19/fa/0172e350361d61febcea941b0cc541d6e6c8d65d153e85f850a7b256ff8a/xxhash-3.6.0-cp313-cp313t-win32.whl", hash = "sha256:1244460adc3a9be84731d72b8e80625788e5815b68da3da8b83f78115a40a7ec", size = 30916, upload-time = "2025-10-02T14:35:35.107Z" }, - { url = "https://files.pythonhosted.org/packages/ad/e6/e8cf858a2b19d6d45820f072eff1bea413910592ff17157cabc5f1227a16/xxhash-3.6.0-cp313-cp313t-win_amd64.whl", hash = "sha256:b1e420ef35c503869c4064f4a2f2b08ad6431ab7b229a05cce39d74268bca6b8", size = 31799, upload-time = "2025-10-02T14:35:36.165Z" }, - { url = "https://files.pythonhosted.org/packages/56/15/064b197e855bfb7b343210e82490ae672f8bc7cdf3ddb02e92f64304ee8a/xxhash-3.6.0-cp313-cp313t-win_arm64.whl", hash = "sha256:ec44b73a4220623235f67a996c862049f375df3b1052d9899f40a6382c32d746", size = 28044, upload-time = "2025-10-02T14:35:37.195Z" }, - { url = "https://files.pythonhosted.org/packages/7e/5e/0138bc4484ea9b897864d59fce9be9086030825bc778b76cb5a33a906d37/xxhash-3.6.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:a40a3d35b204b7cc7643cbcf8c9976d818cb47befcfac8bbefec8038ac363f3e", size = 32754, upload-time = "2025-10-02T14:35:38.245Z" }, - { url = "https://files.pythonhosted.org/packages/18/d7/5dac2eb2ec75fd771957a13e5dda560efb2176d5203f39502a5fc571f899/xxhash-3.6.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a54844be970d3fc22630b32d515e79a90d0a3ddb2644d8d7402e3c4c8da61405", size = 30846, upload-time = "2025-10-02T14:35:39.6Z" }, - { url = "https://files.pythonhosted.org/packages/fe/71/8bc5be2bb00deb5682e92e8da955ebe5fa982da13a69da5a40a4c8db12fb/xxhash-3.6.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:016e9190af8f0a4e3741343777710e3d5717427f175adfdc3e72508f59e2a7f3", size = 194343, upload-time = "2025-10-02T14:35:40.69Z" }, - { url = "https://files.pythonhosted.org/packages/e7/3b/52badfb2aecec2c377ddf1ae75f55db3ba2d321c5e164f14461c90837ef3/xxhash-3.6.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f6f72232f849eb9d0141e2ebe2677ece15adfd0fa599bc058aad83c714bb2c6", size = 213074, upload-time = "2025-10-02T14:35:42.29Z" }, - { url = "https://files.pythonhosted.org/packages/a2/2b/ae46b4e9b92e537fa30d03dbc19cdae57ed407e9c26d163895e968e3de85/xxhash-3.6.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63275a8aba7865e44b1813d2177e0f5ea7eadad3dd063a21f7cf9afdc7054063", size = 212388, upload-time = "2025-10-02T14:35:43.929Z" }, - { url = "https://files.pythonhosted.org/packages/f5/80/49f88d3afc724b4ac7fbd664c8452d6db51b49915be48c6982659e0e7942/xxhash-3.6.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cd01fa2aa00d8b017c97eb46b9a794fbdca53fc14f845f5a328c71254b0abb7", size = 445614, upload-time = "2025-10-02T14:35:45.216Z" }, - { url = "https://files.pythonhosted.org/packages/ed/ba/603ce3961e339413543d8cd44f21f2c80e2a7c5cfe692a7b1f2cccf58f3c/xxhash-3.6.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0226aa89035b62b6a86d3c68df4d7c1f47a342b8683da2b60cedcddb46c4d95b", size = 194024, upload-time = "2025-10-02T14:35:46.959Z" }, - { url = "https://files.pythonhosted.org/packages/78/d1/8e225ff7113bf81545cfdcd79eef124a7b7064a0bba53605ff39590b95c2/xxhash-3.6.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c6e193e9f56e4ca4923c61238cdaced324f0feac782544eb4c6d55ad5cc99ddd", size = 210541, upload-time = "2025-10-02T14:35:48.301Z" }, - { url = "https://files.pythonhosted.org/packages/6f/58/0f89d149f0bad89def1a8dd38feb50ccdeb643d9797ec84707091d4cb494/xxhash-3.6.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:9176dcaddf4ca963d4deb93866d739a343c01c969231dbe21680e13a5d1a5bf0", size = 198305, upload-time = "2025-10-02T14:35:49.584Z" }, - { url = "https://files.pythonhosted.org/packages/11/38/5eab81580703c4df93feb5f32ff8fa7fe1e2c51c1f183ee4e48d4bb9d3d7/xxhash-3.6.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c1ce4009c97a752e682b897aa99aef84191077a9433eb237774689f14f8ec152", size = 210848, upload-time = "2025-10-02T14:35:50.877Z" }, - { url = "https://files.pythonhosted.org/packages/5e/6b/953dc4b05c3ce678abca756416e4c130d2382f877a9c30a20d08ee6a77c0/xxhash-3.6.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:8cb2f4f679b01513b7adbb9b1b2f0f9cdc31b70007eaf9d59d0878809f385b11", size = 414142, upload-time = "2025-10-02T14:35:52.15Z" }, - { url = "https://files.pythonhosted.org/packages/08/a9/238ec0d4e81a10eb5026d4a6972677cbc898ba6c8b9dbaec12ae001b1b35/xxhash-3.6.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:653a91d7c2ab54a92c19ccf43508b6a555440b9be1bc8be553376778be7f20b5", size = 191547, upload-time = "2025-10-02T14:35:53.547Z" }, - { url = "https://files.pythonhosted.org/packages/f1/ee/3cf8589e06c2164ac77c3bf0aa127012801128f1feebf2a079272da5737c/xxhash-3.6.0-cp314-cp314-win32.whl", hash = "sha256:a756fe893389483ee8c394d06b5ab765d96e68fbbfe6fde7aa17e11f5720559f", size = 31214, upload-time = "2025-10-02T14:35:54.746Z" }, - { url = "https://files.pythonhosted.org/packages/02/5d/a19552fbc6ad4cb54ff953c3908bbc095f4a921bc569433d791f755186f1/xxhash-3.6.0-cp314-cp314-win_amd64.whl", hash = "sha256:39be8e4e142550ef69629c9cd71b88c90e9a5db703fecbcf265546d9536ca4ad", size = 32290, upload-time = "2025-10-02T14:35:55.791Z" }, - { url = "https://files.pythonhosted.org/packages/b1/11/dafa0643bc30442c887b55baf8e73353a344ee89c1901b5a5c54a6c17d39/xxhash-3.6.0-cp314-cp314-win_arm64.whl", hash = "sha256:25915e6000338999236f1eb68a02a32c3275ac338628a7eaa5a269c401995679", size = 28795, upload-time = "2025-10-02T14:35:57.162Z" }, - { url = "https://files.pythonhosted.org/packages/2c/db/0e99732ed7f64182aef4a6fb145e1a295558deec2a746265dcdec12d191e/xxhash-3.6.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:c5294f596a9017ca5a3e3f8884c00b91ab2ad2933cf288f4923c3fd4346cf3d4", size = 32955, upload-time = "2025-10-02T14:35:58.267Z" }, - { url = "https://files.pythonhosted.org/packages/55/f4/2a7c3c68e564a099becfa44bb3d398810cc0ff6749b0d3cb8ccb93f23c14/xxhash-3.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1cf9dcc4ab9cff01dfbba78544297a3a01dafd60f3bde4e2bfd016cf7e4ddc67", size = 31072, upload-time = "2025-10-02T14:35:59.382Z" }, - { url = "https://files.pythonhosted.org/packages/c6/d9/72a29cddc7250e8a5819dad5d466facb5dc4c802ce120645630149127e73/xxhash-3.6.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:01262da8798422d0685f7cef03b2bd3f4f46511b02830861df548d7def4402ad", size = 196579, upload-time = "2025-10-02T14:36:00.838Z" }, - { url = "https://files.pythonhosted.org/packages/63/93/b21590e1e381040e2ca305a884d89e1c345b347404f7780f07f2cdd47ef4/xxhash-3.6.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51a73fb7cb3a3ead9f7a8b583ffd9b8038e277cdb8cb87cf890e88b3456afa0b", size = 215854, upload-time = "2025-10-02T14:36:02.207Z" }, - { url = "https://files.pythonhosted.org/packages/ce/b8/edab8a7d4fa14e924b29be877d54155dcbd8b80be85ea00d2be3413a9ed4/xxhash-3.6.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b9c6df83594f7df8f7f708ce5ebeacfc69f72c9fbaaababf6cf4758eaada0c9b", size = 214965, upload-time = "2025-10-02T14:36:03.507Z" }, - { url = "https://files.pythonhosted.org/packages/27/67/dfa980ac7f0d509d54ea0d5a486d2bb4b80c3f1bb22b66e6a05d3efaf6c0/xxhash-3.6.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:627f0af069b0ea56f312fd5189001c24578868643203bca1abbc2c52d3a6f3ca", size = 448484, upload-time = "2025-10-02T14:36:04.828Z" }, - { url = "https://files.pythonhosted.org/packages/8c/63/8ffc2cc97e811c0ca5d00ab36604b3ea6f4254f20b7bc658ca825ce6c954/xxhash-3.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aa912c62f842dfd013c5f21a642c9c10cd9f4c4e943e0af83618b4a404d9091a", size = 196162, upload-time = "2025-10-02T14:36:06.182Z" }, - { url = "https://files.pythonhosted.org/packages/4b/77/07f0e7a3edd11a6097e990f6e5b815b6592459cb16dae990d967693e6ea9/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:b465afd7909db30168ab62afe40b2fcf79eedc0b89a6c0ab3123515dc0df8b99", size = 213007, upload-time = "2025-10-02T14:36:07.733Z" }, - { url = "https://files.pythonhosted.org/packages/ae/d8/bc5fa0d152837117eb0bef6f83f956c509332ce133c91c63ce07ee7c4873/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:a881851cf38b0a70e7c4d3ce81fc7afd86fbc2a024f4cfb2a97cf49ce04b75d3", size = 200956, upload-time = "2025-10-02T14:36:09.106Z" }, - { url = "https://files.pythonhosted.org/packages/26/a5/d749334130de9411783873e9b98ecc46688dad5db64ca6e04b02acc8b473/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:9b3222c686a919a0f3253cfc12bb118b8b103506612253b5baeaac10d8027cf6", size = 213401, upload-time = "2025-10-02T14:36:10.585Z" }, - { url = "https://files.pythonhosted.org/packages/89/72/abed959c956a4bfc72b58c0384bb7940663c678127538634d896b1195c10/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:c5aa639bc113e9286137cec8fadc20e9cd732b2cc385c0b7fa673b84fc1f2a93", size = 417083, upload-time = "2025-10-02T14:36:12.276Z" }, - { url = "https://files.pythonhosted.org/packages/0c/b3/62fd2b586283b7d7d665fb98e266decadf31f058f1cf6c478741f68af0cb/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5c1343d49ac102799905e115aee590183c3921d475356cb24b4de29a4bc56518", size = 193913, upload-time = "2025-10-02T14:36:14.025Z" }, - { url = "https://files.pythonhosted.org/packages/9a/9a/c19c42c5b3f5a4aad748a6d5b4f23df3bed7ee5445accc65a0fb3ff03953/xxhash-3.6.0-cp314-cp314t-win32.whl", hash = "sha256:5851f033c3030dd95c086b4a36a2683c2ff4a799b23af60977188b057e467119", size = 31586, upload-time = "2025-10-02T14:36:15.603Z" }, - { url = "https://files.pythonhosted.org/packages/03/d6/4cc450345be9924fd5dc8c590ceda1db5b43a0a889587b0ae81a95511360/xxhash-3.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0444e7967dac37569052d2409b00a8860c2135cff05502df4da80267d384849f", size = 32526, upload-time = "2025-10-02T14:36:16.708Z" }, - { url = "https://files.pythonhosted.org/packages/0f/c9/7243eb3f9eaabd1a88a5a5acadf06df2d83b100c62684b7425c6a11bcaa8/xxhash-3.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:bb79b1e63f6fd84ec778a4b1916dfe0a7c3fdb986c06addd5db3a0d413819d95", size = 28898, upload-time = "2025-10-02T14:36:17.843Z" }, +version = "3.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/2f/e183a1b407002f5af81822bee18b61cdb94b8670208ef34734d8d2b8ebe9/xxhash-3.7.0.tar.gz", hash = "sha256:6cc4eefbb542a5d6ffd6d70ea9c502957c925e800f998c5630ecc809d6702bae", size = 82022, upload-time = "2026-04-25T11:10:32.553Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/ca/d5174b4c36d10f64d4ca7050563138c5a599efb01a765858ddefc9c1202a/xxhash-3.7.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:4b6d6b33f141158692bd4eafbb96edbc5aa0dabdb593a962db01a91983d4f8fa", size = 36813, upload-time = "2026-04-25T11:06:51.73Z" }, + { url = "https://files.pythonhosted.org/packages/41/d0/abc6c9d347ba1f1e1e1d98125d0881a0452c7f9a76a9dd03a7b5d2197f23/xxhash-3.7.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:845d347df254d6c619f616afa921331bada8614b8d373d58725c663ba97c3605", size = 35121, upload-time = "2026-04-25T11:06:53.048Z" }, + { url = "https://files.pythonhosted.org/packages/bf/11/4cc834eb3d79f2f2b3a6ef7324195208bcdfbdcf7534d2b17267aa5f3a8f/xxhash-3.7.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:fddbbb69a6fff4f421e7a0d1fa28f894b20112e9e3fab306af451e2dfd0e459b", size = 29624, upload-time = "2026-04-25T11:06:54.311Z" }, + { url = "https://files.pythonhosted.org/packages/23/83/e97d3e7b635fe73a1dfb1e91f805324dd6d930bb42041cbf18f183bc0b6d/xxhash-3.7.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:54876a4e45101cec2bf8f31a973cda073a23e2e108538dad224ba07f85f22487", size = 30638, upload-time = "2026-04-25T11:06:55.864Z" }, + { url = "https://files.pythonhosted.org/packages/f4/40/d84951d80c35db1f4c40a29a64a8520eea5d56e764c603906b4fe763580f/xxhash-3.7.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:0c72fe9c7e3d6dfd7f1e21e224a877917fa09c465694ba4e06464b9511b65544", size = 33323, upload-time = "2026-04-25T11:06:57.336Z" }, + { url = "https://files.pythonhosted.org/packages/89/cc/c7dc6558d97e9ab023f663d69ab28b340ed9bf4d2d94f2c259cf896bb354/xxhash-3.7.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a6d73a830b17ef49bc04e00182bd839164c1b3c59c127cd7c54fcb10c7ed8ee8", size = 33362, upload-time = "2026-04-25T11:06:58.656Z" }, + { url = "https://files.pythonhosted.org/packages/2a/6e/46b84017b1301d54091430353d4ad5901654a3e0871649877a416f7f1644/xxhash-3.7.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:91c3b07cf3362086d8f126c6aecd8e5e9396ad8b2f2219ea7e49a8250c318acd", size = 30874, upload-time = "2026-04-25T11:06:59.834Z" }, + { url = "https://files.pythonhosted.org/packages/df/5e/8f9158e3ab906ad3fec51e09b5ea0093e769f12207bfa42a368ca204e7ab/xxhash-3.7.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:50e879ebbac351c81565ca108db766d7832f5b8b6a5b14b8c0151f7190028e3d", size = 194185, upload-time = "2026-04-25T11:07:01.658Z" }, + { url = "https://files.pythonhosted.org/packages/f3/29/a804ded9f5d3d3758292678d23e7528b08fda7b7e750688d08b052322475/xxhash-3.7.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:921c14e93817842dd0dd9f372890a0f0c72e534650b6ab13c5be5cd0db11d47e", size = 213033, upload-time = "2026-04-25T11:07:03.606Z" }, + { url = "https://files.pythonhosted.org/packages/8b/91/1ce5a7d2fdc975267320e2c78fc1cecfe7ab735ccbcf6993ec5dd541cb2c/xxhash-3.7.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e64a7c9d7dfca3e0fafcbc5e455519090706a3e36e95d655cec3e04e79f95aaa", size = 236140, upload-time = "2026-04-25T11:07:05.396Z" }, + { url = "https://files.pythonhosted.org/packages/34/04/fd595a4fd8617b05fa27bd9b684ecb4985bfed27917848eea85d54036d06/xxhash-3.7.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2220af08163baf5fa36c2b8af079dc2cbe6e66ae061385267f9472362dfd53c6", size = 212291, upload-time = "2026-04-25T11:07:06.966Z" }, + { url = "https://files.pythonhosted.org/packages/03/fb/f1a379cbc372ae5b9f4ab36154c48a849ca6ebe3ac477067a57865bf3bc6/xxhash-3.7.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f14bb8b22a4a91325813e3d553b8963c10cf8c756cff65ee50c194431296c655", size = 445532, upload-time = "2026-04-25T11:07:08.525Z" }, + { url = "https://files.pythonhosted.org/packages/65/59/172424b79f8cfd4b6d8a122b2193e6b8ad4b11f7159bb3b6f9b3191329bb/xxhash-3.7.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:496736f86a9bedaf64b0dc70e3539d0766df01c71ea22032698e88f3f04a1ce9", size = 193990, upload-time = "2026-04-25T11:07:10.315Z" }, + { url = "https://files.pythonhosted.org/packages/b9/19/aeac22161d953f139f07ba5586cb4a17c5b7b6dff985122803bb12933500/xxhash-3.7.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0ff71596bd79816975b3de7130ab1ff4541410285a3c084584eeb1c8239996fd", size = 284876, upload-time = "2026-04-25T11:07:12.15Z" }, + { url = "https://files.pythonhosted.org/packages/77/d5/4fd0b59e7a02242953da05ff679fbb961b0a4368eac97a217e11dae110c1/xxhash-3.7.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1ad86695c19b1d46fe106925db3c7a37f16be37669dcf58dcc70a9dd6e324676", size = 210495, upload-time = "2026-04-25T11:07:13.952Z" }, + { url = "https://files.pythonhosted.org/packages/aa/fb/976a3165c728c7faf74aa1b5ab3cf6a85e6d731612894741840524c7d28c/xxhash-3.7.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:970f9f8c50961d639cbd0d988c96f80ddf66006de93641719282c4fe7a87c5e6", size = 241331, upload-time = "2026-04-25T11:07:15.557Z" }, + { url = "https://files.pythonhosted.org/packages/4a/2c/6763d5901d53ac9e6ba296e5717ae599025c9d268396e8faa8b4b0a8e0ac/xxhash-3.7.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:5886ad85e9e347911783760a1d16cb6b393e8f9e3b52c982568226cb56927bdc", size = 198037, upload-time = "2026-04-25T11:07:17.563Z" }, + { url = "https://files.pythonhosted.org/packages/61/2b/876e722d533833f5f9a83473e6ba993e48745701096944e77bbecf29b2c3/xxhash-3.7.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:6e934bbae1e0ec74e27d5f0d7f37ef547ce5ff9f0a7e63fb39e559fc99526734", size = 210744, upload-time = "2026-04-25T11:07:19.055Z" }, + { url = "https://files.pythonhosted.org/packages/21/e6/d7e7baef7ce24166b4668d3c48557bb35a23b92ecadcac7e7718d099ab69/xxhash-3.7.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:3b6b3d28228af044ebcded71c4a3dd86e1dbd7e2f4645bf40f7b5da65bb5fb5a", size = 275406, upload-time = "2026-04-25T11:07:20.908Z" }, + { url = "https://files.pythonhosted.org/packages/92/fe/198b3763b2e01ca908f2154969a2352ec99bda892b574a11a9a151c5ede4/xxhash-3.7.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:6be4d70d9ab76c9f324ead9c01af6ff52c324745ea0c3731682a0cf99720f1fe", size = 414125, upload-time = "2026-04-25T11:07:23.037Z" }, + { url = "https://files.pythonhosted.org/packages/3a/6d/019a11affd5a5499137cacca53808659964785439855b5aa40dfd3412916/xxhash-3.7.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:151d7520838d4465461a0b7f4ae488b3b00de16183dd3214c1a6b14bf89d7fb6", size = 191555, upload-time = "2026-04-25T11:07:24.991Z" }, + { url = "https://files.pythonhosted.org/packages/76/21/b96d58568df2d01533244c3e0e5cbdd0c8b2b25c4bec4d72f19259a292d7/xxhash-3.7.0-cp313-cp313-win32.whl", hash = "sha256:d798c1e291bffb8e37b5bbe0dda77fc767cd19e89cadaf66e6ed5d0ff88c9fe6", size = 30668, upload-time = "2026-04-25T11:07:26.665Z" }, + { url = "https://files.pythonhosted.org/packages/99/57/d849a8d3afa1f8f4bc6a831cd89f49f9706fbbad94d2975d6140a171988c/xxhash-3.7.0-cp313-cp313-win_amd64.whl", hash = "sha256:875811ba23c543b1a1c3143c926e43996eb27ebb8f52d3500744aa608c275aed", size = 31524, upload-time = "2026-04-25T11:07:27.92Z" }, + { url = "https://files.pythonhosted.org/packages/81/52/bacc753e92dee78b058af8dcef0a50815f5f860986c664a92d75f965b6a5/xxhash-3.7.0-cp313-cp313-win_arm64.whl", hash = "sha256:54a675cb300dda83d71daae2a599389d22db8021a0f8db0dd659e14626eb3ecc", size = 27768, upload-time = "2026-04-25T11:07:29.113Z" }, + { url = "https://files.pythonhosted.org/packages/1c/47/ddbd683b7fc7e592c1a8d9d65f73ce9ab513f082b3967eee2baf549b8fc6/xxhash-3.7.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a3b19a42111c4057c1547a4a1396a53961dca576a0f6b82bfa88a2d1561764b2", size = 33576, upload-time = "2026-04-25T11:07:30.469Z" }, + { url = "https://files.pythonhosted.org/packages/07/f2/36d3310161db7f72efb4562aadde0ed429f1d0531782dd6345b12d2da527/xxhash-3.7.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8f4608a06e4d61b7a3425665a46d00e0579122e1a2fae97a0c52953a3aad9aa3", size = 31123, upload-time = "2026-04-25T11:07:31.989Z" }, + { url = "https://files.pythonhosted.org/packages/0d/3f/75937a5c69556ed213021e43cbedd84c8e0279d0d74e7d41a255d84ba4b1/xxhash-3.7.0-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ad37c7792479e49cf96c1ab25517d7003fe0d93687a772ba19a097d235bbe41e", size = 196491, upload-time = "2026-04-25T11:07:33.358Z" }, + { url = "https://files.pythonhosted.org/packages/22/29/f10d7ff8c7a733d4403a43b9de18c8fabc005f98cec054644f04418659ee/xxhash-3.7.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc026e3b89d98e30a8288c95cb696e77d150b3f0fb7a51f73dcd49ee6b5577fa", size = 215793, upload-time = "2026-04-25T11:07:34.919Z" }, + { url = "https://files.pythonhosted.org/packages/8b/fd/778f60aa295f58907938f030a8b514611f391405614a525cccd2ffc00eb5/xxhash-3.7.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c9b31ab1f28b078a6a1ac1a54eb35e7d5390deddd56870d0be3a0a733d1c321c", size = 237993, upload-time = "2026-04-25T11:07:36.638Z" }, + { url = "https://files.pythonhosted.org/packages/70/f5/736db5de387b4a540e37a05b84b40dc58a1ce974bfd2b4e5754ce29b68c3/xxhash-3.7.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3bb5fd680c038fd5229e44e9c493782f90df9bef632fd0499d442374688ff70b", size = 214887, upload-time = "2026-04-25T11:07:38.564Z" }, + { url = "https://files.pythonhosted.org/packages/4d/aa/09a095f22fdb9a27fbb716841fbff52119721f9ca4261952d07a912f7839/xxhash-3.7.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:030c0fd688fce3569fbb49a2feefd4110cbb0b650186fb4610759ecfac677548", size = 448407, upload-time = "2026-04-25T11:07:40.552Z" }, + { url = "https://files.pythonhosted.org/packages/74/8a/b745efeeca9e34a91c26fdc97ad8514c43d5a81ac78565cba80a1353870a/xxhash-3.7.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5b1bde10324f4c31812ae0d0502e92d916ae8917cad7209353f122b8b8f610c3", size = 196119, upload-time = "2026-04-25T11:07:42.101Z" }, + { url = "https://files.pythonhosted.org/packages/8a/5c/0cfceb024af90c191f665c7933b1f318ee234f4797858383bebd1881d52f/xxhash-3.7.0-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:503722d52a615f2604f5e7611de7d43878df010dc0053094ef91cb9a9ac3d987", size = 286751, upload-time = "2026-04-25T11:07:43.568Z" }, + { url = "https://files.pythonhosted.org/packages/0b/0a/0793e405dc3cf8f4ebe2c1acec1e4e4608cd9e7e50ea691dabbc2a95ccbb/xxhash-3.7.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c72500a3b6d6c30ebfc135035bcace9eb5884f2dc220804efcaaba43e9f611dd", size = 212961, upload-time = "2026-04-25T11:07:45.388Z" }, + { url = "https://files.pythonhosted.org/packages/0c/7e/721118ffc63bfff94aa565bcf2555a820f9f4bdb0f001e0d609bdfad70de/xxhash-3.7.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:43475925a766d01ca8cd9a857fd87f3d50406983c8506a4c07c4df12adcc867f", size = 243703, upload-time = "2026-04-25T11:07:47.053Z" }, + { url = "https://files.pythonhosted.org/packages/6e/18/16f6267160488b8276fd3d449d425712512add292ba545c1b6946bfdb7dd/xxhash-3.7.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:8d09dfd2ab135b985daf868b594315ebe11ad86cd9fea46e6c69f19b28f7d25a", size = 200894, upload-time = "2026-04-25T11:07:48.657Z" }, + { url = "https://files.pythonhosted.org/packages/2d/94/80ba841287fd97e3e9cac1d228788c8ef623746f570404961eec748ecb5c/xxhash-3.7.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c50269d0055ac1faecfd559886d2cbe4b730de236585aba0e873f9d9dadbe585", size = 213357, upload-time = "2026-04-25T11:07:50.257Z" }, + { url = "https://files.pythonhosted.org/packages/a1/7e/106d4067130c59f1e18a55ffadcd876d8c68534883a1e02685b29d3d8153/xxhash-3.7.0-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1910df4756a5ab58cfad8744fc2d0f23926e3efcc346ee76e87b974abab922f4", size = 277600, upload-time = "2026-04-25T11:07:51.745Z" }, + { url = "https://files.pythonhosted.org/packages/c5/86/a081dd30da71d720b2612a792bfd55e45fa9a07ac76a0507f60487473c25/xxhash-3.7.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:d006faf3b491957efcb433489be3c149efe4787b7063d5cddb8ddaefdc60e0c1", size = 416980, upload-time = "2026-04-25T11:07:53.504Z" }, + { url = "https://files.pythonhosted.org/packages/35/29/1a95221a029a3c1293773869e1ab47b07cbbdd82444a42809e8c60156626/xxhash-3.7.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:abb65b4e947e958f7b3b0d71db3ce447d1bc5f37f5eab871ce7223bda8768a04", size = 193840, upload-time = "2026-04-25T11:07:55.103Z" }, + { url = "https://files.pythonhosted.org/packages/c5/e0/db909dd0823285de2286f67e10ee4d81e96ad35d7d8e964ecb07fccd8af9/xxhash-3.7.0-cp313-cp313t-win32.whl", hash = "sha256:178959906cb1716a1ce08e0d69c82886c70a15a6f2790fc084fdd146ca30cd49", size = 30966, upload-time = "2026-04-25T11:07:56.524Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ff/d705b15b22f21ee106adce239cb65d35067a158c630b240270f09b17c2e6/xxhash-3.7.0-cp313-cp313t-win_amd64.whl", hash = "sha256:2524a1e20d4c231d13b50f7cf39e44265b055669a64a7a4b9a2a44faa03f19b6", size = 31784, upload-time = "2026-04-25T11:07:57.758Z" }, + { url = "https://files.pythonhosted.org/packages/a2/1f/b2cf83c3638fd0588e0b17f22e5a9400bdfb1a3e3755324ac0aee2250b88/xxhash-3.7.0-cp313-cp313t-win_arm64.whl", hash = "sha256:37d994d0ffe81ef087bb330d392caa809bb5853c77e22ea3f71db024a0543dba", size = 27932, upload-time = "2026-04-25T11:07:59.109Z" }, + { url = "https://files.pythonhosted.org/packages/0e/cc/431db584f6fbb9312e40a173af027644e5580d39df1f73603cbb9dca4d6b/xxhash-3.7.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:8c5fcfd806c335bfa2adf1cd0b3110a44fc7b6995c3a648c27489bae85801465", size = 36644, upload-time = "2026-04-25T11:08:00.658Z" }, + { url = "https://files.pythonhosted.org/packages/bc/01/255ec513e0a705d1f9a61413e78dfce4e3235203f0ed525a24c2b4b56345/xxhash-3.7.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:506a0b488f190f0a06769575e30caf71615c898ed93ab18b0dbcb6dec5c3713c", size = 35003, upload-time = "2026-04-25T11:08:02.338Z" }, + { url = "https://files.pythonhosted.org/packages/68/70/c55fc33c93445b44d8fc5a17b41ed99e3cebe92bcf8396809e63fc9a1165/xxhash-3.7.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:ec68dbba21532c0173a9872298e65c89749f7c9d21538c3a78b5bb6105871568", size = 29655, upload-time = "2026-04-25T11:08:03.701Z" }, + { url = "https://files.pythonhosted.org/packages/c2/72/ff8de73df000d74467d12a59ce6d6e2b2a368b978d41ab7b1fba5ed442be/xxhash-3.7.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:fa77e7ec1450d415d20129961814787c9abd9a07f98872f070b1fe96c5084611", size = 30664, upload-time = "2026-04-25T11:08:05.011Z" }, + { url = "https://files.pythonhosted.org/packages/b6/91/08416d9bd9bc3bf39d831abe8a5631ac2db5141dfd6fe81c3fe59a1f9264/xxhash-3.7.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:fe32736295ea38e43e7d9424053c8c47c9f64fecfc7c895fb3da9b30b131c9ee", size = 33317, upload-time = "2026-04-25T11:08:06.413Z" }, + { url = "https://files.pythonhosted.org/packages/0e/3b/86b1caa4dee10a99f4bf9521e623359341c5e50d05158fa10c275b2bd079/xxhash-3.7.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ab9dd2c83c4bbd63e422181a76f13502d049d3ddcac9a1bdc29196263d692bb8", size = 33457, upload-time = "2026-04-25T11:08:08.099Z" }, + { url = "https://files.pythonhosted.org/packages/ed/38/98ea14ad1517e1461292a65906951458d520689782bfbae111050145bdba/xxhash-3.7.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3afec3a336a2286601a437cb07562ab0227685e6fbb9ec17e8c18457ff348ecf", size = 30894, upload-time = "2026-04-25T11:08:09.429Z" }, + { url = "https://files.pythonhosted.org/packages/61/a2/074654d0b893606541199993c7db70067d9fc63b748e0d60020a52a1bd36/xxhash-3.7.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:565df64437a9390f84465dcca33e7377114c7ede8d05cd2cf20081f831ea788e", size = 194409, upload-time = "2026-04-25T11:08:10.91Z" }, + { url = "https://files.pythonhosted.org/packages/e2/26/6d2a1afc468189f77ca28c32e1c83e1b9da1178231e05641dbc1b350e332/xxhash-3.7.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:12eca820a5d558633d423bf8bb78ce72a55394823f64089247f788a7e0ae691e", size = 213135, upload-time = "2026-04-25T11:08:12.575Z" }, + { url = "https://files.pythonhosted.org/packages/8e/0e/d8aecf95e09c42547453137be74d2f7b8b14e08f5177fa2fab6144a19061/xxhash-3.7.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f262b8f7599516567e070abf607b9af649052b2c4bd6f9be02b0cb41b7024805", size = 236379, upload-time = "2026-04-25T11:08:14.206Z" }, + { url = "https://files.pythonhosted.org/packages/f2/74/8140e8210536b3dd0cc816c4faaeb5ba6e63e8125ab25af4bcddd6a037b3/xxhash-3.7.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f1598916cb197681e03e601901e4ab96a9a963de398c59d0964f8a6f44a2b361", size = 212447, upload-time = "2026-04-25T11:08:15.79Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d2/462001d2903b4bee5a5689598a0a55e5e7cd1ac7f4247a5545cff10d3ebb/xxhash-3.7.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:322b2f0622230f526aeb1738149948a7ae357a9e2ceb1383c6fd1fdaecdafa16", size = 445660, upload-time = "2026-04-25T11:08:17.441Z" }, + { url = "https://files.pythonhosted.org/packages/23/09/2bd1ed7f8689b20e51727952cac8329d50c694dc32b2eba06ba5bc742b37/xxhash-3.7.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24cc22070880cc57b830a65cde4e65fa884c6d9b28ae4803b5ee05911e7bafba", size = 194076, upload-time = "2026-04-25T11:08:19.134Z" }, + { url = "https://files.pythonhosted.org/packages/c9/6e/692302cd0a5f4ac4e6289f37fa888dc2e1e07750b68fe3e4bfe939b8cea3/xxhash-3.7.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb5a888a968b2434abf9ecda357b5d43f10d7b5a6da6fdbbe036208473aff0e2", size = 284990, upload-time = "2026-04-25T11:08:20.618Z" }, + { url = "https://files.pythonhosted.org/packages/05/d9/e54b159b3d9df7999d2a7c676ce7b323d1b5588a64f8f51ed8172567bd87/xxhash-3.7.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a999771ff97bec27d18341be4f3a36b163bb1ac41ec17bef6d2dabd84acd33c7", size = 210590, upload-time = "2026-04-25T11:08:22.24Z" }, + { url = "https://files.pythonhosted.org/packages/50/93/0e0df1a3a196ced4ca71de76d65ead25d8e87bbfb87b64306ea47a40c00d/xxhash-3.7.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ed4a6efe2dee1655adb73e7ad40c6aa955a6892422b1e3b95de6a34de56e3cbb", size = 241442, upload-time = "2026-04-25T11:08:23.844Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a9/d917a7a814e90b218f8a0d37967105eea91bf752c3303683c99a1f7bfc1f/xxhash-3.7.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:9fd17f14ac0faa12126c2f9ca774a8cf342957265ec3c8669c144e5e6cdb478c", size = 198356, upload-time = "2026-04-25T11:08:25.99Z" }, + { url = "https://files.pythonhosted.org/packages/89/5e/f2ba1877c39469abbefc72991d6ebdcbd4c0880db01ae8cb1f553b0c537d/xxhash-3.7.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:05fd1254268c59b5cb2a029dfc204275e9fc52de2913f1e53aa8d01442c96b4d", size = 210898, upload-time = "2026-04-25T11:08:27.608Z" }, + { url = "https://files.pythonhosted.org/packages/90/c6/be56b58e73de531f39a10de1355bb77ceb663900dc4bf2d6d3002a9c3f9e/xxhash-3.7.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a2eae53197c6276d5b317f75a1be226bbf440c20b58bf525f36b5d0e1f657ca6", size = 275519, upload-time = "2026-04-25T11:08:29.301Z" }, + { url = "https://files.pythonhosted.org/packages/92/e2/17ddc85d5765b9c709f192009ed8f5a1fc876f4eb35bba7c307b5b1169f9/xxhash-3.7.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:bfe6f92e3522dcbe8c4281efd74fa7542a336cb00b0e3272c4ec0edabeaeaf67", size = 414191, upload-time = "2026-04-25T11:08:31.16Z" }, + { url = "https://files.pythonhosted.org/packages/9c/42/85f5b79f4bf1ec7ba052491164adfd4f4e9519f5dc7246de4fbd64a1bd56/xxhash-3.7.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7ab9a49c410d8c6c786ab99e79c529938d894c01433130353dd0fe999111077a", size = 191604, upload-time = "2026-04-25T11:08:32.862Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d0/6127b623aa4cca18d8b7743592b048d689fd6c6e37ff26a22cddf6cd9d7f/xxhash-3.7.0-cp314-cp314-win32.whl", hash = "sha256:040ea63668f9185b92bc74942df09c7e65703deed71431333678fc6e739a9955", size = 31271, upload-time = "2026-04-25T11:08:34.651Z" }, + { url = "https://files.pythonhosted.org/packages/64/4f/44fc4788568004c43921701cbc127f48218a1eede2c9aea231115323564d/xxhash-3.7.0-cp314-cp314-win_amd64.whl", hash = "sha256:2a61e2a3fb23c892496d587b470dee7fa1b58b248a187719c65ea8e94ec13257", size = 32284, upload-time = "2026-04-25T11:08:35.987Z" }, + { url = "https://files.pythonhosted.org/packages/6d/77/18bb895eb60a49453d16e17d67990e5caff557c78eafc90ad4e2eabf4570/xxhash-3.7.0-cp314-cp314-win_arm64.whl", hash = "sha256:c7741c7524961d8c0cb4d4c21b28957ff731a3fd5b5cd8b856dc80a40e9e5acc", size = 28701, upload-time = "2026-04-25T11:08:37.767Z" }, + { url = "https://files.pythonhosted.org/packages/45/a0/46f72244570c550fbbb7db1ef554183dd5ebe9136385f30e032b781ae8f6/xxhash-3.7.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:fc84bf7aa7592f31ec63a3e7b11d624f468a3f19f5238cec7282a42e838ab1d7", size = 33646, upload-time = "2026-04-25T11:08:39.109Z" }, + { url = "https://files.pythonhosted.org/packages/4a/3a/453846a7eceea11e75def361eed01ec6a0205b9822c19927ed364ccae7cc/xxhash-3.7.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9f1563fdc8abfc389748e6932c7e4e99c89a53e4ec37d4563c24fc06f5e5644b", size = 31125, upload-time = "2026-04-25T11:08:40.467Z" }, + { url = "https://files.pythonhosted.org/packages/bd/3e/49434aba738885d512f9e486db1bdd19db28dfa40372b56da26ef7a4e738/xxhash-3.7.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2d415f18becf6f153046ab6adc97da77e3643a0ee205dae61c4012604113a020", size = 196633, upload-time = "2026-04-25T11:08:41.943Z" }, + { url = "https://files.pythonhosted.org/packages/a4/e9/006cb6127baeb9f8abe6d15e62faa01349f09b34e2bfd65175b2422d026b/xxhash-3.7.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bb16aa13ed175bc9be5c2491ba031b85a9b51c4ed90e0b3d4ebe63cf3fb54f8e", size = 215899, upload-time = "2026-04-25T11:08:43.645Z" }, + { url = "https://files.pythonhosted.org/packages/27/e4/cc57d72e66df0ae29b914335f1c6dcf61e8f3746ddf0ae3c471aa4f15e00/xxhash-3.7.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f9fd595f1e5941b3d7863e4774e4b30caa6731fc34b9277da032295aa5656ee5", size = 238116, upload-time = "2026-04-25T11:08:45.698Z" }, + { url = "https://files.pythonhosted.org/packages/af/78/3531d4a3fd8a0038cc6be1f265a69c1b3587f557a10b677dd736de2202c1/xxhash-3.7.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1295325c5a98d552333fa53dc2b026b0ef0ec9c8e73ca3a952990b4c7d65d459", size = 215012, upload-time = "2026-04-25T11:08:47.355Z" }, + { url = "https://files.pythonhosted.org/packages/b4/f6/259fb1eaaec921f59b17203b0daee69829761226d3b980d5191d7723dd83/xxhash-3.7.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3573a651d146912da9daa9e29e5fbc45994420daaa9ef1e2fa5823e1dc485513", size = 448534, upload-time = "2026-04-25T11:08:49.149Z" }, + { url = "https://files.pythonhosted.org/packages/7b/16/a66d0eaf6a7e68532c07714361ddc904c663ec940f3b028c1ae4a21a7b9d/xxhash-3.7.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5ec1e080a3d02d94ea9335bfab0e3374b877e25411422c18f51a943fa4b46381", size = 196217, upload-time = "2026-04-25T11:08:50.805Z" }, + { url = "https://files.pythonhosted.org/packages/8d/ef/d2efc7fc51756dc52509109d1a25cefc859d74bc4b19a167b12dbd8c2786/xxhash-3.7.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:84415265192072d8638a3afc3c1bc5995e310570cd9acb54dc46d3939e364fe0", size = 286906, upload-time = "2026-04-25T11:08:52.418Z" }, + { url = "https://files.pythonhosted.org/packages/fc/67/25decd1d4a4018582ec4db2a868a2b7e40640f4adb20dfeb19ac923aa825/xxhash-3.7.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8d4dea659b57443989ef32f4295104fd6912c73d0bf26d1d148bb88a9f159b02", size = 213057, upload-time = "2026-04-25T11:08:54.105Z" }, + { url = "https://files.pythonhosted.org/packages/0d/5d/17651eb29d06786cdc40c60ae3d27d645aa5d61d2eca6237a7ba0b94789b/xxhash-3.7.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:05ece0fe4d9c9c2728912d1981ae1566cfc83a011571b24732cbf76e1fb70dca", size = 243886, upload-time = "2026-04-25T11:08:56.109Z" }, + { url = "https://files.pythonhosted.org/packages/8a/d4/174d9cf7502243d586e6a9ae842b1ae23026620995114f85f1380e588bc9/xxhash-3.7.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:fd880353cf1ffaf321bc18dd663e111976dbd0d3bbd8a66d58d2b470dfa7f396", size = 201015, upload-time = "2026-04-25T11:08:57.777Z" }, + { url = "https://files.pythonhosted.org/packages/91/8c/2254e2d06c3ac5e6fe22eaf3da791b87ea823ae9f2c17b4af66755c5752d/xxhash-3.7.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4e15cc9e2817f6481160f930c62842b3ff419e20e13072bcbab12230943092bc", size = 213457, upload-time = "2026-04-25T11:08:59.826Z" }, + { url = "https://files.pythonhosted.org/packages/79/a2/e3daa762545921173e3360f3b4ff7fc63c2d27359f7230ec1a7a74e117f6/xxhash-3.7.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:90b9d1a8bd37d768ffc92a1f651ec69afc532a96fa1ac2ea7abbed5d630b3237", size = 277738, upload-time = "2026-04-25T11:09:01.423Z" }, + { url = "https://files.pythonhosted.org/packages/e1/4c/e186da2c46b87f5204640e008d42730bf3c1ee9f0efb71ae1ebcdfeac681/xxhash-3.7.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:157c49475b34ecea8809e51123d9769a534e139d1247942f7a4bc67710bb2533", size = 417127, upload-time = "2026-04-25T11:09:03.592Z" }, + { url = "https://files.pythonhosted.org/packages/17/28/3798e15007a3712d0da3d3fe70f8e11916569858b5cc371053bc26270832/xxhash-3.7.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5a6ddec83325685e729ca119d1f5c518ec39294212ecd770e60693cdc5f7eb79", size = 193962, upload-time = "2026-04-25T11:09:06.228Z" }, + { url = "https://files.pythonhosted.org/packages/ad/95/a26baa93b5241fd7630998816a4ec47a5a0bad193b3f8fc8f3593e1a4a67/xxhash-3.7.0-cp314-cp314t-win32.whl", hash = "sha256:a04a6cab47e2166435aaf5b9e5ee41d1532cc8300efdef87f2a4d0acb7db19ed", size = 31643, upload-time = "2026-04-25T11:09:08.153Z" }, + { url = "https://files.pythonhosted.org/packages/44/36/5454f13c447e395f9b06a3e91274c59f503d31fad84e1836efe3bdb71f6a/xxhash-3.7.0-cp314-cp314t-win_amd64.whl", hash = "sha256:8653dd7c2eda020545bb2c71c7f7039b53fe7434d0fc1a0a9deb79ab3f1a4fc1", size = 32522, upload-time = "2026-04-25T11:09:09.534Z" }, + { url = "https://files.pythonhosted.org/packages/74/35/698e7e3ff38e22992ea24870a511d8762474fb6783627a2910ff22a185c2/xxhash-3.7.0-cp314-cp314t-win_arm64.whl", hash = "sha256:468f0fc114faaa4b36699f8e328bbc3bb11dc418ba94ac52c26dd736d4b6c637", size = 28807, upload-time = "2026-04-25T11:09:11.234Z" }, ] [[package]] name = "yarl" -version = "1.23.0" +version = "1.24.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "idna" }, { name = "multidict" }, { name = "propcache" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/23/6e/beb1beec874a72f23815c1434518bfc4ed2175065173fb138c3705f658d4/yarl-1.23.0.tar.gz", hash = "sha256:53b1ea6ca88ebd4420379c330aea57e258408dd0df9af0992e5de2078dc9f5d5", size = 194676, upload-time = "2026-03-01T22:07:53.373Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/4b/a0a6e5d0ee8a2f3a373ddef8a4097d74ac901ac363eea1440464ccbe0898/yarl-1.23.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:16c6994ac35c3e74fb0ae93323bf8b9c2a9088d55946109489667c510a7d010e", size = 123796, upload-time = "2026-03-01T22:05:41.412Z" }, - { url = "https://files.pythonhosted.org/packages/67/b6/8925d68af039b835ae876db5838e82e76ec87b9782ecc97e192b809c4831/yarl-1.23.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4a42e651629dafb64fd5b0286a3580613702b5809ad3f24934ea87595804f2c5", size = 86547, upload-time = "2026-03-01T22:05:42.841Z" }, - { url = "https://files.pythonhosted.org/packages/ae/50/06d511cc4b8e0360d3c94af051a768e84b755c5eb031b12adaaab6dec6e5/yarl-1.23.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7c6b9461a2a8b47c65eef63bb1c76a4f1c119618ffa99ea79bc5bb1e46c5821b", size = 85854, upload-time = "2026-03-01T22:05:44.85Z" }, - { url = "https://files.pythonhosted.org/packages/c4/f4/4e30b250927ffdab4db70da08b9b8d2194d7c7b400167b8fbeca1e4701ca/yarl-1.23.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2569b67d616eab450d262ca7cb9f9e19d2f718c70a8b88712859359d0ab17035", size = 98351, upload-time = "2026-03-01T22:05:46.836Z" }, - { url = "https://files.pythonhosted.org/packages/86/fc/4118c5671ea948208bdb1492d8b76bdf1453d3e73df051f939f563e7dcc5/yarl-1.23.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e9d9a4d06d3481eab79803beb4d9bd6f6a8e781ec078ac70d7ef2dcc29d1bea5", size = 92711, upload-time = "2026-03-01T22:05:48.316Z" }, - { url = "https://files.pythonhosted.org/packages/56/11/1ed91d42bd9e73c13dc9e7eb0dd92298d75e7ac4dd7f046ad0c472e231cd/yarl-1.23.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f514f6474e04179d3d33175ed3f3e31434d3130d42ec153540d5b157deefd735", size = 106014, upload-time = "2026-03-01T22:05:50.028Z" }, - { url = "https://files.pythonhosted.org/packages/ce/c9/74e44e056a23fbc33aca71779ef450ca648a5bc472bdad7a82339918f818/yarl-1.23.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fda207c815b253e34f7e1909840fd14299567b1c0eb4908f8c2ce01a41265401", size = 105557, upload-time = "2026-03-01T22:05:51.416Z" }, - { url = "https://files.pythonhosted.org/packages/66/fe/b1e10b08d287f518994f1e2ff9b6d26f0adeecd8dd7d533b01bab29a3eda/yarl-1.23.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34b6cf500e61c90f305094911f9acc9c86da1a05a7a3f5be9f68817043f486e4", size = 101559, upload-time = "2026-03-01T22:05:52.872Z" }, - { url = "https://files.pythonhosted.org/packages/72/59/c5b8d94b14e3d3c2a9c20cb100119fd534ab5a14b93673ab4cc4a4141ea5/yarl-1.23.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d7504f2b476d21653e4d143f44a175f7f751cd41233525312696c76aa3dbb23f", size = 100502, upload-time = "2026-03-01T22:05:54.954Z" }, - { url = "https://files.pythonhosted.org/packages/77/4f/96976cb54cbfc5c9fd73ed4c51804f92f209481d1fb190981c0f8a07a1d7/yarl-1.23.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:578110dd426f0d209d1509244e6d4a3f1a3e9077655d98c5f22583d63252a08a", size = 98027, upload-time = "2026-03-01T22:05:56.409Z" }, - { url = "https://files.pythonhosted.org/packages/63/6e/904c4f476471afdbad6b7e5b70362fb5810e35cd7466529a97322b6f5556/yarl-1.23.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:609d3614d78d74ebe35f54953c5bbd2ac647a7ddb9c30a5d877580f5e86b22f2", size = 95369, upload-time = "2026-03-01T22:05:58.141Z" }, - { url = "https://files.pythonhosted.org/packages/9d/40/acfcdb3b5f9d68ef499e39e04d25e141fe90661f9d54114556cf83be8353/yarl-1.23.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4966242ec68afc74c122f8459abd597afd7d8a60dc93d695c1334c5fd25f762f", size = 105565, upload-time = "2026-03-01T22:06:00.286Z" }, - { url = "https://files.pythonhosted.org/packages/5e/c6/31e28f3a6ba2869c43d124f37ea5260cac9c9281df803c354b31f4dd1f3c/yarl-1.23.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e0fd068364a6759bc794459f0a735ab151d11304346332489c7972bacbe9e72b", size = 99813, upload-time = "2026-03-01T22:06:01.712Z" }, - { url = "https://files.pythonhosted.org/packages/08/1f/6f65f59e72d54aa467119b63fc0b0b1762eff0232db1f4720cd89e2f4a17/yarl-1.23.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:39004f0ad156da43e86aa71f44e033de68a44e5a31fc53507b36dd253970054a", size = 105632, upload-time = "2026-03-01T22:06:03.188Z" }, - { url = "https://files.pythonhosted.org/packages/a3/c4/18b178a69935f9e7a338127d5b77d868fdc0f0e49becd286d51b3a18c61d/yarl-1.23.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e5723c01a56c5028c807c701aa66722916d2747ad737a046853f6c46f4875543", size = 101895, upload-time = "2026-03-01T22:06:04.651Z" }, - { url = "https://files.pythonhosted.org/packages/8f/54/f5b870b5505663911dba950a8e4776a0dbd51c9c54c0ae88e823e4b874a0/yarl-1.23.0-cp313-cp313-win32.whl", hash = "sha256:1b6b572edd95b4fa8df75de10b04bc81acc87c1c7d16bcdd2035b09d30acc957", size = 82356, upload-time = "2026-03-01T22:06:06.04Z" }, - { url = "https://files.pythonhosted.org/packages/7a/84/266e8da36879c6edcd37b02b547e2d9ecdfea776be49598e75696e3316e1/yarl-1.23.0-cp313-cp313-win_amd64.whl", hash = "sha256:baaf55442359053c7d62f6f8413a62adba3205119bcb6f49594894d8be47e5e3", size = 87515, upload-time = "2026-03-01T22:06:08.107Z" }, - { url = "https://files.pythonhosted.org/packages/00/fd/7e1c66efad35e1649114fa13f17485f62881ad58edeeb7f49f8c5e748bf9/yarl-1.23.0-cp313-cp313-win_arm64.whl", hash = "sha256:fb4948814a2a98e3912505f09c9e7493b1506226afb1f881825368d6fb776ee3", size = 81785, upload-time = "2026-03-01T22:06:10.181Z" }, - { url = "https://files.pythonhosted.org/packages/9c/fc/119dd07004f17ea43bb91e3ece6587759edd7519d6b086d16bfbd3319982/yarl-1.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:aecfed0b41aa72b7881712c65cf764e39ce2ec352324f5e0837c7048d9e6daaa", size = 130719, upload-time = "2026-03-01T22:06:11.708Z" }, - { url = "https://files.pythonhosted.org/packages/e6/0d/9f2348502fbb3af409e8f47730282cd6bc80dec6630c1e06374d882d6eb2/yarl-1.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a41bcf68efd19073376eb8cf948b8d9be0af26256403e512bb18f3966f1f9120", size = 89690, upload-time = "2026-03-01T22:06:13.429Z" }, - { url = "https://files.pythonhosted.org/packages/50/93/e88f3c80971b42cfc83f50a51b9d165a1dbf154b97005f2994a79f212a07/yarl-1.23.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:cde9a2ecd91668bcb7f077c4966d8ceddb60af01b52e6e3e2680e4cf00ad1a59", size = 89851, upload-time = "2026-03-01T22:06:15.53Z" }, - { url = "https://files.pythonhosted.org/packages/1c/07/61c9dd8ba8f86473263b4036f70fb594c09e99c0d9737a799dfd8bc85651/yarl-1.23.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5023346c4ee7992febc0068e7593de5fa2bf611848c08404b35ebbb76b1b0512", size = 95874, upload-time = "2026-03-01T22:06:17.553Z" }, - { url = "https://files.pythonhosted.org/packages/9e/e9/f9ff8ceefba599eac6abddcfb0b3bee9b9e636e96dbf54342a8577252379/yarl-1.23.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d1009abedb49ae95b136a8904a3f71b342f849ffeced2d3747bf29caeda218c4", size = 88710, upload-time = "2026-03-01T22:06:19.004Z" }, - { url = "https://files.pythonhosted.org/packages/eb/78/0231bfcc5d4c8eec220bc2f9ef82cb4566192ea867a7c5b4148f44f6cbcd/yarl-1.23.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a8d00f29b42f534cc8aa3931cfe773b13b23e561e10d2b26f27a8d309b0e82a1", size = 101033, upload-time = "2026-03-01T22:06:21.203Z" }, - { url = "https://files.pythonhosted.org/packages/cd/9b/30ea5239a61786f18fd25797151a17fbb3be176977187a48d541b5447dd4/yarl-1.23.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:95451e6ce06c3e104556d73b559f5da6c34a069b6b62946d3ad66afcd51642ea", size = 100817, upload-time = "2026-03-01T22:06:22.738Z" }, - { url = "https://files.pythonhosted.org/packages/62/e2/a4980481071791bc83bce2b7a1a1f7adcabfa366007518b4b845e92eeee3/yarl-1.23.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:531ef597132086b6cf96faa7c6c1dcd0361dd5f1694e5cc30375907b9b7d3ea9", size = 97482, upload-time = "2026-03-01T22:06:24.21Z" }, - { url = "https://files.pythonhosted.org/packages/e5/1e/304a00cf5f6100414c4b5a01fc7ff9ee724b62158a08df2f8170dfc72a2d/yarl-1.23.0-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:88f9fb0116fbfcefcab70f85cf4b74a2b6ce5d199c41345296f49d974ddb4123", size = 95949, upload-time = "2026-03-01T22:06:25.697Z" }, - { url = "https://files.pythonhosted.org/packages/68/03/093f4055ed4cae649ac53bca3d180bd37102e9e11d048588e9ab0c0108d0/yarl-1.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e7b0460976dc75cb87ad9cc1f9899a4b97751e7d4e77ab840fc9b6d377b8fd24", size = 95839, upload-time = "2026-03-01T22:06:27.309Z" }, - { url = "https://files.pythonhosted.org/packages/b9/28/4c75ebb108f322aa8f917ae10a8ffa4f07cae10a8a627b64e578617df6a0/yarl-1.23.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:115136c4a426f9da976187d238e84139ff6b51a20839aa6e3720cd1026d768de", size = 90696, upload-time = "2026-03-01T22:06:29.048Z" }, - { url = "https://files.pythonhosted.org/packages/23/9c/42c2e2dd91c1a570402f51bdf066bfdb1241c2240ba001967bad778e77b7/yarl-1.23.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:ead11956716a940c1abc816b7df3fa2b84d06eaed8832ca32f5c5e058c65506b", size = 100865, upload-time = "2026-03-01T22:06:30.525Z" }, - { url = "https://files.pythonhosted.org/packages/74/05/1bcd60a8a0a914d462c305137246b6f9d167628d73568505fce3f1cb2e65/yarl-1.23.0-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:fe8f8f5e70e6dbdfca9882cd9deaac058729bcf323cf7a58660901e55c9c94f6", size = 96234, upload-time = "2026-03-01T22:06:32.692Z" }, - { url = "https://files.pythonhosted.org/packages/90/b2/f52381aac396d6778ce516b7bc149c79e65bfc068b5de2857ab69eeea3b7/yarl-1.23.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:a0e317df055958a0c1e79e5d2aa5a5eaa4a6d05a20d4b0c9c3f48918139c9fc6", size = 100295, upload-time = "2026-03-01T22:06:34.268Z" }, - { url = "https://files.pythonhosted.org/packages/e5/e8/638bae5bbf1113a659b2435d8895474598afe38b4a837103764f603aba56/yarl-1.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6f0fd84de0c957b2d280143522c4f91a73aada1923caee763e24a2b3fda9f8a5", size = 97784, upload-time = "2026-03-01T22:06:35.864Z" }, - { url = "https://files.pythonhosted.org/packages/80/25/a3892b46182c586c202629fc2159aa13975d3741d52ebd7347fd501d48d5/yarl-1.23.0-cp313-cp313t-win32.whl", hash = "sha256:93a784271881035ab4406a172edb0faecb6e7d00f4b53dc2f55919d6c9688595", size = 88313, upload-time = "2026-03-01T22:06:37.39Z" }, - { url = "https://files.pythonhosted.org/packages/43/68/8c5b36aa5178900b37387937bc2c2fe0e9505537f713495472dcf6f6fccc/yarl-1.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:dd00607bffbf30250fe108065f07453ec124dbf223420f57f5e749b04295e090", size = 94932, upload-time = "2026-03-01T22:06:39.579Z" }, - { url = "https://files.pythonhosted.org/packages/c6/cc/d79ba8292f51f81f4dc533a8ccfb9fc6992cabf0998ed3245de7589dc07c/yarl-1.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:ac09d42f48f80c9ee1635b2fcaa819496a44502737660d3c0f2ade7526d29144", size = 84786, upload-time = "2026-03-01T22:06:41.988Z" }, - { url = "https://files.pythonhosted.org/packages/90/98/b85a038d65d1b92c3903ab89444f48d3cee490a883477b716d7a24b1a78c/yarl-1.23.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:21d1b7305a71a15b4794b5ff22e8eef96ff4a6d7f9657155e5aa419444b28912", size = 124455, upload-time = "2026-03-01T22:06:43.615Z" }, - { url = "https://files.pythonhosted.org/packages/39/54/bc2b45559f86543d163b6e294417a107bb87557609007c007ad889afec18/yarl-1.23.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:85610b4f27f69984932a7abbe52703688de3724d9f72bceb1cca667deff27474", size = 86752, upload-time = "2026-03-01T22:06:45.425Z" }, - { url = "https://files.pythonhosted.org/packages/24/f9/e8242b68362bffe6fb536c8db5076861466fc780f0f1b479fc4ffbebb128/yarl-1.23.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23f371bd662cf44a7630d4d113101eafc0cfa7518a2760d20760b26021454719", size = 86291, upload-time = "2026-03-01T22:06:46.974Z" }, - { url = "https://files.pythonhosted.org/packages/ea/d8/d1cb2378c81dd729e98c716582b1ccb08357e8488e4c24714658cc6630e8/yarl-1.23.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4a80f77dc1acaaa61f0934176fccca7096d9b1ff08c8ba9cddf5ae034a24319", size = 99026, upload-time = "2026-03-01T22:06:48.459Z" }, - { url = "https://files.pythonhosted.org/packages/0a/ff/7196790538f31debe3341283b5b0707e7feb947620fc5e8236ef28d44f72/yarl-1.23.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:bd654fad46d8d9e823afbb4f87c79160b5a374ed1ff5bde24e542e6ba8f41434", size = 92355, upload-time = "2026-03-01T22:06:50.306Z" }, - { url = "https://files.pythonhosted.org/packages/c1/56/25d58c3eddde825890a5fe6aa1866228377354a3c39262235234ab5f616b/yarl-1.23.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:682bae25f0a0dd23a056739f23a134db9f52a63e2afd6bfb37ddc76292bbd723", size = 106417, upload-time = "2026-03-01T22:06:52.1Z" }, - { url = "https://files.pythonhosted.org/packages/51/8a/882c0e7bc8277eb895b31bce0138f51a1ba551fc2e1ec6753ffc1e7c1377/yarl-1.23.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a82836cab5f197a0514235aaf7ffccdc886ccdaa2324bc0aafdd4ae898103039", size = 106422, upload-time = "2026-03-01T22:06:54.424Z" }, - { url = "https://files.pythonhosted.org/packages/42/2b/fef67d616931055bf3d6764885990a3ac647d68734a2d6a9e1d13de437a2/yarl-1.23.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1c57676bdedc94cd3bc37724cf6f8cd2779f02f6aba48de45feca073e714fe52", size = 101915, upload-time = "2026-03-01T22:06:55.895Z" }, - { url = "https://files.pythonhosted.org/packages/18/6a/530e16aebce27c5937920f3431c628a29a4b6b430fab3fd1c117b26ff3f6/yarl-1.23.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c7f8dc16c498ff06497c015642333219871effba93e4a2e8604a06264aca5c5c", size = 100690, upload-time = "2026-03-01T22:06:58.21Z" }, - { url = "https://files.pythonhosted.org/packages/88/08/93749219179a45e27b036e03260fda05190b911de8e18225c294ac95bbc9/yarl-1.23.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:5ee586fb17ff8f90c91cf73c6108a434b02d69925f44f5f8e0d7f2f260607eae", size = 98750, upload-time = "2026-03-01T22:06:59.794Z" }, - { url = "https://files.pythonhosted.org/packages/d9/cf/ea424a004969f5d81a362110a6ac1496d79efdc6d50c2c4b2e3ea0fc2519/yarl-1.23.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:17235362f580149742739cc3828b80e24029d08cbb9c4bda0242c7b5bc610a8e", size = 94685, upload-time = "2026-03-01T22:07:01.375Z" }, - { url = "https://files.pythonhosted.org/packages/e2/b7/14341481fe568e2b0408bcf1484c652accafe06a0ade9387b5d3fd9df446/yarl-1.23.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:0793e2bd0cf14234983bbb371591e6bea9e876ddf6896cdcc93450996b0b5c85", size = 106009, upload-time = "2026-03-01T22:07:03.151Z" }, - { url = "https://files.pythonhosted.org/packages/0a/e6/5c744a9b54f4e8007ad35bce96fbc9218338e84812d36f3390cea616881a/yarl-1.23.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3650dc2480f94f7116c364096bc84b1d602f44224ef7d5c7208425915c0475dd", size = 100033, upload-time = "2026-03-01T22:07:04.701Z" }, - { url = "https://files.pythonhosted.org/packages/0c/23/e3bfc188d0b400f025bc49d99793d02c9abe15752138dcc27e4eaf0c4a9e/yarl-1.23.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f40e782d49630ad384db66d4d8b73ff4f1b8955dc12e26b09a3e3af064b3b9d6", size = 106483, upload-time = "2026-03-01T22:07:06.231Z" }, - { url = "https://files.pythonhosted.org/packages/72/42/f0505f949a90b3f8b7a363d6cbdf398f6e6c58946d85c6d3a3bc70595b26/yarl-1.23.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:94f8575fbdf81749008d980c17796097e645574a3b8c28ee313931068dad14fe", size = 102175, upload-time = "2026-03-01T22:07:08.4Z" }, - { url = "https://files.pythonhosted.org/packages/aa/65/b39290f1d892a9dd671d1c722014ca062a9c35d60885d57e5375db0404b5/yarl-1.23.0-cp314-cp314-win32.whl", hash = "sha256:c8aa34a5c864db1087d911a0b902d60d203ea3607d91f615acd3f3108ac32169", size = 83871, upload-time = "2026-03-01T22:07:09.968Z" }, - { url = "https://files.pythonhosted.org/packages/a9/5b/9b92f54c784c26e2a422e55a8d2607ab15b7ea3349e28359282f84f01d43/yarl-1.23.0-cp314-cp314-win_amd64.whl", hash = "sha256:63e92247f383c85ab00dd0091e8c3fa331a96e865459f5ee80353c70a4a42d70", size = 89093, upload-time = "2026-03-01T22:07:11.501Z" }, - { url = "https://files.pythonhosted.org/packages/e0/7d/8a84dc9381fd4412d5e7ff04926f9865f6372b4c2fd91e10092e65d29eb8/yarl-1.23.0-cp314-cp314-win_arm64.whl", hash = "sha256:70efd20be968c76ece7baa8dafe04c5be06abc57f754d6f36f3741f7aa7a208e", size = 83384, upload-time = "2026-03-01T22:07:13.069Z" }, - { url = "https://files.pythonhosted.org/packages/dd/8d/d2fad34b1c08aa161b74394183daa7d800141aaaee207317e82c790b418d/yarl-1.23.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9a18d6f9359e45722c064c97464ec883eb0e0366d33eda61cb19a244bf222679", size = 131019, upload-time = "2026-03-01T22:07:14.903Z" }, - { url = "https://files.pythonhosted.org/packages/19/ff/33009a39d3ccf4b94d7d7880dfe17fb5816c5a4fe0096d9b56abceea9ac7/yarl-1.23.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2803ed8b21ca47a43da80a6fd1ed3019d30061f7061daa35ac54f63933409412", size = 89894, upload-time = "2026-03-01T22:07:17.372Z" }, - { url = "https://files.pythonhosted.org/packages/0c/f1/dab7ac5e7306fb79c0190766a3c00b4cb8d09a1f390ded68c85a5934faf5/yarl-1.23.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:394906945aa8b19fc14a61cf69743a868bb8c465efe85eee687109cc540b98f4", size = 89979, upload-time = "2026-03-01T22:07:19.361Z" }, - { url = "https://files.pythonhosted.org/packages/aa/b1/08e95f3caee1fad6e65017b9f26c1d79877b502622d60e517de01e72f95d/yarl-1.23.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:71d006bee8397a4a89f469b8deb22469fe7508132d3c17fa6ed871e79832691c", size = 95943, upload-time = "2026-03-01T22:07:21.266Z" }, - { url = "https://files.pythonhosted.org/packages/c0/cc/6409f9018864a6aa186c61175b977131f373f1988e198e031236916e87e4/yarl-1.23.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:62694e275c93d54f7ccedcfef57d42761b2aad5234b6be1f3e3026cae4001cd4", size = 88786, upload-time = "2026-03-01T22:07:23.129Z" }, - { url = "https://files.pythonhosted.org/packages/76/40/cc22d1d7714b717fde2006fad2ced5efe5580606cb059ae42117542122f3/yarl-1.23.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a31de1613658308efdb21ada98cbc86a97c181aa050ba22a808120bb5be3ab94", size = 101307, upload-time = "2026-03-01T22:07:24.689Z" }, - { url = "https://files.pythonhosted.org/packages/8f/0d/476c38e85ddb4c6ec6b20b815bdd779aa386a013f3d8b85516feee55c8dc/yarl-1.23.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb1e8b8d66c278b21d13b0a7ca22c41dd757a7c209c6b12c313e445c31dd3b28", size = 100904, upload-time = "2026-03-01T22:07:26.287Z" }, - { url = "https://files.pythonhosted.org/packages/72/32/0abe4a76d59adf2081dcb0397168553ece4616ada1c54d1c49d8936c74f8/yarl-1.23.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:50f9d8d531dfb767c565f348f33dd5139a6c43f5cbdf3f67da40d54241df93f6", size = 97728, upload-time = "2026-03-01T22:07:27.906Z" }, - { url = "https://files.pythonhosted.org/packages/b7/35/7b30f4810fba112f60f5a43237545867504e15b1c7647a785fbaf588fac2/yarl-1.23.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:575aa4405a656e61a540f4a80eaa5260f2a38fff7bfdc4b5f611840d76e9e277", size = 95964, upload-time = "2026-03-01T22:07:30.198Z" }, - { url = "https://files.pythonhosted.org/packages/2d/86/ed7a73ab85ef00e8bb70b0cb5421d8a2a625b81a333941a469a6f4022828/yarl-1.23.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:041b1a4cefacf65840b4e295c6985f334ba83c30607441ae3cf206a0eed1a2e4", size = 95882, upload-time = "2026-03-01T22:07:32.132Z" }, - { url = "https://files.pythonhosted.org/packages/19/90/d56967f61a29d8498efb7afb651e0b2b422a1e9b47b0ab5f4e40a19b699b/yarl-1.23.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d38c1e8231722c4ce40d7593f28d92b5fc72f3e9774fe73d7e800ec32299f63a", size = 90797, upload-time = "2026-03-01T22:07:34.404Z" }, - { url = "https://files.pythonhosted.org/packages/72/00/8b8f76909259f56647adb1011d7ed8b321bcf97e464515c65016a47ecdf0/yarl-1.23.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:d53834e23c015ee83a99377db6e5e37d8484f333edb03bd15b4bc312cc7254fb", size = 101023, upload-time = "2026-03-01T22:07:35.953Z" }, - { url = "https://files.pythonhosted.org/packages/ac/e2/cab11b126fb7d440281b7df8e9ddbe4851e70a4dde47a202b6642586b8d9/yarl-1.23.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:2e27c8841126e017dd2a054a95771569e6070b9ee1b133366d8b31beb5018a41", size = 96227, upload-time = "2026-03-01T22:07:37.594Z" }, - { url = "https://files.pythonhosted.org/packages/c2/9b/2c893e16bfc50e6b2edf76c1a9eb6cb0c744346197e74c65e99ad8d634d0/yarl-1.23.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:76855800ac56f878847a09ce6dba727c93ca2d89c9e9d63002d26b916810b0a2", size = 100302, upload-time = "2026-03-01T22:07:39.334Z" }, - { url = "https://files.pythonhosted.org/packages/28/ec/5498c4e3a6d5f1003beb23405671c2eb9cdbf3067d1c80f15eeafe301010/yarl-1.23.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e09fd068c2e169a7070d83d3bde728a4d48de0549f975290be3c108c02e499b4", size = 98202, upload-time = "2026-03-01T22:07:41.717Z" }, - { url = "https://files.pythonhosted.org/packages/fe/c3/cd737e2d45e70717907f83e146f6949f20cc23cd4bf7b2688727763aa458/yarl-1.23.0-cp314-cp314t-win32.whl", hash = "sha256:73309162a6a571d4cbd3b6a1dcc703c7311843ae0d1578df6f09be4e98df38d4", size = 90558, upload-time = "2026-03-01T22:07:43.433Z" }, - { url = "https://files.pythonhosted.org/packages/e1/19/3774d162f6732d1cfb0b47b4140a942a35ca82bb19b6db1f80e9e7bdc8f8/yarl-1.23.0-cp314-cp314t-win_amd64.whl", hash = "sha256:4503053d296bc6e4cbd1fad61cf3b6e33b939886c4f249ba7c78b602214fabe2", size = 97610, upload-time = "2026-03-01T22:07:45.773Z" }, - { url = "https://files.pythonhosted.org/packages/51/47/3fa2286c3cb162c71cdb34c4224d5745a1ceceb391b2bd9b19b668a8d724/yarl-1.23.0-cp314-cp314t-win_arm64.whl", hash = "sha256:44bb7bef4ea409384e3f8bc36c063d77ea1b8d4a5b2706956c0d6695f07dcc25", size = 86041, upload-time = "2026-03-01T22:07:49.026Z" }, - { url = "https://files.pythonhosted.org/packages/69/68/c8739671f5699c7dc470580a4f821ef37c32c4cb0b047ce223a7f115757f/yarl-1.23.0-py3-none-any.whl", hash = "sha256:a2df6afe50dea8ae15fa34c9f824a3ee958d785fd5d089063d960bae1daa0a3f", size = 48288, upload-time = "2026-03-01T22:07:51.388Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/79/12/1e8f37460ea0f7eb59c221fdaf0ed75e7ac43e97f8093b9c6f411df50a78/yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8", size = 210798, upload-time = "2026-05-19T21:31:05.599Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/62/fcf0ce677f17e5c471c06311dd25964be38a4c586993632910d2e75278bc/yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536", size = 128978, upload-time = "2026-05-19T21:29:23.83Z" }, + { url = "https://files.pythonhosted.org/packages/d3/58/8e63299bb71ed61a834121d9d3fe6c9fcf2a6a5d09754ff4f20f2d20baf5/yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607", size = 91733, upload-time = "2026-05-19T21:29:25.375Z" }, + { url = "https://files.pythonhosted.org/packages/c1/24/16748d5dab6daec8b0ed81ccec639a1cded0f18dcc62a4f696b4fe366c37/yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1", size = 91113, upload-time = "2026-05-19T21:29:26.928Z" }, + { url = "https://files.pythonhosted.org/packages/1b/66/b63fff7b71211e866624b21432d5943cbb633eb0c2872d9ee3070648f22c/yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986", size = 103899, upload-time = "2026-05-19T21:29:28.842Z" }, + { url = "https://files.pythonhosted.org/packages/9d/ac/ba1974b8533909636f7733fe86cf677e3619527c3c2fa913e0ea89c48757/yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488", size = 97862, upload-time = "2026-05-19T21:29:31.086Z" }, + { url = "https://files.pythonhosted.org/packages/1b/a5/123ac993b5c2ba6f554a140305620cb8f150fa543711bbc49be3ec0a65a4/yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b", size = 111060, upload-time = "2026-05-19T21:29:32.657Z" }, + { url = "https://files.pythonhosted.org/packages/23/37/c472d3af3509688392134a88a825276770a187f1daa4de3f6dc0a327a751/yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592", size = 110613, upload-time = "2026-05-19T21:29:34.379Z" }, + { url = "https://files.pythonhosted.org/packages/df/88/09c28dad91e662ccfaa1b78f1c57badde74fc9d0b23e74aef644750ecd73/yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617", size = 107012, upload-time = "2026-05-19T21:29:36.216Z" }, + { url = "https://files.pythonhosted.org/packages/07/ab/9d4f69d571a94f4d112fa7e2e007200f5a54d319f58c82ac7b7baa61f5c6/yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92", size = 105887, upload-time = "2026-05-19T21:29:38.746Z" }, + { url = "https://files.pythonhosted.org/packages/8e/9a/000b2b66c0d772a499fc531d21dab92dfeb73b640a12eed6ba89f49bb2d0/yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a", size = 103620, upload-time = "2026-05-19T21:29:40.368Z" }, + { url = "https://files.pythonhosted.org/packages/41/7c/7c1050f73450fbdaa3f0c72017059f00ce5e13366692f3dba25275a1083d/yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44", size = 100599, upload-time = "2026-05-19T21:29:42.66Z" }, + { url = "https://files.pythonhosted.org/packages/ec/b1/29e5756b3926705f5f6089bd5b9f50a56eaac550da6e260bf713ead44d04/yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a", size = 110604, upload-time = "2026-05-19T21:29:44.632Z" }, + { url = "https://files.pythonhosted.org/packages/a3/4b/8415bc96e9b150cde942fbac9a8182985e58f40ce5c54c34ed015407d3ee/yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf", size = 105161, upload-time = "2026-05-19T21:29:46.755Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d4/cde059abfa229553b7298a2eadde2752e723d50aeedaef86ce59da2718ee/yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056", size = 110619, upload-time = "2026-05-19T21:29:48.972Z" }, + { url = "https://files.pythonhosted.org/packages/e7/2c/d6a6c9a61549f7b6c7e6dc6937d195bcf069582b47b7200dcd0e7b256acf/yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992", size = 107362, upload-time = "2026-05-19T21:29:51Z" }, + { url = "https://files.pythonhosted.org/packages/92/dd/3ae5fe417e9d1c353a548553326eb9935e76b6b727161563b424cc296df3/yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656", size = 92667, upload-time = "2026-05-19T21:29:52.743Z" }, + { url = "https://files.pythonhosted.org/packages/10/cc/a7beb239f78f27fca1b053c8e8595e4179c02e62249b4687ec218c370c50/yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461", size = 87069, upload-time = "2026-05-19T21:29:54.442Z" }, + { url = "https://files.pythonhosted.org/packages/40/0e/e08087695fc12789263821c5dc0f8dc52b5b17efd0887cacf419f8a43ba3/yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2", size = 129670, upload-time = "2026-05-19T21:29:56.631Z" }, + { url = "https://files.pythonhosted.org/packages/3a/98/ab4b5ed1b1b5cd973c8a3eb994c3a6aefb6ce6d399e21bb5f0316c33815c/yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630", size = 91916, upload-time = "2026-05-19T21:29:58.645Z" }, + { url = "https://files.pythonhosted.org/packages/ba/b1/5297bb6a7df4782f7605bffc43b31f5044070935fbbcaa6c705a07e6ac65/yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8", size = 91625, upload-time = "2026-05-19T21:30:00.412Z" }, + { url = "https://files.pythonhosted.org/packages/02/a7/45baabfff76829264e623b185cff0c340d7e11bf3e1cd9ea37e7d17934bd/yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14", size = 104574, upload-time = "2026-05-19T21:30:02.544Z" }, + { url = "https://files.pythonhosted.org/packages/f3/40/3a5ab144d3d650ca37d4f4b57e56169be8af3ca34c448793e064b30baaed/yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535", size = 97534, upload-time = "2026-05-19T21:30:04.319Z" }, + { url = "https://files.pythonhosted.org/packages/9c/b5/5658fef3681fb5776b4513b052bec750009f47b3a592251c705d75375798/yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14", size = 111481, upload-time = "2026-05-19T21:30:05.988Z" }, + { url = "https://files.pythonhosted.org/packages/4c/06/fdcd7dde037f00866dce123ed4ba23dba94beb56fc4cf561668d27be37f2/yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3", size = 111529, upload-time = "2026-05-19T21:30:07.738Z" }, + { url = "https://files.pythonhosted.org/packages/c2/53/d81269aaafccea0d33396c03035de997b743f11e648e6e27a0df99c72980/yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208", size = 107338, upload-time = "2026-05-19T21:30:09.713Z" }, + { url = "https://files.pythonhosted.org/packages/ae/04/23049463f729bd899df203a7960505a75333edd499cda8aa1d5a82b64df5/yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50", size = 106147, upload-time = "2026-05-19T21:30:11.365Z" }, + { url = "https://files.pythonhosted.org/packages/14/18/04a4b5830b43ed5e4c5015b40e9f6241ad91487d71611061b4e111d6ac80/yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd", size = 104272, upload-time = "2026-05-19T21:30:12.978Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f7/8cffdf319aee7a7c1dbd07b61d91c3e3fda460c7a93b5f93e445f3806c4c/yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67", size = 99962, upload-time = "2026-05-19T21:30:15.001Z" }, + { url = "https://files.pythonhosted.org/packages/d7/39/b3cce3b7dbef64ac700ad4cea156a207d01bede0f507587616c364b5468e/yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1", size = 111063, upload-time = "2026-05-19T21:30:16.683Z" }, + { url = "https://files.pythonhosted.org/packages/a1/ea/100818505e7ebf165c7242ff17fdf7d9fee79e27234aeca871c1082920d7/yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1", size = 105438, upload-time = "2026-05-19T21:30:18.769Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d2/e075a0b32aa6625087de9e653087df0759fed5de4a435fef594181102a77/yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b", size = 111458, upload-time = "2026-05-19T21:30:21.024Z" }, + { url = "https://files.pythonhosted.org/packages/e6/5c/ceea7ba98b65c8eb8d947fdc52f9bedfcd43c6a57c9e3c90c17be8f324a3/yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8", size = 107589, upload-time = "2026-05-19T21:30:23.412Z" }, + { url = "https://files.pythonhosted.org/packages/fa/d9/5582d57e2b2db9b85eb6663a22efdd78e08805f3f5389566e9fcad254d1b/yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0", size = 94424, upload-time = "2026-05-19T21:30:25.425Z" }, + { url = "https://files.pythonhosted.org/packages/92/10/7dc07a0e22806a9280f42a57361395506e800c64e22737cd7b0886feab42/yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57", size = 88690, upload-time = "2026-05-19T21:30:27.623Z" }, + { url = "https://files.pythonhosted.org/packages/9e/13/d5b8e2c8667db955bcb3de233f18798fefe7edf1d7429c2c9d4f9c401114/yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b", size = 136248, upload-time = "2026-05-19T21:30:29.297Z" }, + { url = "https://files.pythonhosted.org/packages/de/46/a4a97c05c9c9b8fd266bb2a0df12992c7fbd02391eb9640583411b6dab32/yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761", size = 95084, upload-time = "2026-05-19T21:30:31.031Z" }, + { url = "https://files.pythonhosted.org/packages/95/b2/845cf2074a015e6fe0d0808cf1a2d9e868386c4220d657ebd8302b199043/yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8", size = 95272, upload-time = "2026-05-19T21:30:33.062Z" }, + { url = "https://files.pythonhosted.org/packages/fe/16/e69d4aa244aef45235ddfebc0e04036a6829842bc5a6a795aedc6c998d23/yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed", size = 101497, upload-time = "2026-05-19T21:30:34.842Z" }, + { url = "https://files.pythonhosted.org/packages/15/94/c07107715d621076863ee88b3ddf183fa5e9d4aba5769623c9979828410a/yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543", size = 94002, upload-time = "2026-05-19T21:30:37.724Z" }, + { url = "https://files.pythonhosted.org/packages/a9/35/fc1bbdd895b5e4010b8fdd037f7ed3aa289d3863e08231b30231ca9a0815/yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0", size = 106524, upload-time = "2026-05-19T21:30:40.196Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f2/32b66d0a4ba47c296cf86d03e2c67bff58399fe6d6d84d5205c04c66cc6d/yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024", size = 106165, upload-time = "2026-05-19T21:30:41.888Z" }, + { url = "https://files.pythonhosted.org/packages/95/47/37cb5ff50c5e825d4d38e81bb04d1b7e96bf960f7ab89f9850b162f3f114/yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf", size = 103010, upload-time = "2026-05-19T21:30:43.985Z" }, + { url = "https://files.pythonhosted.org/packages/6f/d2/4597912315096f7bb359e46e13bf8b60994fcbb2db29b804c0902ef4eff5/yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc", size = 101128, upload-time = "2026-05-19T21:30:46.291Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d5/c8e86e120521e646013d02a8e3b8884392e28494be8f392366e50d208efc/yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb", size = 101382, upload-time = "2026-05-19T21:30:48.085Z" }, + { url = "https://files.pythonhosted.org/packages/fa/98/70b229236118f89dbeb739b76f10225bbf53b5497725502594c9a01d699a/yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420", size = 95964, upload-time = "2026-05-19T21:30:49.785Z" }, + { url = "https://files.pythonhosted.org/packages/87/f8/56c386981e3c8648d279fdef2397ffec577e8320fd5649745e34d54faeb7/yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f", size = 106204, upload-time = "2026-05-19T21:30:51.862Z" }, + { url = "https://files.pythonhosted.org/packages/1a/1e/765afe97811ca35933e2a7de70ac57b1997ea2e4ee895719ee7a231fb7e5/yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa", size = 101510, upload-time = "2026-05-19T21:30:53.62Z" }, + { url = "https://files.pythonhosted.org/packages/ee/78/393913f4b9039e1edd09ae8a9bbb9d539be909a8abf6d8a2084585bed4b7/yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe", size = 105584, upload-time = "2026-05-19T21:30:55.962Z" }, + { url = "https://files.pythonhosted.org/packages/78/87/deb17b7049bbe74ea11a713b86f8f27800cc1c8648b0b797243ebb4830ba/yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd", size = 103410, upload-time = "2026-05-19T21:30:57.962Z" }, + { url = "https://files.pythonhosted.org/packages/8f/be/f9f7594e23b5b93affff0318e4593c1920331bcaefda326cabcad94296a1/yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215", size = 102980, upload-time = "2026-05-19T21:30:59.735Z" }, + { url = "https://files.pythonhosted.org/packages/65/a4/ba80dccd3593ff1f01051a818694d07b58cb8232677ee9a22a5a1f93a9fc/yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d", size = 91219, upload-time = "2026-05-19T21:31:01.934Z" }, + { url = "https://files.pythonhosted.org/packages/fd/4d/4b880086bd0d3e034d25647be1d830afc3e3f610e98c4ab3490af6b1b6d5/yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9", size = 53576, upload-time = "2026-05-19T21:31:03.909Z" }, ] [[package]] From 8693629941514b9a645021f3b030d1be9e910206 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 May 2026 21:14:40 +0000 Subject: [PATCH 084/128] Bump ruff from 0.15.11 to 0.15.14 Bumps [ruff](https://github.com/astral-sh/ruff) from 0.15.11 to 0.15.14. - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.15.11...0.15.14) --- updated-dependencies: - dependency-name: ruff dependency-version: 0.15.12 dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- uv.lock | 44 ++++++++++++++++++++++---------------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7e7c54b5..a9980603 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ dev = [ "pytest-cov>=7.1.0", "pytest-env>=1.6.0", "pytest-recording>=0.13.4", - "ruff>=0.15.11", + "ruff>=0.15.14", ] models = [ "genson>=1.3.0", diff --git a/uv.lock b/uv.lock index a7ba1296..e7b32a13 100644 --- a/uv.lock +++ b/uv.lock @@ -479,7 +479,7 @@ dev = [ { name = "pytest-cov", specifier = ">=7.1.0" }, { name = "pytest-env", specifier = ">=1.6.0" }, { name = "pytest-recording", specifier = ">=0.13.4" }, - { name = "ruff", specifier = ">=0.15.11" }, + { name = "ruff", specifier = ">=0.15.14" }, ] models = [ { name = "genson", specifier = ">=1.3.0" }, @@ -3461,27 +3461,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.15.11" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e4/8d/192f3d7103816158dfd5ea50d098ef2aec19194e6cbccd4b3485bdb2eb2d/ruff-0.15.11.tar.gz", hash = "sha256:f092b21708bf0e7437ce9ada249dfe688ff9a0954fc94abab05dcea7dcd29c33", size = 4637264, upload-time = "2026-04-16T18:46:26.58Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/02/1e/6aca3427f751295ab011828e15e9bf452200ac74484f1db4be0197b8170b/ruff-0.15.11-py3-none-linux_armv6l.whl", hash = "sha256:e927cfff503135c558eb581a0c9792264aae9507904eb27809cdcff2f2c847b7", size = 10607943, upload-time = "2026-04-16T18:46:05.967Z" }, - { url = "https://files.pythonhosted.org/packages/e7/26/1341c262e74f36d4e84f3d6f4df0ac68cd53331a66bfc5080daa17c84c0b/ruff-0.15.11-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:7a1b5b2938d8f890b76084d4fa843604d787a912541eae85fd7e233398bbb73e", size = 10988592, upload-time = "2026-04-16T18:46:00.742Z" }, - { url = "https://files.pythonhosted.org/packages/03/71/850b1d6ffa9564fbb6740429bad53df1094082fe515c8c1e74b6d8d05f18/ruff-0.15.11-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d4176f3d194afbdaee6e41b9ccb1a2c287dba8700047df474abfbe773825d1cb", size = 10338501, upload-time = "2026-04-16T18:46:03.723Z" }, - { url = "https://files.pythonhosted.org/packages/f2/11/cc1284d3e298c45a817a6aadb6c3e1d70b45c9b36d8d9cce3387b495a03a/ruff-0.15.11-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b17c886fb88203ced3afe7f14e8d5ae96e9d2f4ccc0ee66aa19f2c2675a27e4", size = 10670693, upload-time = "2026-04-16T18:46:41.941Z" }, - { url = "https://files.pythonhosted.org/packages/ce/9e/f8288b034ab72b371513c13f9a41d9ba3effac54e24bfb467b007daee2ca/ruff-0.15.11-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:49fafa220220afe7758a487b048de4c8f9f767f37dfefad46b9dd06759d003eb", size = 10416177, upload-time = "2026-04-16T18:46:21.717Z" }, - { url = "https://files.pythonhosted.org/packages/85/71/504d79abfd3d92532ba6bbe3d1c19fada03e494332a59e37c7c2dabae427/ruff-0.15.11-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2ab8427e74a00d93b8bda1307b1e60970d40f304af38bccb218e056c220120d", size = 11221886, upload-time = "2026-04-16T18:46:15.086Z" }, - { url = "https://files.pythonhosted.org/packages/43/5a/947e6ab7a5ad603d65b474be15a4cbc6d29832db5d762cd142e4e3a74164/ruff-0.15.11-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:195072c0c8e1fc8f940652073df082e37a5d9cb43b4ab1e4d0566ab8977a13b7", size = 12075183, upload-time = "2026-04-16T18:46:07.944Z" }, - { url = "https://files.pythonhosted.org/packages/9f/a1/0b7bb6268775fdd3a0818aee8efd8f5b4e231d24dd4d528ced2534023182/ruff-0.15.11-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a3a0996d486af3920dec930a2e7daed4847dfc12649b537a9335585ada163e9e", size = 11516575, upload-time = "2026-04-16T18:46:31.687Z" }, - { url = "https://files.pythonhosted.org/packages/30/c3/bb5168fc4d233cc06e95f482770d0f3c87945a0cd9f614b90ea8dc2f2833/ruff-0.15.11-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bef2cb556d509259f1fe440bb9cd33c756222cf0a7afe90d15edf0866702431", size = 11306537, upload-time = "2026-04-16T18:46:36.988Z" }, - { url = "https://files.pythonhosted.org/packages/e4/92/4cfae6441f3967317946f3b788136eecf093729b94d6561f963ed810c82e/ruff-0.15.11-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:030d921a836d7d4a12cf6e8d984a88b66094ccb0e0f17ddd55067c331191bf19", size = 11296813, upload-time = "2026-04-16T18:46:24.182Z" }, - { url = "https://files.pythonhosted.org/packages/43/26/972784c5dde8313acde8ac71ba8ac65475b85db4a2352a76c9934361f9bc/ruff-0.15.11-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:0e783b599b4577788dbbb66b9addcef87e9a8832f4ce0c19e34bf55543a2f890", size = 10633136, upload-time = "2026-04-16T18:46:39.802Z" }, - { url = "https://files.pythonhosted.org/packages/5b/53/3985a4f185020c2f367f2e08a103032e12564829742a1b417980ce1514a0/ruff-0.15.11-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:ae90592246625ba4a34349d68ec28d4400d75182b71baa196ddb9f82db025ef5", size = 10424701, upload-time = "2026-04-16T18:46:10.381Z" }, - { url = "https://files.pythonhosted.org/packages/d3/57/bf0dfb32241b56c83bb663a826133da4bf17f682ba8c096973065f6e6a68/ruff-0.15.11-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1f111d62e3c983ed20e0ca2e800f8d77433a5b1161947df99a5c2a3fb60514f0", size = 10873887, upload-time = "2026-04-16T18:46:29.157Z" }, - { url = "https://files.pythonhosted.org/packages/02/05/e48076b2a57dc33ee8c7a957296f97c744ca891a8ffb4ffb1aaa3b3f517d/ruff-0.15.11-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:06f483d6646f59eaffba9ae30956370d3a886625f511a3108994000480621d1c", size = 11404316, upload-time = "2026-04-16T18:46:19.462Z" }, - { url = "https://files.pythonhosted.org/packages/88/27/0195d15fe7a897cbcba0904792c4b7c9fdd958456c3a17d2ea6093716a9a/ruff-0.15.11-py3-none-win32.whl", hash = "sha256:476a2aa56b7da0b73a3ee80b6b2f0e19cce544245479adde7baa65466664d5f3", size = 10655535, upload-time = "2026-04-16T18:46:12.47Z" }, - { url = "https://files.pythonhosted.org/packages/3a/5e/c927b325bd4c1d3620211a4b96f47864633199feed60fa936025ab27e090/ruff-0.15.11-py3-none-win_amd64.whl", hash = "sha256:8b6756d88d7e234fb0c98c91511aae3cd519d5e3ed271cae31b20f39cb2a12a3", size = 11779692, upload-time = "2026-04-16T18:46:17.268Z" }, - { url = "https://files.pythonhosted.org/packages/63/b6/aeadee5443e49baa2facd51131159fd6301cc4ccfc1541e4df7b021c37dd/ruff-0.15.11-py3-none-win_arm64.whl", hash = "sha256:063fed18cc1bbe0ee7393957284a6fe8b588c6a406a285af3ee3f46da2391ee4", size = 11032614, upload-time = "2026-04-16T18:46:34.487Z" }, +version = "0.15.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/8a/8bce2894573e9dae6ff4d77fe34ad727d79b9e6238ad288c5638990d90f6/ruff-0.15.14.tar.gz", hash = "sha256:48e866b165be4a9bdbf310f7d3c9a07edef2fe8cd63ffeb4e00bb590506ebf9f", size = 4700910, upload-time = "2026-05-21T14:34:55.177Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/c8/74a92c6ff9fcfb4f1f947126d3ebee8389276e161ecc85de5bda7cda51bd/ruff-0.15.14-py3-none-linux_armv6l.whl", hash = "sha256:8dd2db9416e487c8d4b01fa7056bb02c4d05969d4f8d17a08c229c2f4ff3c108", size = 10739177, upload-time = "2026-05-21T14:34:37.332Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/254a35c20acc38a7223c9d2d594af12e794432464f2cdeb52af1dc4a892d/ruff-0.15.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:be4ff55af755bd71a00ab3dc6bd7ffc467bd76e0df6881e286c2e3d23e8fb43b", size = 11144969, upload-time = "2026-05-21T14:34:43.978Z" }, + { url = "https://files.pythonhosted.org/packages/56/9e/d13e40f83b8d0a94430e6778ce1d94a43b38cf2efe63278bdd2b4c65abbf/ruff-0.15.14-py3-none-macosx_11_0_arm64.whl", hash = "sha256:48d5909d7d06276ce7dde6d32bfa4b0d4cb2651145cd8ee4b440722cbc77832f", size = 10478207, upload-time = "2026-05-21T14:34:48.378Z" }, + { url = "https://files.pythonhosted.org/packages/8d/f1/b15a7839fa4f332f8acec78e20564f26bb2d866e3d21710b877fd0263000/ruff-0.15.14-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca8cbfa94c4f90984a67561978602746d4cd27103568f745fa90eee3f0d4107d", size = 10818459, upload-time = "2026-05-21T14:34:22.318Z" }, + { url = "https://files.pythonhosted.org/packages/45/33/53d651177f84f94b400a0e27f8824eeada3dddc9d5ee8aeb048f4352a520/ruff-0.15.14-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9a6bbc0333f1ab053423bcbf6226477d266ca7cec7738c4c8e3f55647803f3c4", size = 10541800, upload-time = "2026-05-21T14:34:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a6/868f87e0bf9786ed24b5d0d0ad8676b8a94fd1912f42cddf9cfc7857818a/ruff-0.15.14-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a24a4f7605d7003a6674d4387651effd939dead3fddd0f36561eb77a9a2e542", size = 11342149, upload-time = "2026-05-21T14:34:46.365Z" }, + { url = "https://files.pythonhosted.org/packages/a7/8b/38cd5c19faffdcc05a408d2b78edccc69492ab9720eadb49ea15ef80d768/ruff-0.15.14-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:049b5326e53ed80978f2fc041a280603f69dd6b0c95464342a2bb4572d9d9e2f", size = 12212563, upload-time = "2026-05-21T14:34:28.579Z" }, + { url = "https://files.pythonhosted.org/packages/3e/4d/a3c5b874a556d5731e3e657aaf04311bb76f0a5c3ec220ed43051be6b64b/ruff-0.15.14-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4ed42e6696c8dfa5f06728e6441993901f548eb92d73bc472cb5a38d1395fbf", size = 11493299, upload-time = "2026-05-21T14:34:41.836Z" }, + { url = "https://files.pythonhosted.org/packages/1e/c0/56472c251d09858a53e51efbd485b09e1995d8731668b76d52e5dd6ee0f1/ruff-0.15.14-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:715c543cf450c4888251f91c52f1942a800541d9bddd7ac060aa4e6b77ae7cba", size = 11455931, upload-time = "2026-05-21T14:34:57.276Z" }, + { url = "https://files.pythonhosted.org/packages/2c/4a/e2e7b4d8dbf233d4eace59c75bc3435fa6d8bd3bae82d351d4e4300c0fd1/ruff-0.15.14-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:72ebab6013ec887d439d8b7593737a0a4ffb06d45d209d4e4bf2e92813082d3f", size = 11400794, upload-time = "2026-05-21T14:34:39.773Z" }, + { url = "https://files.pythonhosted.org/packages/97/c7/83c0539fe34c3e09136204d1e75d6052492364e0b3cb05e9465423f567d7/ruff-0.15.14-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:49072d36abdbe97a8dd7f480afe9c675699c0c495d4c84076e2c1203c4550581", size = 10804759, upload-time = "2026-05-21T14:34:31.045Z" }, + { url = "https://files.pythonhosted.org/packages/86/a6/18f2bfc095a2ab4a78745644e428205532ce6653a5d0fa8501572891534d/ruff-0.15.14-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:958522aee105068640c2c2ceae08f413ae44d922f52a1374ac13d6a96032fc93", size = 10539517, upload-time = "2026-05-21T14:34:53.064Z" }, + { url = "https://files.pythonhosted.org/packages/54/3a/5a8b3b69c654d4e4bf1d246ac5b49cbcdac6eaab6905925f8915f31e3b80/ruff-0.15.14-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f3707da619a143a2e8830e2abab8224478d69ace2d28cb6c20543ae97c36bf61", size = 11065169, upload-time = "2026-05-21T14:34:24.484Z" }, + { url = "https://files.pythonhosted.org/packages/ed/c5/8864e4e7925b836ea354b31d57641ec03830564e281a8b6f061f8c3e0ec1/ruff-0.15.14-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:bb01d645694e3ec0102105d07ef2d53703970407d59c04e59d3ba0b7a1d53553", size = 11560214, upload-time = "2026-05-21T14:34:50.975Z" }, + { url = "https://files.pythonhosted.org/packages/36/38/012bf76752e1f89ed50b77b99532d90f3a3e287bc7918e1fc0948ac866ac/ruff-0.15.14-py3-none-win32.whl", hash = "sha256:6d0c1ad2a0ab718d39b6d8fd2217981ce4d625cd96a720095f798fb47d8b13e6", size = 10805548, upload-time = "2026-05-21T14:34:33.453Z" }, + { url = "https://files.pythonhosted.org/packages/d1/b7/4ea2c170f10ad760fff2a5250beb18897719dc8b52b53a24cddbb9dd3f19/ruff-0.15.14-py3-none-win_amd64.whl", hash = "sha256:802342981e056db3851a7836e5b070f8f15f67d4a685ae2a6160939d364b2902", size = 11939523, upload-time = "2026-05-21T14:34:18.077Z" }, + { url = "https://files.pythonhosted.org/packages/62/d5/bc97ff895ec35cf3925d4bd60f3b39d822f377a446906ec9bcc87405e59b/ruff-0.15.14-py3-none-win_arm64.whl", hash = "sha256:ff47b90a9ef6a40c9e2f3b479c1fb78531adf055b94c1eba0a7ba04b31951826", size = 11208607, upload-time = "2026-05-21T14:34:26.525Z" }, ] [[package]] From 46e522a8904f8d7de2dc294de8124487abdceb0f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 May 2026 21:14:42 +0000 Subject: [PATCH 085/128] Bump click from 8.3.2 to 8.4.1 Bumps [click](https://github.com/pallets/click) from 8.3.2 to 8.4.1. - [Release notes](https://github.com/pallets/click/releases) - [Changelog](https://github.com/pallets/click/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/click/compare/8.3.2...8.4.1) --- updated-dependencies: - dependency-name: click dependency-version: 8.3.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- uv.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7e7c54b5..c203524d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ authors = [ dependencies = [ "bioregistry>=0.13.57", "boto3[crt]>=1.42.55", - "click>=8.3.2", + "click>=8.4.1", "defusedxml>=0.7.1", "delta-spark>=4.1.0", "dlt[deltalake,duckdb,filesystem,iceberg,parquet]>=1.27.0", diff --git a/uv.lock b/uv.lock index a7ba1296..b775bf65 100644 --- a/uv.lock +++ b/uv.lock @@ -457,7 +457,7 @@ xml = [ requires-dist = [ { name = "bioregistry", specifier = ">=0.13.57" }, { name = "boto3", extras = ["crt"], specifier = ">=1.42.55" }, - { name = "click", specifier = ">=8.3.2" }, + { name = "click", specifier = ">=8.4.1" }, { name = "defusedxml", specifier = ">=0.7.1" }, { name = "delta-spark", specifier = ">=4.1.0" }, { name = "dlt", extras = ["deltalake", "duckdb", "filesystem", "iceberg", "parquet"], specifier = ">=1.27.0" }, @@ -645,14 +645,14 @@ wheels = [ [[package]] name = "click" -version = "8.3.2" +version = "8.4.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/57/75/31212c6bf2503fdf920d87fee5d7a86a2e3bcf444984126f13d8e4016804/click-8.3.2.tar.gz", hash = "sha256:14162b8b3b3550a7d479eafa77dfd3c38d9dc8951f6f69c78913a8f9a7540fd5", size = 302856, upload-time = "2026-04-03T19:14:45.118Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9b/98/518d8e5081007684232226f475082b30087d0f585e8457db087298259f49/click-8.4.1.tar.gz", hash = "sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96", size = 353007, upload-time = "2026-05-22T04:08:37.769Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e4/20/71885d8b97d4f3dde17b1fdb92dbd4908b00541c5a3379787137285f602e/click-8.3.2-py3-none-any.whl", hash = "sha256:1924d2c27c5653561cd2cae4548d1406039cb79b858b747cfea24924bbc1616d", size = 108379, upload-time = "2026-04-03T19:14:43.505Z" }, + { url = "https://files.pythonhosted.org/packages/c7/0d/67e5b4109ea4a837e80daa87c2c696711955e40449a97e8926672534def2/click-8.4.1-py3-none-any.whl", hash = "sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2", size = 116639, upload-time = "2026-05-22T04:08:35.26Z" }, ] [[package]] From dac873ba8b316beb228c126824972c59c283d965 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 28 May 2026 14:45:19 -0700 Subject: [PATCH 086/128] address reviewer comments --- docker-compose.yml | 2 +- notebooks/.gitignore | 1 + notebooks/ncbi_ftp_download.ipynb | 2 - notebooks/ncbi_ftp_manifest.ipynb | 10 +- notebooks/ncbi_ftp_promote.ipynb | 3 +- scripts/entrypoint.sh | 4 +- scripts/s3_local.py | 4 +- src/cdm_data_loaders/ncbi_ftp/assembly.py | 73 ++--- src/cdm_data_loaders/ncbi_ftp/manifest.py | 261 +++++++++--------- src/cdm_data_loaders/ncbi_ftp/metadata.py | 2 +- src/cdm_data_loaders/ncbi_ftp/promote.py | 38 +-- .../pipelines/ncbi_ftp_download.py | 64 ++--- src/cdm_data_loaders/utils/checksums.py | 6 +- src/cdm_data_loaders/utils/ftp_client.py | 5 +- src/cdm_data_loaders/utils/s3.py | 16 +- tests/integration/conftest.py | 8 +- tests/integration/test_download_e2e.py | 7 +- tests/integration/test_full_pipeline.py | 14 +- tests/integration/test_manifest_e2e.py | 4 +- tests/integration/test_promote_e2e.py | 9 +- tests/ncbi_ftp/conftest.py | 2 +- tests/ncbi_ftp/test_assembly.py | 5 +- tests/ncbi_ftp/test_manifest.py | 194 +++++-------- tests/ncbi_ftp/test_metadata.py | 8 +- tests/ncbi_ftp/test_notebooks.py | 6 +- tests/ncbi_ftp/test_promote.py | 15 +- tests/pipelines/test_ncbi_ftp_download.py | 24 +- tests/utils/test_s3.py | 2 +- 28 files changed, 369 insertions(+), 420 deletions(-) create mode 100644 notebooks/.gitignore diff --git a/docker-compose.yml b/docker-compose.yml index ba488cce..04cffa62 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,5 +40,5 @@ services: fi echo 'Waiting for MinIO...' && sleep 1 done - exec /app/scripts/entrypoint.sh integration-test + exec /app/scripts/entrypoint.sh integration_test command: [] diff --git a/notebooks/.gitignore b/notebooks/.gitignore new file mode 100644 index 00000000..9b1960e7 --- /dev/null +++ b/notebooks/.gitignore @@ -0,0 +1 @@ +output/ \ No newline at end of file diff --git a/notebooks/ncbi_ftp_download.ipynb b/notebooks/ncbi_ftp_download.ipynb index c8e10603..2af2c757 100644 --- a/notebooks/ncbi_ftp_download.ipynb +++ b/notebooks/ncbi_ftp_download.ipynb @@ -50,10 +50,8 @@ "source": [ "\"\"\"Imports and S3 client initialisation.\"\"\"\n", "\n", - "import json\n", "\n", "from cdm_data_loaders.pipelines.ncbi_ftp_download import (\n", - " DEFAULT_STAGING_KEY_PREFIX,\n", " download_and_stage,\n", ")" ] diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb index 49045123..19c00f2c 100644 --- a/notebooks/ncbi_ftp_manifest.ipynb +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -53,7 +53,6 @@ "\n", "from cdm_data_loaders.ncbi_ftp.assembly import FTP_HOST\n", "from cdm_data_loaders.ncbi_ftp.manifest import (\n", - " AssemblyRecord,\n", " compute_diff,\n", " download_assembly_summary,\n", " filter_by_prefix_range,\n", @@ -73,7 +72,7 @@ "metadata": {}, "outputs": [], "source": [ - "from cdm_data_loaders.utils.s3 import get_s3_client, reset_s3_client\n", + "from cdm_data_loaders.utils.s3 import reset_s3_client\n", "\n", "# Provide S3 credentials (use for local testing against MinIO test container)\n", "PROVIDE_CREDENTIALS = False # Set to False to rely on environment credentials (e.g. IAM role)\n", @@ -188,9 +187,11 @@ " print(f\"Loaded {len(previous)} assemblies from synthetic summary\")\n", " else:\n", " import time as _time\n", - " from cdm_data_loaders.ncbi_ftp.manifest import scan_store_to_synthetic_summary\n", + "\n", " from tqdm.notebook import tqdm\n", "\n", + " from cdm_data_loaders.ncbi_ftp.manifest import scan_store_to_synthetic_summary\n", + "\n", " print(f\"Scanning s3://{LAKEHOUSE_BUCKET}/{STORE_KEY_PREFIX} for existing {DATABASE} assemblies ...\")\n", " print(\"Note: large stores (500K+ assemblies) may take 15-30+ minutes.\")\n", " progress = tqdm(unit=\"assembly\", desc=\"Scanning store\", leave=True, mininterval=2.0)\n", @@ -309,9 +310,10 @@ "\n", "# -- Verify against Lakehouse --\n", "if LAKEHOUSE_BUCKET:\n", - " from cdm_data_loaders.ncbi_ftp.manifest import verify_transfer_candidates\n", " from tqdm.notebook import tqdm\n", "\n", + " from cdm_data_loaders.ncbi_ftp.manifest import verify_transfer_candidates\n", + "\n", " candidates = diff.new + diff.updated\n", " total = len(candidates)\n", " print(f\"Verifying {total} candidates against s3://{LAKEHOUSE_BUCKET}/{STORE_KEY_PREFIX} ...\")\n", diff --git a/notebooks/ncbi_ftp_promote.ipynb b/notebooks/ncbi_ftp_promote.ipynb index 6e16e1a4..7af5ba0f 100644 --- a/notebooks/ncbi_ftp_promote.ipynb +++ b/notebooks/ncbi_ftp_promote.ipynb @@ -131,7 +131,7 @@ "metadata": {}, "outputs": [], "source": [ - "from cdm_data_loaders.utils.s3 import get_s3_client, reset_s3_client\n", + "from cdm_data_loaders.utils.s3 import reset_s3_client\n", "\n", "# Provide S3 credentials (use for local testing against MinIO test container)\n", "PROVIDE_CREDENTIALS = False # Set to False to rely on environment credentials (e.g. IAM role)\n", @@ -273,7 +273,6 @@ "source": [ "\"\"\"Inspect frictionless descriptors written to metadata/.\"\"\"\n", "\n", - "from cdm_data_loaders.ncbi_ftp.metadata import build_descriptor_key\n", "\n", "s3 = get_s3_client()\n", "paginator = s3.get_paginator(\"list_objects_v2\")\n", diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index c1c0b6d3..c8bf9606 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -VALID_COMMANDS=(all_the_bacteria ncbi_ftp_sync ncbi_rest_api uniprot uniref xml_split test integration-test bash) +VALID_COMMANDS=(all_the_bacteria ncbi_ftp_sync ncbi_rest_api uniprot uniref xml_split test integration_test bash) usage() { local joined @@ -40,7 +40,7 @@ case "$cmd" in test) exec /usr/bin/tini -- uv run --no-sync pytest -m "not requires_spark" ;; - integration-test) + integration_test) # run the integration tests (requires a running MinIO instance) exec /usr/bin/tini -- uv run --no-sync pytest -m "integration" -v "$@" ;; diff --git a/scripts/s3_local.py b/scripts/s3_local.py index 60bac49f..b8a27612 100755 --- a/scripts/s3_local.py +++ b/scripts/s3_local.py @@ -42,7 +42,7 @@ def _split(uri: str) -> tuple[str, str]: return parts[0], parts[1] if len(parts) > 1 else "" -# ── subcommands ───────────────────────────────────────────────────────── +# subcommands def cmd_mb(args: list[str]) -> None: @@ -110,7 +110,7 @@ def cmd_head(args: list[str]) -> None: print(json.dumps(meta, indent=2)) -# ── dispatch ──────────────────────────────────────────────────────────── +# dispatch COMMANDS = {"mb": cmd_mb, "cp": cmd_cp, "ls": cmd_ls, "head": cmd_head} diff --git a/src/cdm_data_loaders/ncbi_ftp/assembly.py b/src/cdm_data_loaders/ncbi_ftp/assembly.py index b424702b..827ebb31 100644 --- a/src/cdm_data_loaders/ncbi_ftp/assembly.py +++ b/src/cdm_data_loaders/ncbi_ftp/assembly.py @@ -33,6 +33,17 @@ "_normalized_gene_expression_counts.txt.gz", ] +# Pre-compile regex patterns for performance + +# Extracts database (GCF or GCA) and 3-digit prefixes from an assembly directory name +# (e.g. "GCF_000001215.4" → ("GCF", "000", "001", "215")) +ASSEMBLY_DIR_REGEX = re.compile(r"(GC[AF])_(\d{3})(\d{3})(\d{3})\.\d+.*") + +# Extracts database, full assembly directory, and accession from an FTP path +# (e.g. "/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/" +# → ("GCF", "GCF_000001215.4_Release_6_plus_ISO1_MT", "GCF_000001215.4")) +ASSEMBLY_PATH_REGEX = re.compile(r"/(GC[AF])/\d{3}/\d{3}/\d{3}/((GC[AF]_\d{9}\.\d+)_[^/]+)/?$") + def parse_md5_checksums_file(text: str) -> dict[str, str]: """Parse an NCBI ``md5checksums.txt`` file into a filename-to-hash mapping. @@ -44,17 +55,14 @@ def parse_md5_checksums_file(text: str) -> dict[str, str]: """ checksums: dict[str, str] = {} for raw_line in text.strip().splitlines(): - stripped = raw_line.strip() - if not stripped: - continue - parts = stripped.split(" ", maxsplit=1) + parts = raw_line.strip().split(" ", maxsplit=1) if len(parts) == 2: # noqa: PLR2004 md5_hash, filename = parts checksums[filename.removeprefix("./")] = md5_hash.strip() return checksums -# ── Path helpers ───────────────────────────────────────────────────────── +# Path helpers def build_accession_path(assembly_dir: str) -> str: @@ -66,12 +74,12 @@ def build_accession_path(assembly_dir: str) -> str: :return: relative path string :raises ValueError: if the assembly directory name cannot be parsed """ - m = re.match(r"GC[AF]_(\d{3})(\d{3})(\d{3})\.\d+.*", assembly_dir) + m = ASSEMBLY_DIR_REGEX.match(assembly_dir) if not m: msg = f"Cannot parse accession: {assembly_dir}" raise ValueError(msg) - p1, p2, p3 = m.groups() - return f"raw_data/{assembly_dir[:3]}/{p1}/{p2}/{p3}/{assembly_dir}/" + db, p1, p2, p3 = m.groups() + return f"raw_data/{db}/{p1}/{p2}/{p3}/{assembly_dir}/" def parse_assembly_path(assembly_path: str) -> tuple[str, str, str]: @@ -81,17 +89,13 @@ def parse_assembly_path(assembly_path: str) -> tuple[str, str, str]: :return: tuple of ``(database, assembly_dir, accession)`` :raises ValueError: if the path cannot be parsed """ - m = re.search( - r"/(GC[AF])/\d{3}/\d{3}/\d{3}/((GC[AF]_\d{9}\.\d+)_[^/]+)/?$", - assembly_path.rstrip("/"), - ) - if not m: - msg = f"Cannot parse assembly path: {assembly_path}" - raise ValueError(msg) - return m.group(1), m.group(2), m.group(3) + if m := ASSEMBLY_PATH_REGEX.search(assembly_path.rstrip("/")): + return m.group(1), m.group(2), m.group(3) + msg = f"Cannot parse assembly path: {assembly_path}" + raise ValueError(msg) -# ── Single assembly download ──────────────────────────────────────────── +# Single assembly download def _download_and_verify( # noqa: PLR0913 @@ -107,36 +111,39 @@ def _download_and_verify( # noqa: PLR0913 local_file = dest_dir / filename expected_md5 = md5_checksums.get(filename) - for attempt in range(1, 4): - logger.debug(" Downloading %s (attempt %d/3)", filename, attempt) - with local_file.open("wb") as f: - ftp.retrbinary(f"RETR {filename}", f.write) - last_activity = time.monotonic() - + # Define a helper function to validate the MD5 checksum of the downloaded file if one exists. + # Returns False if file checksums mismatch, True otherwise + def validate_file(local_file: Path) -> bool: if expected_md5: actual_md5 = compute_md5(str(local_file)) - if actual_md5 != expected_md5: + if actual_md5 == expected_md5: + (dest_dir / f"{filename}.md5").write_text(expected_md5) + else: logger.warning( " MD5 mismatch for %s: expected %s, got %s", filename, expected_md5, actual_md5, ) - if attempt < 3: # noqa: PLR2004 - continue - stats["files_skipped_checksum_mismatch"] += 1 - local_file.unlink(missing_ok=True) - return last_activity + return False logger.debug(" MD5 verified: %s", filename) else: stats["files_without_checksum"] += 1 + return True - if expected_md5: - (dest_dir / f"{filename}.md5").write_text(expected_md5) + # Try 3 times to download and validate the file; if it fails, delete any partial file and return + for attempt in range(1, 4): + logger.debug(" Downloading %s (attempt %d/3)", filename, attempt) + with local_file.open("wb") as f: + ftp.retrbinary(f"RETR {filename}", f.write) + last_activity = time.monotonic() - stats["files_downloaded"] += 1 - return last_activity + if validate_file(local_file): + stats["files_downloaded"] += 1 + return last_activity + stats["files_skipped_checksum_mismatch"] += 1 + local_file.unlink(missing_ok=True) return last_activity diff --git a/src/cdm_data_loaders/ncbi_ftp/manifest.py b/src/cdm_data_loaders/ncbi_ftp/manifest.py index b14d2923..653f00ae 100644 --- a/src/cdm_data_loaders/ncbi_ftp/manifest.py +++ b/src/cdm_data_loaders/ncbi_ftp/manifest.py @@ -25,18 +25,32 @@ parse_md5_checksums_file, ) from cdm_data_loaders.utils.cdm_logger import get_cdm_logger -from cdm_data_loaders.utils.ftp_client import connect_ftp, ftp_noop_keepalive, ftp_retrieve_text -from cdm_data_loaders.utils.s3 import get_s3_client, head_object +from cdm_data_loaders.utils.ftp_client import FTP, connect_ftp, ftp_noop_keepalive, ftp_retrieve_text +from cdm_data_loaders.utils.s3 import get_s3_client, head_object, list_matching_objects logger = get_cdm_logger() +_DATABASE_ACC_PREFIX: dict[str, str] = { + "refseq": "GCF_", + "genbank": "GCA_", +} + SUMMARY_FTP_PATHS: dict[str, str] = { "refseq": "/genomes/ASSEMBLY_REPORTS/assembly_summary_refseq.txt", "genbank": "/genomes/ASSEMBLY_REPORTS/assembly_summary_genbank.txt", } +# Pre-compile regex patterns for performance + +# Extracts the full directory name and accession id from an S3 key +# (e.g. "GCF_000001215.4_Release_6_plus_ISO1_MT" and "GCF_000001215.4" +# from "some/prefix/GCF_000001215.4_Release_6_plus_ISO1_MT/") +ACCESSION_REGEX = re.compile(r"((GC[AF]_\d{9}\.\d+)[^/]*)/") + +# Extracts the 3-digit prefix from an accession (e.g. GCF_000001215.4 → "000") +ACCESSION_PREFIX_REGEX = re.compile(r"GC[AF]_(\d{3})\d{6}\.\d+") -# ── Data structures ───────────────────────────────────────────────────── +# Data structures @dataclass @@ -60,7 +74,7 @@ class DiffResult: suppressed: list[str] = field(default_factory=list) -# ── Assembly summary download & parsing ────────────────────────────────── +# Assembly summary download & parsing def download_assembly_summary(database: str = "refseq", ftp_host: str = FTP_HOST) -> str: @@ -151,7 +165,7 @@ def get_latest_assembly_paths(assemblies: dict[str, AssemblyRecord], ftp_host: s return paths -# ── Prefix filtering ──────────────────────────────────────────────────── +# Prefix filtering def accession_prefix(accession: str) -> str | None: @@ -189,10 +203,10 @@ def filter_by_prefix_range( return filtered -# ── Diff computation ──────────────────────────────────────────────────── +# Diff computation -def compute_diff( # noqa: PLR0912 +def compute_diff( current: dict[str, AssemblyRecord], previous_assemblies: dict[str, AssemblyRecord] | None = None, previous_accessions: set[str] | None = None, @@ -206,12 +220,7 @@ def compute_diff( # noqa: PLR0912 """ diff = DiffResult() - if previous_assemblies is not None: - known = set(previous_assemblies.keys()) - elif previous_accessions is not None: - known = previous_accessions - else: - known = set() + known = set(previous_assemblies) if previous_assemblies is not None else (previous_accessions or set()) for acc, rec in current.items(): if rec.status == "replaced": @@ -233,10 +242,8 @@ def compute_diff( # noqa: PLR0912 diff.updated.append(acc) # Accessions in previous but entirely absent from current (withdrawn) - current_accs = set(current.keys()) - for acc in known: - if acc not in current_accs and acc not in diff.suppressed: - diff.suppressed.append(acc) + current_accs = set(current) + diff.suppressed.extend(known - current_accs) diff.new.sort() diff.updated.sort() @@ -245,7 +252,7 @@ def compute_diff( # noqa: PLR0912 return diff -# ── FTP URL helpers ────────────────────────────────────────────────────── +# FTP URL helpers def _ftp_dir_from_url(ftp_url: str, ftp_host: str = FTP_HOST) -> str: @@ -257,39 +264,17 @@ def _ftp_dir_from_url(ftp_url: str, ftp_host: str = FTP_HOST) -> str: return ftp_url -# ── Synthetic summary from S3 store scan ──────────────────────────────── - - -def _extract_accession_from_s3_key(key: str) -> str | None: - """Extract the assembly accession from an S3 object key. - - Looks for the pattern GCF_######.# or GCA_######.# in the key path. - - :param key: S3 object key - :return: accession (e.g. "GCF_000001215.4") or None if not found - """ - m = re.search(r"(GC[AF]_\d{3}\d{6}\.\d+)", key) - return m.group(1) if m else None - +# Synthetic summary from S3 store scan -def _extract_assembly_dir_from_s3_key(key: str) -> str | None: - """Extract the assembly directory name from an S3 object key. - The assembly directory is the path component that follows the accession - and contains assembly metadata (e.g. "GCF_000001215.4_Release_6_plus_ISO1_MT"). +def _extract_accession_dir_and_id_from_s3_key(key: str) -> tuple[str | None, str | None]: + """Extract both accession and assembly directory from an S3 object key. - :param key: S3 object key - :return: assembly directory name or None if not found + e.g. "some/prefix/GCF_000001215.4_Release_6_plus_ISO1_MT/file.gz" + → ("GCF_000001215.4_Release_6_plus_ISO1_MT", "GCF_000001215.4") """ - # Match accession followed by underscore and then capture until next / - m = re.search(r"(GC[AF]_\d{3}\d{6}\.\d+[^/]*)/", key) - return m.group(1) if m else None - - -_DATABASE_ACC_PREFIX: dict[str, str] = { - "refseq": "GCF_", - "genbank": "GCA_", -} + m = ACCESSION_REGEX.search(key) + return (m.group(1), m.group(2)) if m else (None, None) def scan_store_to_synthetic_summary( @@ -326,7 +311,7 @@ def scan_store_to_synthetic_summary( :return: dict mapping accession to ``AssemblyRecord`` """ try: - datetime.strptime(release_date, "%Y/%m/%d") + datetime.strptime(release_date, "%Y/%m/%d").astimezone(UTC) except ValueError as exc: msg = f"Invalid release_date '{release_date}'. Expected format YYYY/MM/DD." raise ValueError(msg) from exc @@ -336,54 +321,105 @@ def scan_store_to_synthetic_summary( msg = f"Unknown database: {database!r}. Expected 'refseq' or 'genbank'." raise ValueError(msg) - s3 = get_s3_client() assemblies: dict[str, AssemblyRecord] = {} processed_count = 0 try: - paginator = s3.get_paginator("list_objects_v2") - pages = paginator.paginate(Bucket=bucket, Prefix=key_prefix) - - for page in pages: - for obj in page.get("Contents", []): - acc = _extract_accession_from_s3_key(obj["Key"]) - if not acc or not acc.startswith(acc_prefix): - continue - assembly_dir = _extract_assembly_dir_from_s3_key(obj["Key"]) - - if not acc or not assembly_dir: - continue - - if acc not in assemblies: - # First object for this accession; store it. - # Construct a fake FTP path that ends with assembly_dir so - # that round-tripping through parse_assembly_summary (which - # derives assembly_dir via ftp_path.rstrip("/").split("/")[-1]) - # yields the correct assembly_dir and therefore correct diffs. - fake_ftp_path = f"https://ftp.ncbi.nlm.nih.gov/synthetic/{assembly_dir}" - assemblies[acc] = AssemblyRecord( - accession=acc, - status="latest", - seq_rel_date=release_date, - ftp_path=fake_ftp_path, - assembly_dir=assembly_dir, - ) - processed_count += 1 - if progress_callback is not None: - progress_callback(processed_count, acc) - - except Exception as e: # noqa: BLE001 - logger.error("Error scanning store: %s", e) + objs = list_matching_objects(f"{bucket}/{key_prefix}") + + for obj in objs: + assembly_dir, acc = _extract_accession_dir_and_id_from_s3_key(obj["Key"]) + if not acc or not acc.startswith(acc_prefix): + continue + + if not assembly_dir: + continue + + if acc not in assemblies: + # First object for this accession; store it. + # Construct a fake FTP path that ends with assembly_dir so + # that round-tripping through parse_assembly_summary (which + # derives assembly_dir via ftp_path.rstrip("/").split("/")[-1]) + # yields the correct assembly_dir and therefore correct diffs. + fake_ftp_path = f"https://ftp.ncbi.nlm.nih.gov/synthetic/{assembly_dir}" + assemblies[acc] = AssemblyRecord( + accession=acc, + status="latest", + seq_rel_date=release_date, + ftp_path=fake_ftp_path, + assembly_dir=assembly_dir, + ) + processed_count += 1 + if progress_callback is not None: + progress_callback(processed_count, acc) + + except Exception: + logger.exception("Error scanning store") raise logger.info("Scanned S3 store: found %d unique assemblies", len(assemblies)) return assemblies -# ── Checksum verification against S3 store ─────────────────────────────── +# Checksum verification against S3 store + + +def _fetch_accession_checksums_from_ftp( + ftp: FTP | None, ftp_host: str, last_activity: float, current_accession: str +) -> dict[str, str]: + """Fetch and parse the md5checksums.txt file for a given accession from FTP. + + :param ftp: FTP connection object + :param ftp_host: NCBI FTP hostname + :param last_activity: timestamp of the last FTP activity + :param current_accession: accession identifier + :return: dictionary mapping file names to MD5 checksums + """ + if ftp is None: + ftp = connect_ftp(ftp_host) + last_activity = ftp_noop_keepalive(ftp, last_activity) + ftp_dir = _ftp_dir_from_url(current_accession, ftp_host) + try: + md5_text = ftp_retrieve_text(ftp, ftp_dir.rstrip("/") + "/md5checksums.txt") + last_activity = time.monotonic() + ftp_checksums = parse_md5_checksums_file(md5_text) + except Exception: # noqa: BLE001 + logger.warning("Cannot fetch md5checksums.txt for %s, keeping in transfer list", current_accession) + return {} + return { + fname: md5 for fname, md5 in ftp_checksums.items() if any(fname.endswith(suffix) for suffix in FILE_FILTERS) + } + + +def _does_accession_need_update( + target_checksums: dict[str, str], + bucket: str, + s3_prefix: str, +) -> bool: + """Check if any file for an accession needs updating by comparing FTP checksums to S3 metadata. + + :param target_checksums: dict mapping file names to expected MD5 checksums from FTP + :param bucket: S3 bucket name + :param s3_prefix: S3 key prefix for the assembly in question + :return: True if any file is missing or has a checksum mismatch, False otherwise + """ + for fname, expected_md5 in target_checksums.items(): + s3_path = f"{bucket}/{s3_prefix}{fname}" + obj_info = head_object(s3_path) + + if obj_info is None: + logger.debug("File missing from store: %s", s3_path) + return True + + s3_md5 = obj_info["metadata"].get("md5", "") + if s3_md5 != expected_md5: + logger.debug("MD5 mismatch for %s: S3=%s FTP=%s", s3_path, s3_md5, expected_md5) + return True + + return False -def verify_transfer_candidates( # noqa: PLR0912, PLR0915 +def verify_transfer_candidates( # noqa: PLR0913 accessions: list[str], current_assemblies: dict[str, AssemblyRecord], bucket: str, @@ -421,13 +457,16 @@ def verify_transfer_candidates( # noqa: PLR0912, PLR0915 skipped_missing = 0 last_activity = time.monotonic() + def _progress(done: int, total: int, acc: str) -> None: + if progress_callback is not None: + progress_callback(done, total, acc) + try: for done, acc in enumerate(accessions, start=1): rec = current_assemblies.get(acc) if not rec: confirmed.append(acc) - if progress_callback is not None: - progress_callback(done, len(accessions), acc) + _progress(done, len(accessions), acc) continue # Build S3 prefix for this assembly @@ -440,56 +479,19 @@ def verify_transfer_candidates( # noqa: PLR0912, PLR0915 # Nothing in the store — definitely needs downloading confirmed.append(acc) skipped_missing += 1 - if progress_callback is not None: - progress_callback(done, len(accessions), acc) + _progress(done, len(accessions), acc) continue # Objects exist — need FTP md5 checksums to decide - if ftp is None: - ftp = connect_ftp(ftp_host) - - last_activity = ftp_noop_keepalive(ftp, last_activity) - - ftp_dir = _ftp_dir_from_url(rec.ftp_path, ftp_host) - try: - md5_text = ftp_retrieve_text(ftp, ftp_dir.rstrip("/") + "/md5checksums.txt") - last_activity = time.monotonic() - ftp_checksums = parse_md5_checksums_file(md5_text) - except Exception: # noqa: BLE001 - logger.warning("Cannot fetch md5checksums.txt for %s, keeping in transfer list", acc) - confirmed.append(acc) - if progress_callback is not None: - progress_callback(done, len(accessions), acc) - continue - - # Filter to files we'd actually download - target_checksums = { - fname: md5 - for fname, md5 in ftp_checksums.items() - if any(fname.endswith(suffix) for suffix in FILE_FILTERS) - } + target_checksums = _fetch_accession_checksums_from_ftp(ftp, ftp_host, last_activity, rec.ftp_path) if not target_checksums: confirmed.append(acc) - if progress_callback is not None: - progress_callback(done, len(accessions), acc) + _progress(done, len(accessions), acc) continue # Short-circuit: if any file differs or is missing, keep the assembly - needs_update = False - for fname, expected_md5 in target_checksums.items(): - s3_path = f"{bucket}/{s3_prefix}{fname}" - obj_info = head_object(s3_path) - - if obj_info is None: - needs_update = True - break - - s3_md5 = obj_info["metadata"].get("md5", "") - if s3_md5 != expected_md5: - logger.debug("MD5 mismatch for %s/%s: S3=%s FTP=%s", acc, fname, s3_md5, expected_md5) - needs_update = True - break + needs_update = _does_accession_need_update(target_checksums, bucket, s3_prefix) if needs_update: confirmed.append(acc) @@ -497,8 +499,7 @@ def verify_transfer_candidates( # noqa: PLR0912, PLR0915 pruned += 1 logger.debug("Pruned %s — all files match S3 checksums", acc) - if progress_callback is not None: - progress_callback(done, len(accessions), acc) + _progress(done, len(accessions), acc) finally: if ftp is not None: with contextlib.suppress(Exception): @@ -514,7 +515,7 @@ def verify_transfer_candidates( # noqa: PLR0912, PLR0915 return confirmed -# ── Manifest writing ──────────────────────────────────────────────────── +# Manifest writing def write_transfer_manifest( diff --git a/src/cdm_data_loaders/ncbi_ftp/metadata.py b/src/cdm_data_loaders/ncbi_ftp/metadata.py index b5b6175e..35f15d2a 100644 --- a/src/cdm_data_loaders/ncbi_ftp/metadata.py +++ b/src/cdm_data_loaders/ncbi_ftp/metadata.py @@ -53,7 +53,7 @@ class DescriptorResource(TypedDict, total=False): hash: str | None -# ── Public helpers ──────────────────────────────────────────────────────── +# Public helpers def build_descriptor_key(assembly_dir: str, key_prefix: str) -> str: diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index 4fbe6707..a97cba2a 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -29,6 +29,7 @@ copy_object, delete_objects, get_s3_client, + list_matching_objects, object_exists, upload_file, ) @@ -37,8 +38,10 @@ DEFAULT_LAKEHOUSE_KEY_PREFIX = "tenant-general-warehouse/kbase/datasets/ncbi/" +_MAX_DRY_RUN_LOGS = 10 -# ── Promote from S3 staging prefix ────────────────────────────────────── + +# Promote from S3 staging prefix def promote_from_s3( # noqa: PLR0913 @@ -69,18 +72,13 @@ def promote_from_s3( # noqa: PLR0913 :param dry_run: if True, log actions without side effects :return: report dict with counts """ - s3 = get_s3_client() - paginator = s3.get_paginator("list_objects_v2") + # Get list of objects under the staging prefix normalized_staging_key_prefix = staging_key_prefix.rstrip("/") + "/" - - # Collect all objects under the staging prefix - staged_objects: list[str] = [] - for page in paginator.paginate(Bucket=staging_bucket, Prefix=normalized_staging_key_prefix): - staged_objects.extend(obj["Key"] for obj in page.get("Contents", [])) + staged_objects: list[dict[str, Any]] = list_matching_objects(f"{staging_bucket}/{normalized_staging_key_prefix}") # Separate data files from sidecars - sidecars = {k for k in staged_objects if k.endswith((".crc64nvme", ".md5"))} - data_files = [k for k in staged_objects if k not in sidecars] + sidecars = {k["Key"] for k in staged_objects if k["Key"].endswith((".crc64nvme", ".md5"))} + data_files = [k["Key"] for k in staged_objects if k["Key"] not in sidecars] logger.info("Found %d data files and %d sidecars in staging", len(data_files), len(sidecars)) @@ -136,7 +134,7 @@ def promote_from_s3( # noqa: PLR0913 return report -# ── Promote data files (per-file loop) ────────────────────────────────── +# Promote data files (per-file loop) def _promote_data_files( # noqa: PLR0913, PLR0915 @@ -234,7 +232,7 @@ def _promote_one(staged_key: str) -> tuple[DescriptorResource, str]: for staged_key in files: rel_path = staged_key[len(normalized_staging_prefix) :] final_key = lakehouse_key_prefix + rel_path - if _dry_run_log_count < 10: + if _dry_run_log_count < _MAX_DRY_RUN_LOGS: logger.info("[dry-run] would promote: %s -> %s", staged_key, final_key) else: logger.debug("[dry-run] would promote: %s -> %s", staged_key, final_key) @@ -280,10 +278,12 @@ def _promote_one(staged_key: str) -> tuple[DescriptorResource, str]: # Batch-delete all staged data files and their sidecars in one API call keys_to_delete = list(promoted_keys) - for key in promoted_keys: - for sidecar_ext in (".md5", ".crc64nvme"): - if key + sidecar_ext in sidecars: - keys_to_delete.append(key + sidecar_ext) + keys_to_delete.extend( + key + sidecar_ext + for key in promoted_keys + for sidecar_ext in (".md5", ".crc64nvme") + if key + sidecar_ext in sidecars + ) del_errors = delete_objects(staging_bucket, keys_to_delete) for err in del_errors: logger.warning("Failed to delete staged file %s: %s", err.get("Key"), err.get("Message")) @@ -291,7 +291,7 @@ def _promote_one(staged_key: str) -> tuple[DescriptorResource, str]: return promoted, failed, descriptors_written, promoted_accessions -# ── Archive assemblies ────────────────────────────────────────────────── +# Archive assemblies def _archive_assemblies( # noqa: PLR0913 @@ -365,7 +365,7 @@ def _archive_assemblies( # noqa: PLR0913 if dry_run: for source_key, archive_key in key_pairs: - if _dry_run_log_count < 10: + if _dry_run_log_count < _MAX_DRY_RUN_LOGS: logger.info("[dry-run] would archive: %s -> %s", source_key, archive_key) else: logger.debug("[dry-run] would archive: %s -> %s", source_key, archive_key) @@ -422,7 +422,7 @@ def _archive_assemblies( # noqa: PLR0913 return archived -# ── Manifest trimming ─────────────────────────────────────────────────── +# Manifest trimming def _trim_manifest(manifest_s3_key: str, staging_bucket: str, promoted_accessions: set[str]) -> None: diff --git a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py index 9eff6af3..52e42891 100644 --- a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py +++ b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py @@ -18,8 +18,8 @@ import tqdm from pydantic import AliasChoices, Field -from tenacity import before_sleep_log, retry, retry_if_exception_type, stop_after_attempt, wait_exponential from pydantic_settings import BaseSettings, SettingsConfigDict +from tenacity import before_sleep_log, retry, retry_if_exception_type, stop_after_attempt, wait_exponential from cdm_data_loaders.ncbi_ftp.assembly import ( FTP_HOST, @@ -36,8 +36,6 @@ logger = get_cdm_logger() -# ── Constants ──────────────────────────────────────────────────────────── - DEFAULT_STAGING_KEY_PREFIX = "staging/" @@ -76,7 +74,7 @@ class DownloadSettings(BaseSettings): ) -# ── Private helpers ───────────────────────────────────────────────────── +# Private helpers def _upload_assembly_dir( @@ -113,7 +111,7 @@ def _upload_assembly_dir( return count -# ── Batch download ─────────────────────────────────────────────────────── +# Batch download def download_batch( @@ -170,18 +168,20 @@ def _attempt() -> dict[str, Any]: return path, None try: - with tqdm.tqdm( - total=len(assembly_paths), unit="assembly", desc="Downloading from NCBI FTP", smoothing=0.01 - ) as pbar: - with ThreadPoolExecutor(max_workers=threads) as executor: - futures = {executor.submit(_download_one, p): p for p in assembly_paths} - for future in as_completed(futures): - path, error = future.result() - if error: - logger.error("FAILED: %s: %s", path, error) - with lock: - failed.append({"path": path, "error": str(error)}) - pbar.update(1) + with ( + tqdm.tqdm( + total=len(assembly_paths), unit="assembly", desc="Downloading from NCBI FTP", smoothing=0.01 + ) as pbar, + ThreadPoolExecutor(max_workers=threads) as executor, + ): + futures = {executor.submit(_download_one, p): p for p in assembly_paths} + for future in as_completed(futures): + path, error = future.result() + if error: + logger.error("FAILED: %s: %s", path, error) + with lock: + failed.append({"path": path, "error": str(error)}) + pbar.update(1) finally: pool.close_all() @@ -210,7 +210,7 @@ def _attempt() -> dict[str, Any]: return report -# ── CTS entry point ───────────────────────────────────────────────────── +# CTS entry point def run_download(config: DownloadSettings) -> None: @@ -235,7 +235,7 @@ def cli() -> None: run_cli(DownloadSettings, run_download) -# ── Notebook / interactive entry point ────────────────────────────────── +# Notebook / interactive entry point def download_and_stage( @@ -334,18 +334,20 @@ def _attempt() -> dict[str, Any]: return path, None try: - with tqdm.tqdm( - total=len(assembly_paths), unit="assembly", desc="Downloading & staging", smoothing=0.01 - ) as pbar: - with ThreadPoolExecutor(max_workers=threads) as executor: - futures = {executor.submit(_download_upload_one, p): p for p in assembly_paths} - for future in as_completed(futures): - path, error = future.result() - if error: - logger.error("FAILED: %s: %s", path, error) - with lock: - failed.append({"path": path, "error": str(error)}) - pbar.update(1) + with ( + tqdm.tqdm( + total=len(assembly_paths), unit="assembly", desc="Downloading & staging", smoothing=0.01 + ) as pbar, + ThreadPoolExecutor(max_workers=threads) as executor, + ): + futures = {executor.submit(_download_upload_one, p): p for p in assembly_paths} + for future in as_completed(futures): + path, error = future.result() + if error: + logger.error("FAILED: %s: %s", path, error) + with lock: + failed.append({"path": path, "error": str(error)}) + pbar.update(1) finally: pool.close_all() diff --git a/src/cdm_data_loaders/utils/checksums.py b/src/cdm_data_loaders/utils/checksums.py index 021098a6..8d0bf391 100644 --- a/src/cdm_data_loaders/utils/checksums.py +++ b/src/cdm_data_loaders/utils/checksums.py @@ -9,9 +9,7 @@ import hashlib from pathlib import Path -from cdm_data_loaders.utils.cdm_logger import get_cdm_logger - -logger = get_cdm_logger() +from awscrt.checksums import crc64nvme as _crc64nvme def compute_md5(file_path: str | Path) -> str: @@ -46,8 +44,6 @@ def compute_crc64nvme(file_path: str | Path) -> str: :param file_path: path to the file :return: base64-encoded CRC64/NVME checksum """ - from awscrt.checksums import crc64nvme as _crc64nvme # noqa: PLC0415 - crc = 0 with Path(file_path).open("rb") as f: for chunk in iter(lambda: f.read(1 << 20), b""): diff --git a/src/cdm_data_loaders/utils/ftp_client.py b/src/cdm_data_loaders/utils/ftp_client.py index bd372924..8bf5f1f7 100644 --- a/src/cdm_data_loaders/utils/ftp_client.py +++ b/src/cdm_data_loaders/utils/ftp_client.py @@ -168,9 +168,8 @@ def get(self) -> FTP: # Connection is stale — discard it and reconnect below with contextlib.suppress(Exception): ftp.quit() - with self._lock: - with contextlib.suppress(ValueError): - self._connections.remove(ftp) + with self._lock, contextlib.suppress(ValueError): + self._connections.remove(ftp) ftp = None self._local.ftp = None if ftp is None: diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index 738309e8..bf285ed3 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -66,7 +66,7 @@ def get_s3_client(args: dict[str, str] | None = None) -> botocore.client.BaseCli "aws_access_key_id": settings.MINIO_ACCESS_KEY, "aws_secret_access_key": settings.MINIO_SECRET_KEY, } - except (ModuleNotFoundError, ImportError, NameError) as e: + except (ModuleNotFoundError, ImportError, NameError): logger.exception("Failed to load berdl settings") raise @@ -205,8 +205,8 @@ def upload_file( ) -> bool: """Upload an object to an S3 bucket. - When *metadata* is supplied the file is always uploaded (no existence check) - and the dict is attached as S3 user metadata. When *metadata* is ``None`` + When *tags* is supplied the file is always uploaded (no existence check) + and the dict is attached as S3 user metadata. When *tags* is ``None`` (the default) the existing behaviour is preserved: the upload is skipped if the object is already present. @@ -216,8 +216,10 @@ def upload_file( :type destination_dir: str :param object_name: S3 object name. If not specified, the name of the file from local_file_path is used. :type object_name: str | None - :param metadata: user metadata key/value pairs to attach to the object; when provided the upload always runs - :type metadata: dict[str, str] | None + :param tags: user metadata key/value pairs to attach to the object; when provided the upload always runs + :type tags: dict[str, str] | None + :param show_progress: whether to display a tqdm progress bar during upload, defaults to True + :type show_progress: bool, optional :return: True if file was uploaded, else False :rtype: bool """ @@ -262,7 +264,7 @@ def upload_file( ExtraArgs=extra_args, ) except Exception as e: # noqa: BLE001 - logger.exception("Error uploading to s3") + logger.exception(f"Error uploading to s3: {e}") return False return True @@ -466,6 +468,7 @@ def copy_object( CopySource={"Bucket": current_s3_bucket, "Key": current_s3_key}, Bucket=new_s3_bucket, Key=new_s3_key, + **DEFAULT_EXTRA_ARGS, ) @@ -518,6 +521,7 @@ def copy_directory(current_s3_path: str, new_s3_path: str) -> tuple[dict[str, st CopySource={"Bucket": current_s3_bucket, "Key": current_key}, Bucket=new_s3_bucket, Key=new_key, + **DEFAULT_EXTRA_ARGS, ) if resp["ResponseMetadata"]["HTTPStatusCode"] == SUCCESS_RESPONSE: successes[source_path] = dest_path diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b8bdf9ba..3f44b6c3 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -24,7 +24,7 @@ from cdm_data_loaders.ncbi_ftp.assembly import build_accession_path from cdm_data_loaders.utils.s3 import reset_s3_client -# ── MinIO connection defaults ─────────────────────────────────────────── +# MinIO connection defaults MINIO_ENDPOINT_URL = os.environ["MINIO_ENDPOINT_URL"] MINIO_ACCESS_KEY = os.environ["MINIO_ACCESS_KEY"] @@ -34,7 +34,7 @@ _MAX_BUCKET_LEN = 63 -# ── MinIO reachability check ──────────────────────────────────────────── +# MinIO reachability check _minio_available: bool | None = None @@ -72,7 +72,7 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item item.add_marker(skip_marker) -# ── Fixtures ──────────────────────────────────────────────────────────── +# Fixtures @pytest.fixture @@ -177,7 +177,7 @@ def staging_test_bucket(minio_s3_client: botocore.client.BaseClient, request: py return bucket -# ── Helpers ───────────────────────────────────────────────────────────── +# Helpers def stage_files_to_minio( diff --git a/tests/integration/test_download_e2e.py b/tests/integration/test_download_e2e.py index 37456e3e..469c65b6 100644 --- a/tests/integration/test_download_e2e.py +++ b/tests/integration/test_download_e2e.py @@ -6,12 +6,11 @@ """ import json - -import pytest - from pathlib import Path from unittest.mock import patch +import pytest + import cdm_data_loaders.utils.s3 as s3_utils from cdm_data_loaders.ncbi_ftp.manifest import ( compute_diff, @@ -20,7 +19,7 @@ parse_assembly_summary, write_transfer_manifest, ) -from cdm_data_loaders.pipelines.ncbi_ftp_download import download_batch, download_and_stage +from cdm_data_loaders.pipelines.ncbi_ftp_download import download_and_stage, download_batch # Use same stable prefix as manifest tests STABLE_PREFIX = "900" diff --git a/tests/integration/test_full_pipeline.py b/tests/integration/test_full_pipeline.py index 63776ae5..9c957df0 100644 --- a/tests/integration/test_full_pipeline.py +++ b/tests/integration/test_full_pipeline.py @@ -52,7 +52,7 @@ def test_full_pipeline_small_batch( """Single assembly flows through all three phases into MinIO.""" s3 = minio_s3_client - # ── Phase 1: Manifest generation ──────────────────────────────── + # Step 1: Manifest generation raw = download_assembly_summary(database="refseq") full = parse_assembly_summary(raw) filtered = filter_by_prefix_range(full, prefix_from=STABLE_PREFIX, prefix_to=STABLE_PREFIX) @@ -63,7 +63,7 @@ def test_full_pipeline_small_batch( manifest_path = tmp_path / "transfer_manifest.txt" _ = write_transfer_manifest(diff, filtered, manifest_path) - # ── Phase 2: Download one assembly from real FTP ──────────────── + # Step 2: Download one assembly from real FTP output_dir = tmp_path / "output" output_dir.mkdir() @@ -76,11 +76,11 @@ def test_full_pipeline_small_batch( assert report["succeeded"] >= 1 assert report["failed"] == 0 - # ── Upload local output to MinIO staging ──────────────────────── + # Upload local output to MinIO staging keys = stage_files_to_minio(s3, staging_test_bucket, output_dir, STAGING_PREFIX) assert len(keys) > 0, "Expected files staged to MinIO" - # ── Phase 3: Promote from staging to final path ───────────────── + # Step 3: Promote from staging to final path promote_report = promote_from_s3( staging_key_prefix=STAGING_PREFIX, staging_bucket=staging_test_bucket, @@ -90,7 +90,7 @@ def test_full_pipeline_small_batch( assert promote_report["promoted"] >= 1 assert promote_report["failed"] == 0 - # ── Verify final state ────────────────────────────────────────── + # Verify final state final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") assert len(final_keys) >= 1, "Expected files at final Lakehouse path" @@ -120,7 +120,7 @@ def test_full_pipeline_incremental( """Second sync archives the old version and promotes the new one.""" s3 = minio_s3_client - # ── First sync: Phase 1 → 2 → 3 ──────────────────────────────── + # First sync: Steps 1 -> 2 -> 3 raw = download_assembly_summary(database="refseq") full = parse_assembly_summary(raw) filtered = filter_by_prefix_range(full, prefix_from=STABLE_PREFIX, prefix_to=STABLE_PREFIX) @@ -154,7 +154,7 @@ def test_full_pipeline_incremental( first_sync_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") assert len(first_sync_keys) >= 1 - # ── Second sync: Manufacture "previous" with a tweak ──────────── + # Second sync: Manufacture "previous" with a tweak # Treat first-sync state as "previous", but modify one assembly's # seq_rel_date so it shows up as "updated". previous: dict[str, AssemblyRecord] = {} diff --git a/tests/integration/test_manifest_e2e.py b/tests/integration/test_manifest_e2e.py index fb46ad3a..d85cccfe 100644 --- a/tests/integration/test_manifest_e2e.py +++ b/tests/integration/test_manifest_e2e.py @@ -36,7 +36,7 @@ STABLE_PREFIX = "900" -# ── Helpers ───────────────────────────────────────────────────────────── +# Helpers def _download_and_filter() -> tuple[dict[str, AssemblyRecord], dict[str, AssemblyRecord]]: @@ -50,7 +50,7 @@ def _download_and_filter() -> tuple[dict[str, AssemblyRecord], dict[str, Assembl return full, filtered -# ── Tests ─────────────────────────────────────────────────────────────── +# Tests @pytest.mark.integration diff --git a/tests/integration/test_promote_e2e.py b/tests/integration/test_promote_e2e.py index 20c72022..f837d1c6 100644 --- a/tests/integration/test_promote_e2e.py +++ b/tests/integration/test_promote_e2e.py @@ -10,6 +10,7 @@ import hashlib import json +from pathlib import Path import pytest @@ -23,8 +24,6 @@ from .conftest import get_object_metadata, list_all_keys, seed_lakehouse, staging_test_bucket # noqa: F401 -from pathlib import Path - # Fake assembly details used across tests ACCESSION_A = "GCF_900000001.1" ASSEMBLY_DIR_A = "GCF_900000001.1_FakeAssemblyA" @@ -74,7 +73,7 @@ def _write_manifest(tmp_path: Path, accessions: list[str], name: str) -> Path: return path -# ── Tests ─────────────────────────────────────────────────────────────── +# Tests @pytest.mark.integration @@ -524,7 +523,7 @@ def test_dry_run_no_descriptor(self, minio_s3_client: object, test_bucket: str, assert len(metadata_keys) == 0, f"Dry-run should not create descriptor files, found: {metadata_keys}" -# ── Parallel archiving tests ───────────────────────────────────────────── +# Parallel archiving tests @pytest.mark.integration @@ -946,7 +945,7 @@ def test_dry_run_no_copies_no_deletes( assert len(remaining) == 1, f"Source missing after dry-run: {key}" -# ── Concurrent promotion tests ──────────────────────────────────────────── +# Concurrent promotion tests def _stage_many( diff --git a/tests/ncbi_ftp/conftest.py b/tests/ncbi_ftp/conftest.py index 07fff0c6..e9d8cdaf 100644 --- a/tests/ncbi_ftp/conftest.py +++ b/tests/ncbi_ftp/conftest.py @@ -10,8 +10,8 @@ import cdm_data_loaders.ncbi_ftp.promote as promote_mod import cdm_data_loaders.utils.s3 as s3_utils -from tests.s3_helpers import strip_checksum_algorithm from cdm_data_loaders.utils.s3 import CDM_LAKE_BUCKET, reset_s3_client +from tests.s3_helpers import strip_checksum_algorithm AWS_REGION = "us-east-1" TEST_BUCKET = CDM_LAKE_BUCKET diff --git a/tests/ncbi_ftp/test_assembly.py b/tests/ncbi_ftp/test_assembly.py index 261f4676..d15df95d 100644 --- a/tests/ncbi_ftp/test_assembly.py +++ b/tests/ncbi_ftp/test_assembly.py @@ -8,8 +8,7 @@ parse_md5_checksums_file, ) - -# ── Path helpers ───────────────────────────────────────────────────────── +# Path helpers @pytest.mark.parametrize( @@ -64,7 +63,7 @@ def test_parse_assembly_path_invalid() -> None: parse_assembly_path("/random/path/") -# ── parse_md5_checksums_file ───────────────────────────────────────────── +# parse_md5_checksums_file @pytest.mark.parametrize( diff --git a/tests/ncbi_ftp/test_manifest.py b/tests/ncbi_ftp/test_manifest.py index 5611e925..f8b321d6 100644 --- a/tests/ncbi_ftp/test_manifest.py +++ b/tests/ncbi_ftp/test_manifest.py @@ -1,7 +1,9 @@ """Tests for ncbi_ftp.manifest module — assembly summary parsing, diff, filtering, writing.""" import json +from datetime import UTC, datetime from pathlib import Path +from typing import Any from unittest.mock import MagicMock, patch import pytest @@ -9,8 +11,7 @@ from cdm_data_loaders.ncbi_ftp.manifest import ( AssemblyRecord, DiffResult, - _extract_accession_from_s3_key, - _extract_assembly_dir_from_s3_key, + _extract_accession_dir_and_id_from_s3_key, _ftp_dir_from_url, accession_prefix, compute_diff, @@ -30,7 +31,7 @@ _EXPECTED_TWO = 2 -# ── parse_assembly_summary ─────────────────────────────────────────────── +# parse_assembly_summary _EXPECTED_ASSEMBLIES = { @@ -88,7 +89,7 @@ def test_parse_assembly_summary_input_types(source: str, tmp_path: Path) -> None assert parse_assembly_summary(arg) == _EXPECTED_ASSEMBLIES -# ── get_latest_assembly_paths ──────────────────────────────────────────── +# get_latest_assembly_paths def test_get_latest_assembly_paths() -> None: @@ -104,7 +105,7 @@ def test_get_latest_assembly_paths_empty() -> None: assert get_latest_assembly_paths(parse_assembly_summary("# empty\n")) == [] -# ── compute_diff ───────────────────────────────────────────────────────── +# compute_diff def test_compute_diff_new() -> None: @@ -149,7 +150,7 @@ def test_compute_diff_scan_store_fallback() -> None: assert "GCF_000001405.40" in diff.new -# ── accession_prefix & filter_by_prefix_range ──────────────────────────── +# accession_prefix & filter_by_prefix_range @pytest.mark.parametrize( @@ -173,7 +174,7 @@ def test_filter_by_prefix_range() -> None: assert len(filter_by_prefix_range(assemblies)) == len(assemblies) -# ── Manifest writing ──────────────────────────────────────────────────── +# Manifest writing def test_write_transfer_manifest(tmp_path: Path) -> None: @@ -232,7 +233,7 @@ def test_write_diff_summary(tmp_path: Path) -> None: } -# ── _ftp_dir_from_url ─────────────────────────────────────────────────── +# _ftp_dir_from_url @pytest.mark.parametrize( @@ -268,7 +269,7 @@ def test_ftp_dir_from_url(url: str, expected: str, kwargs: dict) -> None: assert _ftp_dir_from_url(url, **kwargs) == expected -# ── verify_transfer_candidates ─────────────────────────────────────────── +# verify_transfer_candidates _MD5_CHECKSUMS_TXT = ( @@ -454,7 +455,7 @@ def test_verify_transfer_candidates_skips_ftp_when_folder_missing( mock_connect.assert_not_called() -# ── Synthetic summary from S3 store scan ──────────────────────────────── +# Synthetic summary from S3 store scan @pytest.mark.parametrize( @@ -462,73 +463,50 @@ def test_verify_transfer_candidates_skips_ftp_when_folder_missing( [ pytest.param( "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6_plus_ISO1_MT/file.gz", - "GCF_000001215.4", + ("GCF_000001215.4_Release_6_plus_ISO1_MT", "GCF_000001215.4"), id="long_path", ), - pytest.param("some/path/GCA_999999999.1_whatever/data.txt", "GCA_999999999.1", id="short_path"), - pytest.param("some/random/path", None, id="no_accession"), - pytest.param("", None, id="empty"), - ], -) -def test_extract_accession_from_s3_key(key: str, expected: str | None) -> None: - """Accession is extracted from S3 key paths; invalid/empty paths return None.""" - assert _extract_accession_from_s3_key(key) == expected - - -@pytest.mark.parametrize( - ("key", "expected"), - [ pytest.param( - "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6_plus_ISO1_MT/file.gz", - "GCF_000001215.4_Release_6_plus_ISO1_MT", - id="long_path", + "some/path/GCA_999999999.1_whatever/data.txt", + ("GCA_999999999.1_whatever", "GCA_999999999.1"), + id="short_path", ), pytest.param( "prefix/GCA_999999999.1_assembly_name/subdir/data.txt", - "GCA_999999999.1_assembly_name", + ("GCA_999999999.1_assembly_name", "GCA_999999999.1"), id="subdir", ), - pytest.param("some/random/path", None, id="no_assembly_dir"), - pytest.param("", None, id="empty"), + pytest.param("some/random/path", (None, None), id="no_accession"), + pytest.param("", (None, None), id="empty"), ], ) -def test_extract_assembly_dir_from_s3_key(key: str, expected: str | None) -> None: - """Assembly directory is extracted from S3 key paths; invalid/empty paths return None.""" - assert _extract_assembly_dir_from_s3_key(key) == expected - +def test_extract_accession_dir_and_id_from_s3_key(key: str, expected: str | None) -> None: + """Accession is extracted from S3 key paths; invalid/empty paths return None.""" + assert _extract_accession_dir_and_id_from_s3_key(key) == expected -def _make_mock_s3_paginator() -> MagicMock: - """Return a mock S3 client with two assemblies (GCF_000001215.4, GCF_000005845.2).""" - from datetime import datetime, timezone - mock = MagicMock() - mock_paginator = MagicMock() - mock.get_paginator.return_value = mock_paginator - mock_paginator.paginate.return_value = [ +def _make_mock_list_object_return_value() -> list[dict[str, Any]]: + """Return a response for list objects with two assemblies (GCF_000001215.4, GCF_000005845.2).""" + return [ { - "Contents": [ - { - "Key": "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6/file1.gz", - "LastModified": datetime(2024, 1, 15, tzinfo=timezone.utc), - }, - { - "Key": "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6/file2.gz", - "LastModified": datetime(2024, 1, 16, tzinfo=timezone.utc), - }, - { - "Key": "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000005845.2_Assembly/file.gz", - "LastModified": datetime(2024, 2, 20, tzinfo=timezone.utc), - }, - ] - } + "Key": "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6/file1.gz", + "LastModified": datetime(2024, 1, 15, tzinfo=UTC), + }, + { + "Key": "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6/file2.gz", + "LastModified": datetime(2024, 1, 16, tzinfo=UTC), + }, + { + "Key": "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000005845.2_Assembly/file.gz", + "LastModified": datetime(2024, 2, 20, tzinfo=UTC), + }, ] - return mock -@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") -def test_scan_store_builds_summary(mock_get_s3: MagicMock) -> None: +@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects") +def test_scan_store_builds_summary(mock_list_matching_objects: MagicMock) -> None: """Synthetic summary is built correctly with provided release_date for all assemblies.""" - mock_get_s3.return_value = _make_mock_s3_paginator() + mock_list_matching_objects.return_value = _make_mock_list_object_return_value() assert scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/01/31") == { "GCF_000001215.4": AssemblyRecord( accession="GCF_000001215.4", @@ -547,47 +525,37 @@ def test_scan_store_builds_summary(mock_get_s3: MagicMock) -> None: } -@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") -def test_scan_store_applies_release_date_to_all(mock_get_s3: MagicMock) -> None: +@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects") +def test_scan_store_applies_release_date_to_all(mock_list_matching_objects: MagicMock) -> None: """Provided release_date is used even when files have different LastModified dates.""" - from datetime import datetime, timezone - - mock = MagicMock() - mock_paginator = MagicMock() - mock.get_paginator.return_value = mock_paginator - mock_paginator.paginate.return_value = [ + mock_list_matching_objects.return_value = [ { - "Contents": [ - { - "Key": "prefix/GCF_000001215.4_v1/file_newer.gz", - "LastModified": datetime(2024, 3, 20, tzinfo=timezone.utc), - }, - { - "Key": "prefix/GCF_000001215.4_v1/file_older.gz", - "LastModified": datetime(2024, 1, 10, tzinfo=timezone.utc), - }, - ] - } + "Key": "prefix/GCF_000001215.4_v1/file_newer.gz", + "LastModified": datetime(2024, 3, 20, tzinfo=UTC), + }, + { + "Key": "prefix/GCF_000001215.4_v1/file_older.gz", + "LastModified": datetime(2024, 1, 10, tzinfo=UTC), + }, ] - mock_get_s3.return_value = mock assert ( scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/03/31")["GCF_000001215.4"].seq_rel_date == "2024/03/31" ) -@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") -def test_scan_store_raises_for_invalid_release_date(mock_get_s3: MagicMock) -> None: +@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects") +def test_scan_store_raises_for_invalid_release_date(mock_list_matching_objects: MagicMock) -> None: """Invalid release_date format is rejected.""" - mock_get_s3.return_value = _make_mock_s3_paginator() + mock_list_matching_objects.return_value = _make_mock_list_object_return_value() with pytest.raises(ValueError, match="Invalid release_date"): scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024-03-31") -@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") -def test_scan_store_invokes_progress_callback(mock_get_s3: MagicMock) -> None: +@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects") +def test_scan_store_invokes_progress_callback(mock_list_matching_objects: MagicMock) -> None: """Progress callback is called once per unique assembly discovered.""" - mock_get_s3.return_value = _make_mock_s3_paginator() + mock_list_matching_objects.return_value = _make_mock_list_object_return_value() calls: list[tuple[int, str]] = [] scan_store_to_synthetic_summary( "test-bucket", "prefix/", "2024/01/31", progress_callback=lambda n, a: calls.append((n, a)) @@ -597,65 +565,41 @@ def test_scan_store_invokes_progress_callback(mock_get_s3: MagicMock) -> None: assert calls[1][0] == 2 -@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") -def test_scan_store_handles_empty_store(mock_get_s3: MagicMock) -> None: +@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects") +def test_scan_store_handles_empty_store(mock_list_matching_objects: MagicMock) -> None: """Empty store returns empty dict.""" - mock = MagicMock() - mock_paginator = MagicMock() - mock.get_paginator.return_value = mock_paginator - mock_paginator.paginate.return_value = [{"Contents": []}] - mock_get_s3.return_value = mock + mock_list_matching_objects.return_value = [] assert scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/01/31") == {} -@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") -def test_scan_store_skips_objects_without_accession(mock_get_s3: MagicMock) -> None: +@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects") +def test_scan_store_skips_objects_without_accession(mock_list_matching_objects: MagicMock) -> None: """Objects without valid accessions in the key are skipped.""" - from datetime import datetime, timezone - - mock = MagicMock() - mock_paginator = MagicMock() - mock.get_paginator.return_value = mock_paginator - mock_paginator.paginate.return_value = [ + mock_list_matching_objects.return_value = [ + {"Key": "prefix/some/random/file.txt", "LastModified": datetime(2024, 1, 1, tzinfo=UTC)}, { - "Contents": [ - {"Key": "prefix/some/random/file.txt", "LastModified": datetime(2024, 1, 1, tzinfo=timezone.utc)}, - { - "Key": "prefix/GCF_000001215.4_Assembly/valid_file.gz", - "LastModified": datetime(2024, 2, 1, tzinfo=timezone.utc), - }, - ] - } + "Key": "prefix/GCF_000001215.4_Assembly/valid_file.gz", + "LastModified": datetime(2024, 2, 1, tzinfo=UTC), + }, ] - mock_get_s3.return_value = mock result = scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/01/31") assert len(result) == 1 assert "GCF_000001215.4" in result -@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client") -def test_scan_store_assembly_dir_survives_round_trip(mock_get_s3: MagicMock, tmp_path: Path) -> None: +@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects") +def test_scan_store_assembly_dir_survives_round_trip(mock_list_matching_objects: MagicMock, tmp_path: Path) -> None: """assembly_dir is preserved after save-to-file / parse-back round-trip. Regression: previously ftp_path was written as "" causing assembly_dir="" and compute_diff flagging every assembly as updated. """ - from datetime import datetime, timezone - - mock = MagicMock() - mock_paginator = MagicMock() - mock_paginator.paginate.return_value = [ + mock_list_matching_objects.return_value = [ { - "Contents": [ - { - "Key": "prefix/GCF_000001215.4_Release_6_plus_ISO1_MT/file.gz", - "LastModified": datetime(2024, 3, 10, tzinfo=timezone.utc), - } - ] + "Key": "prefix/GCF_000001215.4_Release_6_plus_ISO1_MT/file.gz", + "LastModified": datetime(2024, 3, 10, tzinfo=UTC), } ] - mock.get_paginator.return_value = mock_paginator - mock_get_s3.return_value = mock synthetic = scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/03/10") diff --git a/tests/ncbi_ftp/test_metadata.py b/tests/ncbi_ftp/test_metadata.py index df1df775..d9b86c4f 100644 --- a/tests/ncbi_ftp/test_metadata.py +++ b/tests/ncbi_ftp/test_metadata.py @@ -57,7 +57,7 @@ ] -# ── build_descriptor_key / build_archive_descriptor_key ───────────────── +# build_descriptor_key / build_archive_descriptor_key @pytest.mark.parametrize("prefix", [_KEY_PREFIX, _KEY_PREFIX.rstrip("/")]) @@ -83,7 +83,7 @@ def test_build_archive_descriptor_key(prefix: str, tag: str) -> None: assert "//" not in key -# ── create_descriptor ──────────────────────────────────────────────────── +# create_descriptor def test_create_descriptor() -> None: @@ -169,7 +169,7 @@ def test_create_descriptor_empty_resources() -> None: assert d["resources"] == [] -# ── validate_descriptor ────────────────────────────────────────────────── +# validate_descriptor def test_validate_descriptor_valid() -> None: @@ -186,7 +186,7 @@ def test_validate_descriptor_empty_raises() -> None: validate_descriptor({}, _ACCESSION) -# ── upload_descriptor / archive_descriptor ─────────────────────────────── +# upload_descriptor / archive_descriptor @pytest.fixture diff --git a/tests/ncbi_ftp/test_notebooks.py b/tests/ncbi_ftp/test_notebooks.py index 345572db..d905089b 100644 --- a/tests/ncbi_ftp/test_notebooks.py +++ b/tests/ncbi_ftp/test_notebooks.py @@ -6,7 +6,7 @@ import pytest -from cdm_data_loaders.ncbi_ftp.assembly import FTP_HOST # noqa: F401 +from cdm_data_loaders.ncbi_ftp.assembly import FTP_HOST from cdm_data_loaders.ncbi_ftp.manifest import ( # noqa: F401 AssemblyRecord, compute_diff, @@ -22,7 +22,7 @@ DEFAULT_LAKEHOUSE_KEY_PREFIX, promote_from_s3, ) -from cdm_data_loaders.utils.s3 import split_s3_path # noqa: F401 +from cdm_data_loaders.utils.s3 import split_s3_path NOTEBOOKS_DIR = Path(__file__).resolve().parents[2] / "notebooks" @@ -77,7 +77,7 @@ def test_promote_notebook_imports() -> None: def test_download_notebook_imports() -> None: """All download notebook imports resolve without error.""" - from cdm_data_loaders.pipelines.ncbi_ftp_download import ( # noqa: F401 + from cdm_data_loaders.pipelines.ncbi_ftp_download import ( DEFAULT_STAGING_KEY_PREFIX, download_and_stage, ) diff --git a/tests/ncbi_ftp/test_promote.py b/tests/ncbi_ftp/test_promote.py index ae0ff887..95740f3e 100644 --- a/tests/ncbi_ftp/test_promote.py +++ b/tests/ncbi_ftp/test_promote.py @@ -14,8 +14,7 @@ ) from tests.ncbi_ftp.conftest import TEST_BUCKET - -# ── Promotion test constants ───────────────────────────────────────────── +# Promotion test constants _STAGE_PREFIX = "staging/run1/" @@ -279,7 +278,7 @@ def test_archive_assemblies_unknown_release_fallback( assert mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_key).get("KeyCount", 0) == 1 -# ── Concurrent / multi-file archive (new behaviour) ───────────────────── +# Concurrent / multi-file archive (new behaviour) @pytest.mark.s3 @@ -391,7 +390,7 @@ def test_archive_assemblies_multi_file_delete_all( assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 -# ── Partial-archive idempotency ────────────────────────────────────────── +# Partial-archive idempotency @pytest.mark.s3 @@ -600,7 +599,7 @@ def test_archive_assemblies_invalid_accession_skipped( assert archived == 1 -# ── Concurrent / multi-file promotion (new behaviour) ──────────────────── +# Concurrent / multi-file promotion (new behaviour) @pytest.mark.s3 @@ -794,7 +793,7 @@ def _download_one_fail(Bucket: str, Key: str, Filename: str, **kw: object) -> No resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=key) assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200, ( f"Expected staged file to survive partial failure: {key}" - ) # noqa: PLR2004 + ) @pytest.mark.s3 @@ -803,7 +802,7 @@ def test_promote_partial_failure_failed_count( ) -> None: """report[\"failed\"] reflects the number of files that could not be promoted.""" file_names = [f"{_ACC1}_genomic.fna.gz", f"{_ACC1}_protein.faa.gz", f"{_ACC1}_rna.fna.gz"] - _stage(mock_s3_client_no_checksum, _STG1, {f: b"data" for f in file_names}) + _stage(mock_s3_client_no_checksum, _STG1, dict.fromkeys(file_names, b"data")) failing_key = f"{_STG1}{file_names[1]}" original_download = mock_s3_client_no_checksum.download_file @@ -876,7 +875,7 @@ def _patched(Bucket: str, Key: str, Filename: str, **kw: object) -> None: # noq resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{_STG2}{fname}") assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200, ( f"Assembly 2 staging must survive partial failure: {fname}" - ) # noqa: PLR2004 + ) @pytest.mark.s3 diff --git a/tests/pipelines/test_ncbi_ftp_download.py b/tests/pipelines/test_ncbi_ftp_download.py index 99a9ebfd..8e8602ee 100644 --- a/tests/pipelines/test_ncbi_ftp_download.py +++ b/tests/pipelines/test_ncbi_ftp_download.py @@ -42,7 +42,7 @@ def make_settings(**kwargs: str | int) -> DownloadSettings: return DownloadSettings(_cli_parse_args=[], **kwargs) -# ── Settings defaults ──────────────────────────────────────────────────── +# Settings defaults class TestDownloadSettingsDefaults: @@ -74,7 +74,7 @@ def test_limit_default_none(self) -> None: assert s.limit is None -# ── Settings all params ────────────────────────────────────────────────── +# Settings all params class TestDownloadSettingsAllParams: @@ -96,7 +96,7 @@ def test_all_params(self) -> None: assert s.limit == _CUSTOM_LIMIT -# ── Settings aliases ───────────────────────────────────────────────────── +# Settings aliases class TestDownloadSettingsAliases: @@ -123,7 +123,7 @@ def test_limit_alias_l(self) -> None: assert s.limit == _ALIAS_LIMIT -# ── Settings validation ────────────────────────────────────────────────── +# Settings validation class TestDownloadSettingsValidation: @@ -155,7 +155,7 @@ def test_limit_must_be_positive(self) -> None: make_settings(limit=0) -# ── download_batch ─────────────────────────────────────────────────────── +# download_batch class TestDownloadBatch: @@ -245,7 +245,7 @@ def test_handles_download_failure(self, tmp_path: Path) -> None: assert report["succeeded"] == 0 -# ── Helpers shared by download_and_stage tests ─────────────────────────── +# Helpers shared by download_and_stage tests _MANIFEST_CONTENT = ( "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/\n" @@ -262,7 +262,7 @@ def _make_moto_s3(): return client -# ── download_and_stage — manifest source ──────────────────────────────── +# download_and_stage — manifest source @pytest.mark.parametrize( @@ -322,7 +322,7 @@ def _fake_download(path, output_dir, **kwargs): # noqa: ARG001 reset_s3_client() -# ── download_and_stage — exactly one source required ──────────────────── +# download_and_stage — exactly one source required @pytest.mark.parametrize( @@ -386,7 +386,7 @@ def test_download_and_stage_exactly_one_source_required( reset_s3_client() -# ── download_and_stage — uploads to staging ────────────────────────────── +# download_and_stage — uploads to staging @mock_aws @@ -442,7 +442,7 @@ def _fake_download(path, output_dir, **kwargs): # noqa: ARG001 reset_s3_client() -# ── download_and_stage — dry_run skips upload ──────────────────────────── +# download_and_stage — dry_run skips upload @mock_aws @@ -488,7 +488,7 @@ def _fake_download(path, output_dir, **kwargs): # noqa: ARG001 reset_s3_client() -# ── download_and_stage — limit forwarded ──────────────────────────────── +# download_and_stage — limit forwarded @pytest.mark.parametrize( @@ -534,7 +534,7 @@ def test_download_and_stage_limit_forwarded(tmp_path: Path, limit: int) -> None: reset_s3_client() -# ── download_and_stage — report shape ─────────────────────────────────── +# download_and_stage — report shape @mock_aws diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index 8c6c2f9b..3b1118ba 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -15,7 +15,6 @@ from requests.exceptions import HTTPError import cdm_data_loaders.utils.s3 as s3_utils -from tests.s3_helpers import strip_checksum_algorithm from cdm_data_loaders.utils.s3 import ( CDM_LAKE_BUCKET, DEFAULT_EXTRA_ARGS, @@ -34,6 +33,7 @@ upload_dir, upload_file, ) +from tests.s3_helpers import strip_checksum_algorithm AWS_REGION = "us-east-1" From a78df131b4bf455bc84e232568ed9d924c77a030 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 28 May 2026 15:05:32 -0700 Subject: [PATCH 087/128] address review comments --- src/cdm_data_loaders/ncbi_ftp/manifest.py | 12 +++++++----- src/cdm_data_loaders/utils/s3.py | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/manifest.py b/src/cdm_data_loaders/ncbi_ftp/manifest.py index 653f00ae..0df571ea 100644 --- a/src/cdm_data_loaders/ncbi_ftp/manifest.py +++ b/src/cdm_data_loaders/ncbi_ftp/manifest.py @@ -366,14 +366,14 @@ def scan_store_to_synthetic_summary( def _fetch_accession_checksums_from_ftp( ftp: FTP | None, ftp_host: str, last_activity: float, current_accession: str -) -> dict[str, str]: +) -> tuple[dict[str, str], float]: """Fetch and parse the md5checksums.txt file for a given accession from FTP. :param ftp: FTP connection object :param ftp_host: NCBI FTP hostname :param last_activity: timestamp of the last FTP activity :param current_accession: accession identifier - :return: dictionary mapping file names to MD5 checksums + :return: dictionary mapping file names to MD5 checksums and updated last_activity timestamp """ if ftp is None: ftp = connect_ftp(ftp_host) @@ -385,10 +385,10 @@ def _fetch_accession_checksums_from_ftp( ftp_checksums = parse_md5_checksums_file(md5_text) except Exception: # noqa: BLE001 logger.warning("Cannot fetch md5checksums.txt for %s, keeping in transfer list", current_accession) - return {} + return {}, last_activity return { fname: md5 for fname, md5 in ftp_checksums.items() if any(fname.endswith(suffix) for suffix in FILE_FILTERS) - } + }, last_activity def _does_accession_need_update( @@ -483,7 +483,9 @@ def _progress(done: int, total: int, acc: str) -> None: continue # Objects exist — need FTP md5 checksums to decide - target_checksums = _fetch_accession_checksums_from_ftp(ftp, ftp_host, last_activity, rec.ftp_path) + target_checksums, last_activity = _fetch_accession_checksums_from_ftp( + ftp, ftp_host, last_activity, rec.ftp_path + ) if not target_checksums: confirmed.append(acc) diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index bf285ed3..b6386dc0 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -263,8 +263,8 @@ def upload_file( Key=key, ExtraArgs=extra_args, ) - except Exception as e: # noqa: BLE001 - logger.exception(f"Error uploading to s3: {e}") + except Exception: # noqa: BLE001 + logger.exception("Error uploading to s3") return False return True From 08bd2e0ef3d1915f7c83c96165ac64a48a3dacfa Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Thu, 28 May 2026 15:17:25 -0700 Subject: [PATCH 088/128] update function arg --- src/cdm_data_loaders/ncbi_ftp/promote.py | 2 +- src/cdm_data_loaders/utils/s3.py | 15 ++++++++------- tests/utils/test_s3.py | 10 +++++----- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index a97cba2a..e7cfb6ea 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -199,7 +199,7 @@ def _promote_one(staged_key: str) -> tuple[DescriptorResource, str]: upload_succeeded = upload_file( tmp_path, f"{lakehouse_bucket}/{final_key_path.parent}", - tags=metadata, + user_metadata=metadata, object_name=final_key_path.name, show_progress=False, ) diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index b6386dc0..4dfd80ea 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -200,13 +200,14 @@ def upload_file( local_file_path: Path | str, destination_dir: str, object_name: str | None = None, - tags: dict[str, str] | None = None, + user_metadata: dict[str, str] | None = None, + *, show_progress: bool = True, ) -> bool: """Upload an object to an S3 bucket. - When *tags* is supplied the file is always uploaded (no existence check) - and the dict is attached as S3 user metadata. When *tags* is ``None`` + When *user_metadata* is supplied the file is always uploaded (no existence check) + and the dict is attached as S3 user metadata. When *user_metadata* is ``None`` (the default) the existing behaviour is preserved: the upload is skipped if the object is already present. @@ -216,8 +217,8 @@ def upload_file( :type destination_dir: str :param object_name: S3 object name. If not specified, the name of the file from local_file_path is used. :type object_name: str | None - :param tags: user metadata key/value pairs to attach to the object; when provided the upload always runs - :type tags: dict[str, str] | None + :param user_metadata: user metadata key/value pairs to attach to the object; when provided the upload always runs + :type user_metadata: dict[str, str] | None :param show_progress: whether to display a tqdm progress bar during upload, defaults to True :type show_progress: bool, optional :return: True if file was uploaded, else False @@ -234,14 +235,14 @@ def upload_file( object_name = local_file_path.name s3_path = f"{destination_dir.removesuffix('/')}/{object_name}" - if tags is None and object_exists(s3_path): + if user_metadata is None and object_exists(s3_path): logger.debug("File already present: %s", s3_path) return True s3 = get_s3_client() (bucket, key) = split_s3_path(s3_path) - extra_args = {**DEFAULT_EXTRA_ARGS, **(({"Metadata": tags}) if tags is not None else {})} + extra_args = {**DEFAULT_EXTRA_ARGS, **(({"Metadata": user_metadata}) if user_metadata is not None else {})} # Upload the file logger.debug("uploading %s to %s", str(local_file_path), s3_path) diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index 3b1118ba..a642ed87 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -900,7 +900,7 @@ def test_delete_object_removes_object(mock_s3_client: Any, bucket: str, protocol def test_upload_file_with_metadata_attaches_metadata(mock_s3_client: Any, sample_file: Path, bucket: str) -> None: """Verify that upload_file with metadata stores user metadata on the uploaded object.""" metadata = {"md5": "abc123", "source": "ncbi"} - result = upload_file(sample_file, f"{bucket}/uploads", tags=metadata) + result = upload_file(sample_file, f"{bucket}/uploads", user_metadata=metadata) assert result is True resp = mock_s3_client.head_object(Bucket=bucket, Key=f"uploads/{sample_file.name}") @@ -911,7 +911,7 @@ def test_upload_file_with_metadata_attaches_metadata(mock_s3_client: Any, sample @pytest.mark.s3 def test_upload_file_with_metadata_custom_object_name(mock_s3_client: Any, sample_file: Path) -> None: """Verify that the object_name parameter overrides the filename.""" - result = upload_file(sample_file, f"{CDM_LAKE_BUCKET}/uploads", tags={"k": "v"}, object_name="renamed.txt") + result = upload_file(sample_file, f"{CDM_LAKE_BUCKET}/uploads", user_metadata={"k": "v"}, object_name="renamed.txt") assert result is True obj = mock_s3_client.get_object(Bucket=CDM_LAKE_BUCKET, Key="uploads/renamed.txt") assert obj["Body"].read() == b"hello s3" @@ -921,7 +921,7 @@ def test_upload_file_with_metadata_custom_object_name(mock_s3_client: Any, sampl def test_upload_file_with_metadata_overwrites_existing(mock_s3_client: Any, sample_file: Path) -> None: """Verify that upload_file with metadata uploads even when the object already exists.""" mock_s3_client.put_object(Bucket=CDM_LAKE_BUCKET, Key=f"uploads/{sample_file.name}", Body=b"old") - result = upload_file(sample_file, f"{CDM_LAKE_BUCKET}/uploads", tags={"new": "true"}) + result = upload_file(sample_file, f"{CDM_LAKE_BUCKET}/uploads", user_metadata={"new": "true"}) assert result is True obj = mock_s3_client.get_object(Bucket=CDM_LAKE_BUCKET, Key=f"uploads/{sample_file.name}") assert obj["Body"].read() == b"hello s3" @@ -932,7 +932,7 @@ def test_upload_file_with_metadata_overwrites_existing(mock_s3_client: Any, samp def test_upload_file_with_metadata_raises_on_empty_destination(sample_file: Path) -> None: """Verify ValueError when destination_dir is empty.""" with pytest.raises(ValueError, match="No destination directory"): - upload_file(sample_file, "", tags={"k": "v"}) + upload_file(sample_file, "", user_metadata={"k": "v"}) @pytest.mark.usefixtures("mock_s3_client") @@ -940,7 +940,7 @@ def test_upload_file_with_metadata_raises_on_empty_destination(sample_file: Path @pytest.mark.s3 def test_upload_file_with_metadata_accepts_str_and_path(sample_file: Path, path_type: type[str] | type[Path]) -> None: """Verify that upload_file with metadata accepts both str and Path.""" - result = upload_file(path_type(sample_file), f"{CDM_LAKE_BUCKET}/uploads", tags={}) + result = upload_file(path_type(sample_file), f"{CDM_LAKE_BUCKET}/uploads", user_metadata={}) assert result is True From a51ed3297aac01e2bf797f86f3f6016b66bb0221 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Thu, 28 May 2026 16:58:14 -0700 Subject: [PATCH 089/128] Update s3 module to remove dependency on berdl utils and refs to minio --- pyproject.toml | 13 +- src/cdm_data_loaders/utils/s3.py | 69 ++++++----- tests/utils/test_s3.py | 199 +++++++++++++++++++++++-------- uv.lock | 9 ++ 4 files changed, 200 insertions(+), 90 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ed0c2afc..28c960ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -196,23 +196,20 @@ markers = ["requires_spark: must be run in an environment where spark is availab USER = "fake_user" KBASE_AUTH_TOKEN = "test-token-123" CDM_TASK_SERVICE_URL = "http://localhost:8080" -MINIO_ENDPOINT_URL = "http://localhost:9000" -MINIO_ACCESS_KEY = "minioadmin" -MINIO_SECRET_KEY = "minioadmin" -MINIO_SECURE_FLAG = "false" BERDL_POD_IP = "192.168.1.100" SPARK_MASTER_URL = "spark://localhost:7077" BERDL_HIVE_METASTORE_URI = "thrift://localhost:9083" SPARK_CLUSTER_MANAGER_API_URL = "http://localhost:8000" GOVERNANCE_API_URL = "http://localhost:8000" DATALAKE_MCP_SERVER_URL = "http://localhost:8080" -AWS_ACCESS_KEY_ID = "whatever" -AWS_SECRET_ACCESS_KEY = "whatever" -AWS_SECURITY_TOKEN = "whatever" -AWS_SESSION_TOKEN = "whatever" +S3_ENDPOINT_URL = "http://localhost:9000" +# aws config settings +AWS_CONFIG_FILE = "/dev/null" # ensure tests don't pick up creds from a real config file AWS_DEFAULT_REGION = "us-east-1" AWS_ENDPOINT_URL = { unset = true } AWS_ENDPOINT_URL_S3 = { unset = true} +AWS_ACCESS_KEY_ID = { unset = true} +AWS_SECRET_ACCESS_KEY = { unset = true} [tool.uv.sources] cdm-schema = { git = "https://github.com/kbase/cdm-schema.git" } diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index 5d803f00..3212ce7a 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -30,11 +30,21 @@ logger = get_cdm_logger() -def get_s3_client(args: dict[str, str] | None = None) -> botocore.client.BaseClient: +def get_s3_client(args: dict[str, str | None] | None = None) -> botocore.client.BaseClient: """Create an S3 client using the provided arguments. - The client is created once and cached for subsequent calls. Call - reset_s3_client() to force a new client to be created on the next call. + The client is created once and cached for subsequent calls. + Call reset_s3_client() to force a new client to be created on the next call. + + To configure the client using arguments, provide a dictionary with the following keys: + - aws_access_key_id: the access key ID for the S3 client + - aws_secret_access_key: the secret access key for the S3 client + - endpoint_url: the endpoint URL for the S3 client (e.g., "https://s3.amazonaws.com" or "https://my-s3-server.com") + + If arguments are not provided, the client will be created using boto3's default + configuration method, which looks environment variables (AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, and AWS_ENDPOINT_URL_S3 or AWS_ENDPOINT_URL) or an ``./aws`` config directory. + See the boto3 documentation for more details. :param args: arguments for creating the S3 client, defaults to None :type args: dict[str, str] | None, optional @@ -49,37 +59,38 @@ def get_s3_client(args: dict[str, str] | None = None) -> botocore.client.BaseCli config = Config(retries={"total_max_attempts": AWS_CLIENT_TOTAL_MAX_ATTEMPTS, "mode": AWS_CLIENT_RETRY_MODE}) if not args: - # try using env vars and skip manual configuration - client = boto3.client("s3", config=config) - # check for credentials and endpoint_url - credentials = client._request_signer._credentials # noqa: SLF001 - if credentials.access_key and credentials.secret_key and client.meta.endpoint_url: - _s3_client = client - return _s3_client + args = {} - try: - from berdl_notebook_utils.berdl_settings import get_settings # noqa: PLC0415 - - settings = get_settings() - args = { - "endpoint_url": settings.MINIO_ENDPOINT_URL, - "aws_access_key_id": settings.MINIO_ACCESS_KEY, - "aws_secret_access_key": settings.MINIO_SECRET_KEY, - } - except (ModuleNotFoundError, ImportError, NameError): - logger.exception("Error initialising boto3 client") - raise - except Exception: - raise + valid_kwargs = ["aws_access_key_id", "aws_secret_access_key", "endpoint_url"] + kwargs = {k: v for k, v in args.items() if k in valid_kwargs and v is not None} + + # make sure that if aws_access_key_id or aws_secret_access_key is provided, the other is also provided. + if (kwargs.get("aws_access_key_id") and not kwargs.get("aws_secret_access_key")) or ( + not kwargs.get("aws_access_key_id") and kwargs.get("aws_secret_access_key") + ): + msg = "Cannot initialise s3 client: aws_access_key_id and aws_secret_access_key must be provided together, either via args or environment variables or a config file" + raise ValueError(msg) + + # initialise using boto3's default config behaviour, plus any overrides from args + client = boto3.client("s3", config=config, **kwargs) + + missing = [] + # boto3 will not raise an error on client creation if credentials are missing, so throw an error now + credentials = client._request_signer._credentials # noqa: SLF001 + if not credentials: + missing = ["aws_access_key_id", "aws_secret_access_key"] + else: + if not credentials.access_key: + missing.append("aws_access_key_id") + if not credentials.secret_key: + missing.append("aws_secret_access_key") - required_args = ["endpoint_url", "aws_access_key_id", "aws_secret_access_key"] - keyword_args = {kw: args.get(kw) for kw in required_args} - missing = [kw for kw in required_args if not keyword_args[kw]] if missing: - msg = "Cannot initialise s3 client: missing arguments: " + ", ".join(missing) + msg = "Cannot initialise s3 client: missing configuration values: " + ", ".join(missing) raise ValueError(msg) - _s3_client = boto3.client("s3", config=config, **keyword_args) + # nothing missing: we are good to go! + _s3_client = client return _s3_client diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index 0ba7d567..0955eb59 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -10,7 +10,7 @@ import boto3 import pytest -from botocore.exceptions import ClientError +from botocore.exceptions import ClientError, PartialCredentialsError from moto import mock_aws from requests.exceptions import ConnectionError as ConnError from requests.exceptions import HTTPError @@ -35,6 +35,8 @@ ) AWS_REGION = "us-east-1" +AWS_ENV_VARS = ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_ENDPOINT_URL_S3", "AWS_ENDPOINT_URL"] + SAMPLE_FILES = [ "dir_one/file1.txt", @@ -129,77 +131,172 @@ def populate_mock_s3(client: Any, file_list_by_bucket: dict[str, list[str]]) -> # Client creation / reset +@mock_aws @pytest.mark.s3 -def test_get_s3_client_raises_on_missing_args() -> None: - """Verify that get_s3_client raises ValueError when required arguments are absent.""" +@pytest.mark.parametrize("endpoint_url", ["http://localhost", None]) +def test_get_s3_client_success_via_args(endpoint_url: str | None, monkeypatch: pytest.MonkeyPatch) -> None: + """Verify that get_s3_client creates a client with the correct credentials and endpoint URL using args for the creds.""" reset_s3_client() - with pytest.raises(ValueError, match="missing arguments"): - get_s3_client(args={"endpoint_url": "http://localhost", "aws_access_key_id": "key"}) + assert s3_utils._s3_client is None # noqa: SLF001 + # set up env vars to ensure that the argument takes precedence + monkeypatch.setenv("AWS_ENDPOINT_URL", "http://env-endpoint.com") + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "aws_access_key_id_env_var") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "aws_secret_access_key_env_var") + args = { + "aws_access_key_id": "aws_access_key_id_argument", + "aws_secret_access_key": "aws_secret_access_key_argument", + "endpoint_url": endpoint_url, + } + + client = get_s3_client(args) + assert client is not None + credentials = client._request_signer._credentials # noqa: SLF001 + assert credentials.access_key == "aws_access_key_id_argument" + assert credentials.secret_key == "aws_secret_access_key_argument" # noqa: S105 + expected_endpoint = endpoint_url or "http://env-endpoint.com" + assert client.meta.endpoint_url == expected_endpoint + reset_s3_client() assert s3_utils._s3_client is None # noqa: SLF001 +@mock_aws @pytest.mark.s3 -def test_get_s3_client_returns_client_with_valid_args() -> None: - """Verify that get_s3_client returns a usable client when all required args are provided.""" +@pytest.mark.parametrize("endpoint_url", ["http://localhost", None]) +def test_get_s3_client_success_via_env(endpoint_url: str | None, monkeypatch: pytest.MonkeyPatch) -> None: + """Verify that get_s3_client creates a client with the correct credentials and endpoint URL using env vars for the creds.""" reset_s3_client() - with mock_aws(): - client = get_s3_client( - args={ - "endpoint_url": "http://localhost:9000", - "aws_access_key_id": "key", - "aws_secret_access_key": "secret", - } - ) - assert client is not None + assert s3_utils._s3_client is None # noqa: SLF001 + # set up the endpoint URL as an env var to ensure that the argument takes precedence + monkeypatch.setenv("AWS_ENDPOINT_URL", "http://env-endpoint.com") + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "aws_access_key_id_env_var") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "aws_secret_access_key_env_var") + args = { + "endpoint_url": endpoint_url, + } + + client = get_s3_client(args) + assert client is not None + credentials = client._request_signer._credentials # noqa: SLF001 + assert credentials.access_key == "aws_access_key_id_env_var" + assert credentials.secret_key == "aws_secret_access_key_env_var" # noqa: S105 + expected_endpoint = endpoint_url or "http://env-endpoint.com" + assert client.meta.endpoint_url == expected_endpoint + reset_s3_client() assert s3_utils._s3_client is None # noqa: SLF001 +@mock_aws @pytest.mark.s3 -def test_get_s3_client_returns_same_instance() -> None: - """Verify that repeated calls to get_s3_client return the exact same cached client instance.""" +@pytest.mark.parametrize( + ("aws_access_key_id", "aws_secret_access_key"), [("aws_access_key_id", None), (None, "aws_secret_access_key")] +) +def test_get_s3_client_incomplete_creds_via_args( + aws_access_key_id: str | None, aws_secret_access_key: str | None +) -> None: + """Verify that get_s3_client raises ValueError when only one of aws_access_key_id or aws_secret_access_key is provided via arguments.""" reset_s3_client() assert s3_utils._s3_client is None # noqa: SLF001 - with mock_aws(): - args = { - "endpoint_url": "http://localhost:9000", - "aws_access_key_id": "key", - "aws_secret_access_key": "secret", - } - client_a = get_s3_client(args=args) - assert s3_utils._s3_client is not None # noqa: SLF001 - # call again with no args - should return the stored version - client_b = get_s3_client() - assert client_a is client_b - # call again with invalid args - should return the stored version, ignoring args - client_c = get_s3_client(args={"this": "that", "pip": "pop"}) - assert client_c == client_a - # reset the client and call - reset_s3_client() - assert s3_utils._s3_client is None # noqa: SLF001 - client_d = get_s3_client( - { - "endpoint_url": "http://localhost:9000", - "aws_access_key_id": "not a key", - "aws_secret_access_key": "not a secret", - } - ) - assert client_d != client_a + args = { + "aws_access_key_id": aws_access_key_id, + "aws_secret_access_key": aws_secret_access_key, + } + with pytest.raises( + ValueError, + match="Cannot initialise s3 client: aws_access_key_id and aws_secret_access_key must be provided together", + ): + get_s3_client(args) + assert s3_utils._s3_client is None # noqa: SLF001 + + reset_s3_client() + assert s3_utils._s3_client is None # noqa: SLF001 + + +@mock_aws +@pytest.mark.s3 +@pytest.mark.parametrize( + ("aws_access_key_id", "aws_secret_access_key", "error_type", "error_msg"), + [ + ( + "aws_access_key_id", + None, + PartialCredentialsError, + "Partial credentials found in env, missing: AWS_SECRET_ACCESS_KEY", + ), + (None, "aws_secret_access_key", ValueError, "missing configuration values: aws_access_key_id"), + (None, None, ValueError, "missing configuration values: aws_access_key_id, aws_secret_access_key"), + ], +) +def test_get_s3_client_incomplete_creds_via_env( + aws_access_key_id: str | None, + aws_secret_access_key: str | None, + error_type: type[Exception], + error_msg: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Verify that get_s3_client raises ValueError when only one of aws_access_key_id or aws_secret_access_key is provided.""" + reset_s3_client() + assert s3_utils._s3_client is None # noqa: SLF001 + args = { + "aws_access_key_id": aws_access_key_id, + "aws_secret_access_key": aws_secret_access_key, + } + for key, value in args.items(): + if value: + monkeypatch.setenv(key.upper(), value) + else: + monkeypatch.delenv(key.upper(), raising=False) + monkeypatch.setenv("AWS_CONFIG_FILE", "/dev/null") + + with pytest.raises( + error_type, + match=error_msg, + ): + get_s3_client() + assert s3_utils._s3_client is None # noqa: SLF001 reset_s3_client() assert s3_utils._s3_client is None # noqa: SLF001 +@mock_aws @pytest.mark.s3 -def test_get_s3_client_populates_from_environment() -> None: - # set up the environment +def test_get_s3_client_returns_same_instance() -> None: + """Verify that repeated calls to get_s3_client return the exact same cached client instance.""" + reset_s3_client() + assert s3_utils._s3_client is None # noqa: SLF001 + + args = { + "endpoint_url": "http://localhost:9000", + "aws_access_key_id": "key", + "aws_secret_access_key": "secret", + } + client_a = get_s3_client(args) # type: ignore[reportArgumentType] + assert s3_utils._s3_client is not None # noqa: SLF001 + # call again with no args - should return the stored version + client_b = get_s3_client() + assert client_a is client_b + # call again with invalid args - should return the stored version, ignoring args + client_c = get_s3_client(args={"this": "that", "pip": "pop"}) + assert client_c == client_a + # reset the client and call + reset_s3_client() + assert s3_utils._s3_client is None # noqa: SLF001 + client_d = get_s3_client( + { + "endpoint_url": "http://localhost:9000", + "aws_access_key_id": "not a key", + "aws_secret_access_key": "not a secret", + } + ) + assert client_d != client_a - pass + reset_s3_client() + assert s3_utils._s3_client is None # noqa: SLF001 # split_s3_path - PATH = "path" TO = "to" TO_FILE = "to/file.txt" @@ -239,7 +336,6 @@ def test_get_s3_client_populates_from_environment() -> None: @pytest.mark.parametrize("invalid_path", list(INVALID_PATH_ERRORS.keys())) -@pytest.mark.s3 def test_split_s3_path_errors(invalid_path: str) -> None: """Ensure that an error is thrown if an invalid s3 path is passed in.""" with pytest.raises(ValueError, match=INVALID_PATH_ERRORS[invalid_path]): @@ -247,7 +343,6 @@ def test_split_s3_path_errors(invalid_path: str) -> None: @pytest.mark.parametrize("valid_path", list(EXPECTED.keys())) -@pytest.mark.s3 def test_split_s3_path_success(valid_path: str) -> None: """Verify that a valid path is correctly split into bucket and key.""" (bucket, path) = split_s3_path(valid_path) @@ -703,14 +798,14 @@ def wrapper(*args, **kwargs): @pytest.fixture -def mocked_s3_client_no_checksum(mock_s3_client: Any) -> Generator[Any, Any]: +def mocked_s3_client_no_checksum(mock_s3_client: Any) -> Any: """Yield the mocked S3 client with copy_object patched to strip ChecksumAlgorithm. This works around the moto limitation of not supporting CRC64NVME checksums, allowing copy_object calls that include ChecksumAlgorithm to succeed. """ mock_s3_client.copy_object = strip_checksum_algorithm(mock_s3_client.copy_object) - yield mock_s3_client + return mock_s3_client # copy_object @@ -752,8 +847,6 @@ def test_copy_object_source_bucket_nonexistent() -> None: # copy_directory tests - - def put_objects(mock_s3_client: Any, bucket: str, keys: list[str], body: bytes = b"data") -> None: """Helper to seed objects into a bucket.""" for key in keys: diff --git a/uv.lock b/uv.lock index 0890c787..40088bf8 100644 --- a/uv.lock +++ b/uv.lock @@ -2310,6 +2310,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, ] +[[package]] +name = "nest-asyncio" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, +] + [[package]] name = "numpy" version = "2.4.6" From 8466dcbed646c71585b7ebfa21e30e804724a959 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Fri, 29 May 2026 06:36:20 -0700 Subject: [PATCH 090/128] fix uv lock file --- pyproject.toml | 2 +- uv.lock | 944 ++++++++++++++++++++++++++----------------------- 2 files changed, 507 insertions(+), 439 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 28c960ad..da58937d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ "click>=8.4.1", "defusedxml>=0.7.1", "delta-spark>=4.1.0", - "dlt[deltalake,duckdb,filesystem,iceberg,parquet]>=1.27.0", + "dlt[deltalake,duckdb,filesystem,parquet]>=1.27.0", "frictionless[aws]>=5.19.0", "frozendict>=2.4.7", "ipykernel>=7.2.0", diff --git a/uv.lock b/uv.lock index 40088bf8..08956bc1 100644 --- a/uv.lock +++ b/uv.lock @@ -14,7 +14,7 @@ resolution-markers = [ [[package]] name = "aiobotocore" -version = "3.4.0" +version = "3.7.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -25,9 +25,9 @@ dependencies = [ { name = "python-dateutil" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b8/50/a48ed11b15f926ce3dbb33e7fb0f25af17dbb99bcb7ae3b30c763723eca7/aiobotocore-3.4.0.tar.gz", hash = "sha256:a918b5cb903f81feba7e26835aed4b5e6bb2d0149d7f42bb2dd7d8089e3d9000", size = 122360, upload-time = "2026-04-07T06:12:24.884Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/75/42cce839c2ec263ff74b10b650fe36b066fbb124cbee6f247eac0983e1ab/aiobotocore-3.7.0.tar.gz", hash = "sha256:c64d871ed5491a6571948dd48eabd185b46c6c23b64e3afd0c059fc7593ada30", size = 127054, upload-time = "2026-05-09T10:02:52.332Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/df/d8/ce9386e6d76ea79e61dee15e62aa48cff6be69e89246b0ac4a11857cb02c/aiobotocore-3.4.0-py3-none-any.whl", hash = "sha256:26290eb6830ea92d8a6f5f90b56e9f5cedd6d126074d5db63b195e281d982465", size = 88018, upload-time = "2026-04-07T06:12:22.684Z" }, + { url = "https://files.pythonhosted.org/packages/90/5f/85535dfb3cfd6442d66d1df1694062c5d6df02f895329e7e120b2a3d2b8b/aiobotocore-3.7.0-py3-none-any.whl", hash = "sha256:680bde7c64679a821a9312641b759d9497f790ba8b2e88c6959e6273ee765b8e", size = 89539, upload-time = "2026-05-09T10:02:50.389Z" }, ] [[package]] @@ -148,7 +148,7 @@ wheels = [ [[package]] name = "anthropic" -version = "0.104.1" +version = "0.105.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -160,9 +160,9 @@ dependencies = [ { name = "sniffio" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/22/c7/7a655b948916f777354648ce979f68b94d5b8dbdb5f61fed1f37fad9378c/anthropic-0.104.1.tar.gz", hash = "sha256:17362b6c45f527afcc9b0fdf62011ffd359726ab2ebcb1978ea0cc41bd8d8d40", size = 850081, upload-time = "2026-05-22T15:36:57.432Z" } +sdist = { url = "https://files.pythonhosted.org/packages/46/46/47581b8c689c743ceabf6a0f9ff48472160900ce802d26c0fb50423997b3/anthropic-0.105.2.tar.gz", hash = "sha256:0e26b90841c2dced7cc6e98d21d5517d0be33f1876b8e779f478202e28bcaa07", size = 853789, upload-time = "2026-05-29T00:21:14.104Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b8/12/d9ab42790494d7c428391a46cd28492395566a6a8ccb138d681978594455/anthropic-0.104.1-py3-none-any.whl", hash = "sha256:35c8cb456f5a4405aafe1f10f03f6fcc54fa51fa8ec01d655cc4b437d120e9b7", size = 832996, upload-time = "2026-05-22T15:36:59.519Z" }, + { url = "https://files.pythonhosted.org/packages/83/75/be0c357e33a5a56c8f9db5b4212f886138d2bf59c0952d858f6b75d710ef/anthropic-0.105.2-py3-none-any.whl", hash = "sha256:e53ed5f6bf36fb1ecb9b25d8634cfd30e02fab9fb3374a0c2d5c585874757230", size = 837507, upload-time = "2026-05-29T00:21:15.528Z" }, ] [[package]] @@ -296,30 +296,42 @@ wheels = [ [[package]] name = "awscrt" -version = "0.31.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f6/05/1697c67ad80be475d5deb8961182d10b4a93d29f1cf9f6fdea169bda88c3/awscrt-0.31.2.tar.gz", hash = "sha256:552555de1beff02d72a1f6d384cd49c5a7c283418310eae29d21bcb749c65792", size = 38169245, upload-time = "2026-02-13T10:27:06.441Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/af/8a/c5f8b1a4a3dc37b6b1a2a663887ac2fe67b1e6c877f50e68aa0ac86b74be/awscrt-0.31.2-cp311-abi3-macosx_10_15_universal2.whl", hash = "sha256:49c003d7fe40002dc4e26500f6cc63c61b399d44b4e38e66b5065845d296d230", size = 3457136, upload-time = "2026-02-13T10:26:20.412Z" }, - { url = "https://files.pythonhosted.org/packages/e3/bd/bde40ce3ae7d5d08030ac59442b6121ce8b078651dfec87756ff9f7a4d4e/awscrt-0.31.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ac9d662de99c2f1393011cde357d0c8730aef9df4eedf258505bdf6ff20a3c01", size = 3888743, upload-time = "2026-02-13T10:26:21.563Z" }, - { url = "https://files.pythonhosted.org/packages/1b/88/f4251a2028f0cafee9806060a8538e6b4909bb5136584ba4d219c6d5bf04/awscrt-0.31.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8387e72f856b7a92f7d08ff9a1dfa6960d6e9ed39509c63c5905e240071af23e", size = 4175553, upload-time = "2026-02-13T10:26:22.993Z" }, - { url = "https://files.pythonhosted.org/packages/e2/e4/9aaaaed4016ec142cac0ff01f6a8f5c9bed0c5d3d2dfcc3a269e66dca6e6/awscrt-0.31.2-cp311-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:43b3f774afd5dc2471d38490a16ed3e789814f120b9552c76920cb2fb812376f", size = 3792457, upload-time = "2026-02-13T10:26:24.328Z" }, - { url = "https://files.pythonhosted.org/packages/0e/94/3d8e8732854d7cf9a6468e3074c3d2151cea95699fa10cdda7df86d4e4e4/awscrt-0.31.2-cp311-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:10d5541726b87246fbfeb4c70036373b7aba8b40f489e5ae886eabc09a68ef38", size = 4031362, upload-time = "2026-02-13T10:26:25.763Z" }, - { url = "https://files.pythonhosted.org/packages/bd/21/0d09bc1d1c193026322352f445271ae3f60ca62d8f048d6f07a000a1d869/awscrt-0.31.2-cp311-abi3-win32.whl", hash = "sha256:14e28cabf7857cfe6d82548821410c688e772a819dbf15d167359d7bc54cdb8d", size = 4027275, upload-time = "2026-02-13T10:26:27.248Z" }, - { url = "https://files.pythonhosted.org/packages/8a/d4/94075b06d37b80727738afd24548ac2a20ca6980822a3ce1265850e00b53/awscrt-0.31.2-cp311-abi3-win_amd64.whl", hash = "sha256:ebd98aaaf348334f72d3a38aed18c29b808fe978c295e7c6bc2e21deac5126c8", size = 4177352, upload-time = "2026-02-13T10:26:28.529Z" }, - { url = "https://files.pythonhosted.org/packages/95/eb/9ce53bd498050049ef88e9d074fac6bbe19301ba9593d6f7a834a2321a68/awscrt-0.31.2-cp313-abi3-macosx_10_15_universal2.whl", hash = "sha256:3eb623d0abfbbe5e6666b9c39780737b472766b0e01168296b048a27ef9d13e8", size = 3455722, upload-time = "2026-02-13T10:26:29.818Z" }, - { url = "https://files.pythonhosted.org/packages/9d/88/c9ceaa77cd0384c6f50cc1854fcf5dc0b1aa364349aeb18e1c2d1d6ffdd2/awscrt-0.31.2-cp313-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:03de99bd3077e1b3bbcd1eca9d06a735fdb8fd47b2af8b1d464d43ede00f125a", size = 3880501, upload-time = "2026-02-13T10:26:30.983Z" }, - { url = "https://files.pythonhosted.org/packages/98/ca/27858b8de6a1bbb4e4df035a272b6e80d214b83bc6d6998b286df82be1b5/awscrt-0.31.2-cp313-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1a4adc5ff6eae8a46f5bca4ed70ad68d36f1e272e2fcd60afeef71b4d02afe06", size = 4170259, upload-time = "2026-02-13T10:26:32.261Z" }, - { url = "https://files.pythonhosted.org/packages/da/25/c0668247ac856ab6d033b7ac7ee3f4f32b6628a7900542b303eede4685e0/awscrt-0.31.2-cp313-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:83d45c3ee9e1fe10c2d316b93402157199edb5d20b1584facf24f82981b55190", size = 3783744, upload-time = "2026-02-13T10:26:33.523Z" }, - { url = "https://files.pythonhosted.org/packages/91/52/3ac02206875947a7ed388ba176e7194f77a9a03430333584e63c8011d0c9/awscrt-0.31.2-cp313-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:a1d1f3e07cdd926bbc9a3715826e5794217780e7a326c329bdbf453533d2141a", size = 4026326, upload-time = "2026-02-13T10:26:34.799Z" }, - { url = "https://files.pythonhosted.org/packages/80/16/6256dd1f1bb4172dde19d0b3e35f1fc4eec9c296f214246a7a6628ed03b4/awscrt-0.31.2-cp313-abi3-win32.whl", hash = "sha256:cf02b5db1181811f5e7c70e772986ef4a6577f722a6b3222842ae377df41d261", size = 4021950, upload-time = "2026-02-13T10:26:36.067Z" }, - { url = "https://files.pythonhosted.org/packages/62/cf/14a9357d992338cd05a6389c9f6c51a5fc1e1c5e421fe061bf528f15374c/awscrt-0.31.2-cp313-abi3-win_amd64.whl", hash = "sha256:4b459be11d9aba47d3cb37e10e97702eed2a2858aa381e2586f7f73d15d85bdf", size = 4172066, upload-time = "2026-02-13T10:26:37.436Z" }, +version = "0.32.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/92/cb/980fe60c4209af71d036276217f8b9f372f958e290c15d2849a3de4dcd23/awscrt-0.32.2.tar.gz", hash = "sha256:a4f48805e8a66237923f03b7b692d213994cff42d1ff08125d1d60c74fcaf872", size = 36862073, upload-time = "2026-04-24T22:59:55.835Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/99/a6c712a2f638c766b0879da0eacd9fe5695c64deb89c0357bcc4f1f4df9b/awscrt-0.32.2-cp311-abi3-macosx_10_15_universal2.whl", hash = "sha256:32785f54d0786e07b6491b51f9c2f75ea9e17decd39bb6b66fdc60cd871a49ef", size = 3406247, upload-time = "2026-04-24T22:58:46.268Z" }, + { url = "https://files.pythonhosted.org/packages/a7/03/665c81f3b9d3c56fcdfb8f353c22501bc538d192c431cfaaf307f867d404/awscrt-0.32.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f8a586c52f41ff14c2f1b8afeb764e231ad3d66acfd42a6b9fb6c8afd8da8fe2", size = 3906890, upload-time = "2026-04-24T22:58:47.96Z" }, + { url = "https://files.pythonhosted.org/packages/31/ad/5db0691a0c72d55a17c03c47149cf15d4602b83b470355c322c9a7f115e8/awscrt-0.32.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3ef1664588d35dcad2115377120667e689cd7a517da52a481373c9536811ed96", size = 4196631, upload-time = "2026-04-24T22:58:49.244Z" }, + { url = "https://files.pythonhosted.org/packages/9a/66/63a4654b2996158f33e88d74d79178a5812d94a7e0d6c94ec475115dff18/awscrt-0.32.2-cp311-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:7ed7c209136650fe25659436bb4150e5af6eb43d71a0bf294f0bf414428736ee", size = 3818090, upload-time = "2026-04-24T22:58:50.657Z" }, + { url = "https://files.pythonhosted.org/packages/cd/26/b8fee227918465830a0bbba48f54ebf0a5029c3a7d11d4fa973838a79262/awscrt-0.32.2-cp311-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cfd437122d5a2ec7eb9fffaf2cd8b96543d4a0d7e906b9515b79672005a1607a", size = 4056453, upload-time = "2026-04-24T22:58:52.373Z" }, + { url = "https://files.pythonhosted.org/packages/e3/26/f5b9e9677bf90d1840d4db1513ba92e73601ef82e61a16e747a83493d35c/awscrt-0.32.2-cp311-abi3-win32.whl", hash = "sha256:5197d1e4e780d755632f79f8d32a09a30a9101cccb51ca1694fe25c711b2e801", size = 4066027, upload-time = "2026-04-24T22:58:53.752Z" }, + { url = "https://files.pythonhosted.org/packages/57/0b/fd1798551ecdc8a28d61ecdeda248f99faa3457f83aefa10ab797f466889/awscrt-0.32.2-cp311-abi3-win_amd64.whl", hash = "sha256:80420aa19c074a4c0335f2bd0e4aee3381fa452328d937795a1e0c779f0c052d", size = 4221984, upload-time = "2026-04-24T22:58:55.115Z" }, + { url = "https://files.pythonhosted.org/packages/7b/a3/075e29b39945f398adfb5d5ee4d533e00d37c0f5c68d79b250a1bdb087ba/awscrt-0.32.2-cp313-abi3-macosx_10_15_universal2.whl", hash = "sha256:d2f7aee3bce261ab1ceba1fac404de4d496aa866237161d4257cef92bff9d828", size = 3404901, upload-time = "2026-04-24T22:58:56.492Z" }, + { url = "https://files.pythonhosted.org/packages/e3/41/1c4783b32bf4ec7383156787570ea1221c95c037c2c0f11cbc9e9529ba48/awscrt-0.32.2-cp313-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ff0fff9c2b613d7fabc298b0fd81f0d7056353f3d20271a852a719c5b2f7ccf4", size = 3897627, upload-time = "2026-04-24T22:58:58.144Z" }, + { url = "https://files.pythonhosted.org/packages/af/2d/5736128847ff4a877d50720ea7a48d7e50a56b78741919e4b6ffabffb1ad/awscrt-0.32.2-cp313-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:79bb11d5d1dfdcfd867aac4a026bee11afbd2154279e12b66588442d8c14bdf7", size = 4189579, upload-time = "2026-04-24T22:58:59.56Z" }, + { url = "https://files.pythonhosted.org/packages/90/f5/064b23d4c927e9b63725ee60c88edb75a1e8f5fd1e97d56d4d6a63fe954b/awscrt-0.32.2-cp313-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:47330f421948207a122092042f235cf82d48fa145c446ff4db12cc8cd3a418b6", size = 3809647, upload-time = "2026-04-24T22:59:00.884Z" }, + { url = "https://files.pythonhosted.org/packages/f5/36/f14ea531cccb6ff85c4d50c9e79e61030675520a393b4af23685d24ea018/awscrt-0.32.2-cp313-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:5ec4d8200eb0158b60e35fbb65faa96ef1eb7763f6ce1192827886ee24c6df99", size = 4051532, upload-time = "2026-04-24T22:59:02.593Z" }, + { url = "https://files.pythonhosted.org/packages/64/aa/b12e721c8148a1bbc58a22cbd204d88d0a6263331049d40c057f9dcd3e11/awscrt-0.32.2-cp313-abi3-win32.whl", hash = "sha256:a81f30a501d2eb6ba52c769cd6ecb3f7005512fba4a533211dc717e1115b0d94", size = 4062484, upload-time = "2026-04-24T22:59:04.251Z" }, + { url = "https://files.pythonhosted.org/packages/5c/26/9f5f23647465a4fe1b28afce334e54a4264e23b69365024c28955f0c4119/awscrt-0.32.2-cp313-abi3-win_amd64.whl", hash = "sha256:8d731edda20dce5afc15a04731d91136a31779a244672d1f0a292a8b04aa0fd3", size = 4220425, upload-time = "2026-04-24T22:59:05.627Z" }, + { url = "https://files.pythonhosted.org/packages/dd/76/18077410c1d7b524a7a7486550bc969218f6575288f66e285157dc78e143/awscrt-0.32.2-cp313-cp313t-macosx_10_15_universal2.whl", hash = "sha256:f2a085d0dd4eef974f2ae5467ae3717d2fc08dd8cd508c4fd7a5acb658c68616", size = 3414715, upload-time = "2026-04-24T22:59:07.01Z" }, + { url = "https://files.pythonhosted.org/packages/d3/12/465d72ea6afffc7829f7f48f408a707d4dab9e3f2b694f4e0e63e011478a/awscrt-0.32.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d8a4e52f7312e5e435461119aa903f6424e9996d93a040101fb1eb7b9c4e58dc", size = 4028634, upload-time = "2026-04-24T22:59:08.303Z" }, + { url = "https://files.pythonhosted.org/packages/e0/ee/2e9d3716cf6a24f30b85af24691d473c913c119dc996bcd8c9b2b3fe2c17/awscrt-0.32.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6cb3c858e96ba023de691a3b6478bed9fb59085433042dff78c42e59eed19cf2", size = 4311846, upload-time = "2026-04-24T22:59:10.02Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ce/aa08aad92e48929cfe243ed7da5cf039fbb137c391e84af79e88c848a37a/awscrt-0.32.2-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:7f92b8f40a104a2a87ea5f428b3799220666ec1450b3a90665867d3715749e91", size = 3952242, upload-time = "2026-04-24T22:59:11.632Z" }, + { url = "https://files.pythonhosted.org/packages/42/e3/09ad98aa7dae9cd3ad578c7721b64e9da03f9a5ed444e518c63e82db05a9/awscrt-0.32.2-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:9fd2dbca02f4e22fb5c02d1505327e6e6e9320dbe8ca80fc033cbbb29ed8631a", size = 4187982, upload-time = "2026-04-24T22:59:14.477Z" }, + { url = "https://files.pythonhosted.org/packages/26/be/897b1ad519d83a837d721f4e8a87d98e6f36e5b985fa697f3ffed512a8eb/awscrt-0.32.2-cp313-cp313t-win32.whl", hash = "sha256:023a2f4595804a0f1d61ab49b64dda5612be9bfbe9b13759331e8e31658dda3f", size = 4111958, upload-time = "2026-04-24T22:59:15.857Z" }, + { url = "https://files.pythonhosted.org/packages/71/54/6cca298e8acb6769c8078b39bedbc507f17a3f7fe6ea768d8256d584d4ed/awscrt-0.32.2-cp313-cp313t-win_amd64.whl", hash = "sha256:a2f513f0fb3aafdf7cdf29d7f6f0c46bf4cc7880380c86e88635b7818565e76d", size = 4271679, upload-time = "2026-04-24T22:59:17.425Z" }, + { url = "https://files.pythonhosted.org/packages/c7/09/51fca333d42b53ddb04881f53e3cc0f4462872ac81426fbb34f5d0b1d1fb/awscrt-0.32.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:7404cb551046bbe7cd454ea75f88b4a1b26f018d1b9bea83dbe46c174789d835", size = 3414716, upload-time = "2026-04-24T22:59:18.918Z" }, + { url = "https://files.pythonhosted.org/packages/39/3a/0ae52a8dfb0e3c37f0129232d79307c65e6ee1ea13a3cbdcf301aaad0a6a/awscrt-0.32.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:51d6132e9d70de40da07dfad5f17780a652dd4b351c35ca97c79d0fa0186d645", size = 4028980, upload-time = "2026-04-24T22:59:20.316Z" }, + { url = "https://files.pythonhosted.org/packages/2d/7f/7f52020bfcf29122cad77e936d82abaa1f6857a5a70453e8cc734119f0e2/awscrt-0.32.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:193cc3ecf03a1dd6f989853b6c21549a0a9750c855520fedc8c3c8fbcd32e1bc", size = 4312564, upload-time = "2026-04-24T22:59:21.851Z" }, + { url = "https://files.pythonhosted.org/packages/25/86/2c9b5479e08b8198459d2a781df5524e45d4d0f9c6329bad26979a4d1e85/awscrt-0.32.2-cp314-cp314t-win32.whl", hash = "sha256:25c7e7e6535cb2d2a4d22fd6264f621672d3903491318364dbc59066b63c7186", size = 4195784, upload-time = "2026-04-24T22:59:23.769Z" }, + { url = "https://files.pythonhosted.org/packages/ab/85/a12515514de8969b9e7fa40bb782501a336a8a985cc3093309120a80b627/awscrt-0.32.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a6782c19a00f354c7b232b675f09cde94d1ca37bcf29009b8779b3f6395b27b4", size = 4366435, upload-time = "2026-04-24T22:59:25.53Z" }, ] [[package]] name = "berdl-notebook-utils" version = "0.0.1" -source = { git = "https://github.com/BERDataLakehouse/spark_notebook.git?subdirectory=notebook_utils#dfc45ceea60dab41e5a9fee2eee0b29ffb81bdaa" } +source = { git = "https://github.com/BERDataLakehouse/spark_notebook.git?subdirectory=notebook_utils#afad8c0925cfd2d9fec53cd1bf62b8ac4ed44f84" } dependencies = [ { name = "attrs" }, { name = "cacheout" }, @@ -368,16 +380,16 @@ wheels = [ [[package]] name = "boto3" -version = "1.42.84" +version = "1.43.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, { name = "jmespath" }, { name = "s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/88/89/2d647bd717da55a8cc68602b197f53a5fa36fb95a2f9e76c4aff11a9cfd1/boto3-1.42.84.tar.gz", hash = "sha256:6a84b3293a5d8b3adf827a54588e7dcffcf0a85410d7dadca615544f97d27579", size = 112816, upload-time = "2026-04-06T19:39:07.585Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b7/65/47670987f2f9e181397872c7ee6415b7b95156d711b7eab6c55f66e575bc/boto3-1.43.0.tar.gz", hash = "sha256:80d44a943ef90aba7958ab31d30c155c198acc8a9581b5846b3878b2c8951086", size = 113143, upload-time = "2026-04-29T22:07:49.084Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/31/cdf4326841613d1d181a77b3038a988800fb3373ca50de1639fba9fa87de/boto3-1.42.84-py3-none-any.whl", hash = "sha256:4d03ad3211832484037337292586f71f48707141288d9ac23049c04204f4ab03", size = 140555, upload-time = "2026-04-06T19:39:06.009Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a0/3e6a0b1c1ea6bec76f71473727ef27abf3cd40e9709b3ebcbfbcfaae6f79/boto3-1.43.0-py3-none-any.whl", hash = "sha256:8ebe03754a4b73a5cb6ec2f14cca03ac33bd4760d0adea53da4724845130258b", size = 140497, upload-time = "2026-04-29T22:07:46.216Z" }, ] [package.optional-dependencies] @@ -387,16 +399,16 @@ crt = [ [[package]] name = "botocore" -version = "1.42.84" +version = "1.43.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b4/b7/1c03423843fb0d1795b686511c00ee63fed1234c2400f469aeedfd42212f/botocore-1.42.84.tar.gz", hash = "sha256:234064604c80d9272a5e9f6b3566d260bcaa053a5e05246db90d7eca1c2cf44b", size = 15148615, upload-time = "2026-04-06T19:38:56.673Z" } +sdist = { url = "https://files.pythonhosted.org/packages/28/79/2f4be1896db3db7ccf44504253a175d56b6bd6b669619edc5147d1aa21ea/botocore-1.43.0.tar.gz", hash = "sha256:e933b31a2d644253e1d029d7d39e99ba41b87e29300534f189744cc438cdf928", size = 15286817, upload-time = "2026-04-29T22:07:31.723Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/37/0c0c90361c8a1b9e6c75222ca24ae12996a298c0e18822a72ab229c37207/botocore-1.42.84-py3-none-any.whl", hash = "sha256:15f3fe07dfa6545e46a60c4b049fe2bdf63803c595ae4a4eec90e8f8172764f3", size = 14827061, upload-time = "2026-04-06T19:38:53.613Z" }, + { url = "https://files.pythonhosted.org/packages/bf/4b/afc1fef8a43bafb139f57f73bbd70df82807af5934321e8112ae50668827/botocore-1.43.0-py3-none-any.whl", hash = "sha256:cc5b15eaec3c6eac05d8012cb5ef17ebe891beb88a16ca13c374bfaece1241e6", size = 14970102, upload-time = "2026-04-29T22:07:27Z" }, ] [package.optional-dependencies] @@ -460,7 +472,7 @@ requires-dist = [ { name = "click", specifier = ">=8.4.1" }, { name = "defusedxml", specifier = ">=0.7.1" }, { name = "delta-spark", specifier = ">=4.1.0" }, - { name = "dlt", extras = ["deltalake", "duckdb", "filesystem", "iceberg", "parquet"], specifier = ">=1.27.0" }, + { name = "dlt", extras = ["deltalake", "duckdb", "filesystem", "parquet"], specifier = ">=1.27.0" }, { name = "frictionless", extras = ["aws"], specifier = ">=5.19.0" }, { name = "frozendict", specifier = ">=2.4.7" }, { name = "ipykernel", specifier = ">=7.2.0" }, @@ -509,11 +521,11 @@ dependencies = [ [[package]] name = "certifi" -version = "2026.2.25" +version = "2026.5.20" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/af/2d/7bf41579a8986e348fa033a31cdd0e4121114f6bce2457e8876010b092dd/certifi-2026.2.25.tar.gz", hash = "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7", size = 155029, upload-time = "2026-02-25T02:54:17.342Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422, upload-time = "2026-05-20T11:46:50.073Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" }, + { url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134, upload-time = "2026-05-20T11:46:48.578Z" }, ] [[package]] @@ -675,124 +687,124 @@ wheels = [ [[package]] name = "coverage" -version = "7.13.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9d/e0/70553e3000e345daff267cec284ce4cbf3fc141b6da229ac52775b5428f1/coverage-7.13.5.tar.gz", hash = "sha256:c81f6515c4c40141f83f502b07bbfa5c240ba25bbe73da7b33f1e5b6120ff179", size = 915967, upload-time = "2026-03-17T10:33:18.341Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/74/8c/74fedc9663dcf168b0a059d4ea756ecae4da77a489048f94b5f512a8d0b3/coverage-7.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ec4af212df513e399cf11610cc27063f1586419e814755ab362e50a85ea69c1", size = 219576, upload-time = "2026-03-17T10:31:09.045Z" }, - { url = "https://files.pythonhosted.org/packages/0c/c9/44fb661c55062f0818a6ffd2685c67aa30816200d5f2817543717d4b92eb/coverage-7.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:941617e518602e2d64942c88ec8499f7fbd49d3f6c4327d3a71d43a1973032f3", size = 219942, upload-time = "2026-03-17T10:31:10.708Z" }, - { url = "https://files.pythonhosted.org/packages/5f/13/93419671cee82b780bab7ea96b67c8ef448f5f295f36bf5031154ec9a790/coverage-7.13.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:da305e9937617ee95c2e39d8ff9f040e0487cbf1ac174f777ed5eddd7a7c1f26", size = 250935, upload-time = "2026-03-17T10:31:12.392Z" }, - { url = "https://files.pythonhosted.org/packages/ac/68/1666e3a4462f8202d836920114fa7a5ee9275d1fa45366d336c551a162dd/coverage-7.13.5-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:78e696e1cc714e57e8b25760b33a8b1026b7048d270140d25dafe1b0a1ee05a3", size = 253541, upload-time = "2026-03-17T10:31:14.247Z" }, - { url = "https://files.pythonhosted.org/packages/4e/5e/3ee3b835647be646dcf3c65a7c6c18f87c27326a858f72ab22c12730773d/coverage-7.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02ca0eed225b2ff301c474aeeeae27d26e2537942aa0f87491d3e147e784a82b", size = 254780, upload-time = "2026-03-17T10:31:16.193Z" }, - { url = "https://files.pythonhosted.org/packages/44/b3/cb5bd1a04cfcc49ede6cd8409d80bee17661167686741e041abc7ee1b9a9/coverage-7.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:04690832cbea4e4663d9149e05dba142546ca05cb1848816760e7f58285c970a", size = 256912, upload-time = "2026-03-17T10:31:17.89Z" }, - { url = "https://files.pythonhosted.org/packages/1b/66/c1dceb7b9714473800b075f5c8a84f4588f887a90eb8645282031676e242/coverage-7.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0590e44dd2745c696a778f7bab6aa95256de2cbc8b8cff4f7db8ff09813d6969", size = 251165, upload-time = "2026-03-17T10:31:19.605Z" }, - { url = "https://files.pythonhosted.org/packages/b7/62/5502b73b97aa2e53ea22a39cf8649ff44827bef76d90bf638777daa27a9d/coverage-7.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d7cfad2d6d81dd298ab6b89fe72c3b7b05ec7544bdda3b707ddaecff8d25c161", size = 252908, upload-time = "2026-03-17T10:31:21.312Z" }, - { url = "https://files.pythonhosted.org/packages/7d/37/7792c2d69854397ca77a55c4646e5897c467928b0e27f2d235d83b5d08c6/coverage-7.13.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e092b9499de38ae0fbfbc603a74660eb6ff3e869e507b50d85a13b6db9863e15", size = 250873, upload-time = "2026-03-17T10:31:23.565Z" }, - { url = "https://files.pythonhosted.org/packages/a3/23/bc866fb6163be52a8a9e5d708ba0d3b1283c12158cefca0a8bbb6e247a43/coverage-7.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:48c39bc4a04d983a54a705a6389512883d4a3b9862991b3617d547940e9f52b1", size = 255030, upload-time = "2026-03-17T10:31:25.58Z" }, - { url = "https://files.pythonhosted.org/packages/7d/8b/ef67e1c222ef49860701d346b8bbb70881bef283bd5f6cbba68a39a086c7/coverage-7.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2d3807015f138ffea1ed9afeeb8624fd781703f2858b62a8dd8da5a0994c57b6", size = 250694, upload-time = "2026-03-17T10:31:27.316Z" }, - { url = "https://files.pythonhosted.org/packages/46/0d/866d1f74f0acddbb906db212e096dee77a8e2158ca5e6bb44729f9d93298/coverage-7.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ee2aa19e03161671ec964004fb74b2257805d9710bf14a5c704558b9d8dbaf17", size = 252469, upload-time = "2026-03-17T10:31:29.472Z" }, - { url = "https://files.pythonhosted.org/packages/7a/f5/be742fec31118f02ce42b21c6af187ad6a344fed546b56ca60caacc6a9a0/coverage-7.13.5-cp313-cp313-win32.whl", hash = "sha256:ce1998c0483007608c8382f4ff50164bfc5bd07a2246dd272aa4043b75e61e85", size = 222112, upload-time = "2026-03-17T10:31:31.526Z" }, - { url = "https://files.pythonhosted.org/packages/66/40/7732d648ab9d069a46e686043241f01206348e2bbf128daea85be4d6414b/coverage-7.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:631efb83f01569670a5e866ceb80fe483e7c159fac6f167e6571522636104a0b", size = 222923, upload-time = "2026-03-17T10:31:33.633Z" }, - { url = "https://files.pythonhosted.org/packages/48/af/fea819c12a095781f6ccd504890aaddaf88b8fab263c4940e82c7b770124/coverage-7.13.5-cp313-cp313-win_arm64.whl", hash = "sha256:f4cd16206ad171cbc2470dbea9103cf9a7607d5fe8c242fdf1edf36174020664", size = 221540, upload-time = "2026-03-17T10:31:35.445Z" }, - { url = "https://files.pythonhosted.org/packages/23/d2/17879af479df7fbbd44bd528a31692a48f6b25055d16482fdf5cdb633805/coverage-7.13.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0428cbef5783ad91fe240f673cc1f76b25e74bbfe1a13115e4aa30d3f538162d", size = 220262, upload-time = "2026-03-17T10:31:37.184Z" }, - { url = "https://files.pythonhosted.org/packages/5b/4c/d20e554f988c8f91d6a02c5118f9abbbf73a8768a3048cb4962230d5743f/coverage-7.13.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e0b216a19534b2427cc201a26c25da4a48633f29a487c61258643e89d28200c0", size = 220617, upload-time = "2026-03-17T10:31:39.245Z" }, - { url = "https://files.pythonhosted.org/packages/29/9c/f9f5277b95184f764b24e7231e166dfdb5780a46d408a2ac665969416d61/coverage-7.13.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:972a9cd27894afe4bc2b1480107054e062df08e671df7c2f18c205e805ccd806", size = 261912, upload-time = "2026-03-17T10:31:41.324Z" }, - { url = "https://files.pythonhosted.org/packages/d5/f6/7f1ab39393eeb50cfe4747ae8ef0e4fc564b989225aa1152e13a180d74f8/coverage-7.13.5-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4b59148601efcd2bac8c4dbf1f0ad6391693ccf7a74b8205781751637076aee3", size = 263987, upload-time = "2026-03-17T10:31:43.724Z" }, - { url = "https://files.pythonhosted.org/packages/a0/d7/62c084fb489ed9c6fbdf57e006752e7c516ea46fd690e5ed8b8617c7d52e/coverage-7.13.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:505d7083c8b0c87a8fa8c07370c285847c1f77739b22e299ad75a6af6c32c5c9", size = 266416, upload-time = "2026-03-17T10:31:45.769Z" }, - { url = "https://files.pythonhosted.org/packages/a9/f6/df63d8660e1a0bff6125947afda112a0502736f470d62ca68b288ea762d8/coverage-7.13.5-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:60365289c3741e4db327e7baff2a4aaacf22f788e80fa4683393891b70a89fbd", size = 267558, upload-time = "2026-03-17T10:31:48.293Z" }, - { url = "https://files.pythonhosted.org/packages/5b/02/353ca81d36779bd108f6d384425f7139ac3c58c750dcfaafe5d0bee6436b/coverage-7.13.5-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b88c69c8ef5d4b6fe7dea66d6636056a0f6a7527c440e890cf9259011f5e606", size = 261163, upload-time = "2026-03-17T10:31:50.125Z" }, - { url = "https://files.pythonhosted.org/packages/2c/16/2e79106d5749bcaf3aee6d309123548e3276517cd7851faa8da213bc61bf/coverage-7.13.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5b13955d31d1633cf9376908089b7cebe7d15ddad7aeaabcbe969a595a97e95e", size = 263981, upload-time = "2026-03-17T10:31:51.961Z" }, - { url = "https://files.pythonhosted.org/packages/29/c7/c29e0c59ffa6942030ae6f50b88ae49988e7e8da06de7ecdbf49c6d4feae/coverage-7.13.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:f70c9ab2595c56f81a89620e22899eea8b212a4041bd728ac6f4a28bf5d3ddd0", size = 261604, upload-time = "2026-03-17T10:31:53.872Z" }, - { url = "https://files.pythonhosted.org/packages/40/48/097cdc3db342f34006a308ab41c3a7c11c3f0d84750d340f45d88a782e00/coverage-7.13.5-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:084b84a8c63e8d6fc7e3931b316a9bcafca1458d753c539db82d31ed20091a87", size = 265321, upload-time = "2026-03-17T10:31:55.997Z" }, - { url = "https://files.pythonhosted.org/packages/bb/1f/4994af354689e14fd03a75f8ec85a9a68d94e0188bbdab3fc1516b55e512/coverage-7.13.5-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ad14385487393e386e2ea988b09d62dd42c397662ac2dabc3832d71253eee479", size = 260502, upload-time = "2026-03-17T10:31:58.308Z" }, - { url = "https://files.pythonhosted.org/packages/22/c6/9bb9ef55903e628033560885f5c31aa227e46878118b63ab15dc7ba87797/coverage-7.13.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7f2c47b36fe7709a6e83bfadf4eefb90bd25fbe4014d715224c4316f808e59a2", size = 262688, upload-time = "2026-03-17T10:32:00.141Z" }, - { url = "https://files.pythonhosted.org/packages/14/4f/f5df9007e50b15e53e01edea486814783a7f019893733d9e4d6caad75557/coverage-7.13.5-cp313-cp313t-win32.whl", hash = "sha256:67e9bc5449801fad0e5dff329499fb090ba4c5800b86805c80617b4e29809b2a", size = 222788, upload-time = "2026-03-17T10:32:02.246Z" }, - { url = "https://files.pythonhosted.org/packages/e1/98/aa7fccaa97d0f3192bec013c4e6fd6d294a6ed44b640e6bb61f479e00ed5/coverage-7.13.5-cp313-cp313t-win_amd64.whl", hash = "sha256:da86cdcf10d2519e10cabb8ac2de03da1bcb6e4853790b7fbd48523332e3a819", size = 223851, upload-time = "2026-03-17T10:32:04.416Z" }, - { url = "https://files.pythonhosted.org/packages/3d/8b/e5c469f7352651e5f013198e9e21f97510b23de957dd06a84071683b4b60/coverage-7.13.5-cp313-cp313t-win_arm64.whl", hash = "sha256:0ecf12ecb326fe2c339d93fc131816f3a7367d223db37817208905c89bded911", size = 222104, upload-time = "2026-03-17T10:32:06.65Z" }, - { url = "https://files.pythonhosted.org/packages/8e/77/39703f0d1d4b478bfd30191d3c14f53caf596fac00efb3f8f6ee23646439/coverage-7.13.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fbabfaceaeb587e16f7008f7795cd80d20ec548dc7f94fbb0d4ec2e038ce563f", size = 219621, upload-time = "2026-03-17T10:32:08.589Z" }, - { url = "https://files.pythonhosted.org/packages/e2/3e/51dff36d99ae14639a133d9b164d63e628532e2974d8b1edb99dd1ebc733/coverage-7.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9bb2a28101a443669a423b665939381084412b81c3f8c0fcfbac57f4e30b5b8e", size = 219953, upload-time = "2026-03-17T10:32:10.507Z" }, - { url = "https://files.pythonhosted.org/packages/6a/6c/1f1917b01eb647c2f2adc9962bd66c79eb978951cab61bdc1acab3290c07/coverage-7.13.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bd3a2fbc1c6cccb3c5106140d87cc6a8715110373ef42b63cf5aea29df8c217a", size = 250992, upload-time = "2026-03-17T10:32:12.41Z" }, - { url = "https://files.pythonhosted.org/packages/22/e5/06b1f88f42a5a99df42ce61208bdec3bddb3d261412874280a19796fc09c/coverage-7.13.5-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6c36ddb64ed9d7e496028d1d00dfec3e428e0aabf4006583bb1839958d280510", size = 253503, upload-time = "2026-03-17T10:32:14.449Z" }, - { url = "https://files.pythonhosted.org/packages/80/28/2a148a51e5907e504fa7b85490277734e6771d8844ebcc48764a15e28155/coverage-7.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:380e8e9084d8eb38db3a9176a1a4f3c0082c3806fa0dc882d1d87abc3c789247", size = 254852, upload-time = "2026-03-17T10:32:16.56Z" }, - { url = "https://files.pythonhosted.org/packages/61/77/50e8d3d85cc0b7ebe09f30f151d670e302c7ff4a1bf6243f71dd8b0981fa/coverage-7.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e808af52a0513762df4d945ea164a24b37f2f518cbe97e03deaa0ee66139b4d6", size = 257161, upload-time = "2026-03-17T10:32:19.004Z" }, - { url = "https://files.pythonhosted.org/packages/3b/c4/b5fd1d4b7bf8d0e75d997afd3925c59ba629fc8616f1b3aae7605132e256/coverage-7.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e301d30dd7e95ae068671d746ba8c34e945a82682e62918e41b2679acd2051a0", size = 251021, upload-time = "2026-03-17T10:32:21.344Z" }, - { url = "https://files.pythonhosted.org/packages/f8/66/6ea21f910e92d69ef0b1c3346ea5922a51bad4446c9126db2ae96ee24c4c/coverage-7.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:800bc829053c80d240a687ceeb927a94fd108bbdc68dfbe505d0d75ab578a882", size = 252858, upload-time = "2026-03-17T10:32:23.506Z" }, - { url = "https://files.pythonhosted.org/packages/9e/ea/879c83cb5d61aa2a35fb80e72715e92672daef8191b84911a643f533840c/coverage-7.13.5-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:0b67af5492adb31940ee418a5a655c28e48165da5afab8c7fa6fd72a142f8740", size = 250823, upload-time = "2026-03-17T10:32:25.516Z" }, - { url = "https://files.pythonhosted.org/packages/8a/fb/616d95d3adb88b9803b275580bdeee8bd1b69a886d057652521f83d7322f/coverage-7.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c9136ff29c3a91e25b1d1552b5308e53a1e0653a23e53b6366d7c2dcbbaf8a16", size = 255099, upload-time = "2026-03-17T10:32:27.944Z" }, - { url = "https://files.pythonhosted.org/packages/1c/93/25e6917c90ec1c9a56b0b26f6cad6408e5f13bb6b35d484a0d75c9cf000d/coverage-7.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:cff784eef7f0b8f6cb28804fbddcfa99f89efe4cc35fb5627e3ac58f91ed3ac0", size = 250638, upload-time = "2026-03-17T10:32:29.914Z" }, - { url = "https://files.pythonhosted.org/packages/fc/7b/dc1776b0464145a929deed214aef9fb1493f159b59ff3c7eeeedf91eddd0/coverage-7.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:68a4953be99b17ac3c23b6efbc8a38330d99680c9458927491d18700ef23ded0", size = 252295, upload-time = "2026-03-17T10:32:31.981Z" }, - { url = "https://files.pythonhosted.org/packages/ea/fb/99cbbc56a26e07762a2740713f3c8f9f3f3106e3a3dd8cc4474954bccd34/coverage-7.13.5-cp314-cp314-win32.whl", hash = "sha256:35a31f2b1578185fbe6aa2e74cea1b1d0bbf4c552774247d9160d29b80ed56cc", size = 222360, upload-time = "2026-03-17T10:32:34.233Z" }, - { url = "https://files.pythonhosted.org/packages/8d/b7/4758d4f73fb536347cc5e4ad63662f9d60ba9118cb6785e9616b2ce5d7fa/coverage-7.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:2aa055ae1857258f9e0045be26a6d62bdb47a72448b62d7b55f4820f361a2633", size = 223174, upload-time = "2026-03-17T10:32:36.369Z" }, - { url = "https://files.pythonhosted.org/packages/2c/f2/24d84e1dfe70f8ac9fdf30d338239860d0d1d5da0bda528959d0ebc9da28/coverage-7.13.5-cp314-cp314-win_arm64.whl", hash = "sha256:1b11eef33edeae9d142f9b4358edb76273b3bfd30bc3df9a4f95d0e49caf94e8", size = 221739, upload-time = "2026-03-17T10:32:38.736Z" }, - { url = "https://files.pythonhosted.org/packages/60/5b/4a168591057b3668c2428bff25dd3ebc21b629d666d90bcdfa0217940e84/coverage-7.13.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10a0c37f0b646eaff7cce1874c31d1f1ccb297688d4c747291f4f4c70741cc8b", size = 220351, upload-time = "2026-03-17T10:32:41.196Z" }, - { url = "https://files.pythonhosted.org/packages/f5/21/1fd5c4dbfe4a58b6b99649125635df46decdfd4a784c3cd6d410d303e370/coverage-7.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b5db73ba3c41c7008037fa731ad5459fc3944cb7452fc0aa9f822ad3533c583c", size = 220612, upload-time = "2026-03-17T10:32:43.204Z" }, - { url = "https://files.pythonhosted.org/packages/d6/fe/2a924b3055a5e7e4512655a9d4609781b0d62334fa0140c3e742926834e2/coverage-7.13.5-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:750db93a81e3e5a9831b534be7b1229df848b2e125a604fe6651e48aa070e5f9", size = 261985, upload-time = "2026-03-17T10:32:45.514Z" }, - { url = "https://files.pythonhosted.org/packages/d7/0d/c8928f2bd518c45990fe1a2ab8db42e914ef9b726c975facc4282578c3eb/coverage-7.13.5-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9ddb4f4a5479f2539644be484da179b653273bca1a323947d48ab107b3ed1f29", size = 264107, upload-time = "2026-03-17T10:32:47.971Z" }, - { url = "https://files.pythonhosted.org/packages/ef/ae/4ae35bbd9a0af9d820362751f0766582833c211224b38665c0f8de3d487f/coverage-7.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8a7a2049c14f413163e2bdabd37e41179b1d1ccb10ffc6ccc4b7a718429c607", size = 266513, upload-time = "2026-03-17T10:32:50.1Z" }, - { url = "https://files.pythonhosted.org/packages/9c/20/d326174c55af36f74eac6ae781612d9492f060ce8244b570bb9d50d9d609/coverage-7.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1c85e0b6c05c592ea6d8768a66a254bfb3874b53774b12d4c89c481eb78cb90", size = 267650, upload-time = "2026-03-17T10:32:52.391Z" }, - { url = "https://files.pythonhosted.org/packages/7a/5e/31484d62cbd0eabd3412e30d74386ece4a0837d4f6c3040a653878bfc019/coverage-7.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:777c4d1eff1b67876139d24288aaf1817f6c03d6bae9c5cc8d27b83bcfe38fe3", size = 261089, upload-time = "2026-03-17T10:32:54.544Z" }, - { url = "https://files.pythonhosted.org/packages/e9/d8/49a72d6de146eebb0b7e48cc0f4bc2c0dd858e3d4790ab2b39a2872b62bd/coverage-7.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6697e29b93707167687543480a40f0db8f356e86d9f67ddf2e37e2dfd91a9dab", size = 263982, upload-time = "2026-03-17T10:32:56.803Z" }, - { url = "https://files.pythonhosted.org/packages/06/3b/0351f1bd566e6e4dd39e978efe7958bde1d32f879e85589de147654f57bb/coverage-7.13.5-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8fdf453a942c3e4d99bd80088141c4c6960bb232c409d9c3558e2dbaa3998562", size = 261579, upload-time = "2026-03-17T10:32:59.466Z" }, - { url = "https://files.pythonhosted.org/packages/5d/ce/796a2a2f4017f554d7810f5c573449b35b1e46788424a548d4d19201b222/coverage-7.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:32ca0c0114c9834a43f045a87dcebd69d108d8ffb666957ea65aa132f50332e2", size = 265316, upload-time = "2026-03-17T10:33:01.847Z" }, - { url = "https://files.pythonhosted.org/packages/3d/16/d5ae91455541d1a78bc90abf495be600588aff8f6db5c8b0dae739fa39c9/coverage-7.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8769751c10f339021e2638cd354e13adeac54004d1941119b2c96fe5276d45ea", size = 260427, upload-time = "2026-03-17T10:33:03.945Z" }, - { url = "https://files.pythonhosted.org/packages/48/11/07f413dba62db21fb3fad5d0de013a50e073cc4e2dc4306e770360f6dfc8/coverage-7.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cec2d83125531bd153175354055cdb7a09987af08a9430bd173c937c6d0fba2a", size = 262745, upload-time = "2026-03-17T10:33:06.285Z" }, - { url = "https://files.pythonhosted.org/packages/91/15/d792371332eb4663115becf4bad47e047d16234b1aff687b1b18c58d60ae/coverage-7.13.5-cp314-cp314t-win32.whl", hash = "sha256:0cd9ed7a8b181775459296e402ca4fb27db1279740a24e93b3b41942ebe4b215", size = 223146, upload-time = "2026-03-17T10:33:08.756Z" }, - { url = "https://files.pythonhosted.org/packages/db/51/37221f59a111dca5e85be7dbf09696323b5b9f13ff65e0641d535ed06ea8/coverage-7.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:301e3b7dfefecaca37c9f1aa6f0049b7d4ab8dd933742b607765d757aca77d43", size = 224254, upload-time = "2026-03-17T10:33:11.174Z" }, - { url = "https://files.pythonhosted.org/packages/54/83/6acacc889de8987441aa7d5adfbdbf33d288dad28704a67e574f1df9bcbb/coverage-7.13.5-cp314-cp314t-win_arm64.whl", hash = "sha256:9dacc2ad679b292709e0f5fc1ac74a6d4d5562e424058962c7bb0c658ad25e45", size = 222276, upload-time = "2026-03-17T10:33:13.466Z" }, - { url = "https://files.pythonhosted.org/packages/9e/ee/a4cf96b8ce1e566ed238f0659ac2d3f007ed1d14b181bcb684e19561a69a/coverage-7.13.5-py3-none-any.whl", hash = "sha256:34b02417cf070e173989b3db962f7ed56d2f644307b2cf9d5a0f258e13084a61", size = 211346, upload-time = "2026-03-17T10:33:15.691Z" }, +version = "7.14.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/fd/0ab2772530e946e1be1abd0bc09e647ec9b02e88f0867857601fefca8953/coverage-7.14.1.tar.gz", hash = "sha256:30c08f7d90415aa98b3c990385dea2939b0da55f38515e5b369b83655f8523be", size = 920132, upload-time = "2026-05-26T20:41:36.783Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/9e/5f6d56327c62b185225d145191c607e07515294a0aa6338e58805cd4a5ac/coverage-7.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:be9f2c802dcfce3f71298303aa5dad0dce440a76c52f2f60dacd8656dab78793", size = 220044, upload-time = "2026-05-26T20:39:29.902Z" }, + { url = "https://files.pythonhosted.org/packages/75/92/e82aca356744cbbc0f77a0b623e38918c1872361963413a3bab5d0340393/coverage-7.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6223a72fd0e4c7156353ec0f08a5f93623e1d3034d0e2683b9bb8ea674131b1d", size = 220412, upload-time = "2026-05-26T20:39:31.561Z" }, + { url = "https://files.pythonhosted.org/packages/27/c9/385bde0bf7ed0f4bf3a7ee5367060a86b5d218718cfd6fb943c0f836b34f/coverage-7.14.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7279d2110a28cebc738b6459ecda2771735a4c18465fbbd36b3288fe5ed92247", size = 251412, upload-time = "2026-05-26T20:39:33.337Z" }, + { url = "https://files.pythonhosted.org/packages/51/8c/23faf6a2343a0d17f960a4bd56c43bc7eb4cf312f774dd6ceebd82c7d8fc/coverage-7.14.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9eeb3fcbc13ba40dfbdb22d01d196a28e9cef9ed4c29b60061a1e0e823a9929d", size = 254008, upload-time = "2026-05-26T20:39:35.009Z" }, + { url = "https://files.pythonhosted.org/packages/42/06/36f4aa9ca8a815e6036156e80706a67828bb97bd826948244f6996dda957/coverage-7.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f0cfc27c539f07cf5c0a4cfe211d0b6cae039f8f40526dbaa71944e64b50a7b", size = 255241, upload-time = "2026-05-26T20:39:36.71Z" }, + { url = "https://files.pythonhosted.org/packages/ca/79/95266316352f90f6b1c6736bb413302edfde2453fb32422d3911642691b3/coverage-7.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:221c70f316241a78e77e607c227cefc8808d4e08f28d99c04f35694690e940be", size = 257373, upload-time = "2026-05-26T20:39:38.412Z" }, + { url = "https://files.pythonhosted.org/packages/e3/9c/58316d1f66c488b5fca8a0eb3e98348807813efa8a0d0833b9021be27488/coverage-7.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:da028256b04ec30e5e0114b6f76172938c313991f0a2d3d894271315cf5d5e43", size = 251635, upload-time = "2026-05-26T20:39:40.268Z" }, + { url = "https://files.pythonhosted.org/packages/ef/5a/ca2398a568e16fed7bb713e84ba3603a7164fb65779abe645c565ec890d5/coverage-7.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:76a085d7005236a767e3426148b2c407e53ad61695c562f8a81da2d373324901", size = 253373, upload-time = "2026-05-26T20:39:42.145Z" }, + { url = "https://files.pythonhosted.org/packages/6e/2c/0396562c32deaebe7be51d865b3a41e9a87d7561acafe1a28f53b07e019a/coverage-7.14.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b553d04b5e778a8e56d57eb134aff42a92718ecba45e79c4764ecfa40efd92ff", size = 251341, upload-time = "2026-05-26T20:39:43.907Z" }, + { url = "https://files.pythonhosted.org/packages/fd/8f/a94f9221184c9cae1ee115820e3798e48b6b17777a9f19e46fb9a0c8dc74/coverage-7.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:46f714d2fb8ae2f4f29f23ada7f1e79b759fff5a70f94a1dac23af204c3ec9e4", size = 255497, upload-time = "2026-05-26T20:39:46.166Z" }, + { url = "https://files.pythonhosted.org/packages/71/69/505d70e47db1eaebcd002c39759707621ef184cd6b1ae084d9f41293f323/coverage-7.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:1896f5e19ff3f0431c7ce2172adc54890fd97f86b59ced8ca1649145d9ffe35d", size = 251159, upload-time = "2026-05-26T20:39:48.03Z" }, + { url = "https://files.pythonhosted.org/packages/e0/aa/58681c383aa33a9d2ed40a02d7a22fbf780d1fa4d575396365777828198c/coverage-7.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:62fd185ef9df3c33d1c8178c5af105f762afbad96038de9a4ae100aa6297ca33", size = 252934, upload-time = "2026-05-26T20:39:49.872Z" }, + { url = "https://files.pythonhosted.org/packages/eb/fd/11c928cd6bdffc7074bb5965c173d9ebf517fb00205e1da524b98d29ef92/coverage-7.14.1-cp313-cp313-win32.whl", hash = "sha256:ab4af6352741a604c431c6072fce5bee33bf0f20dc7a56618d6bf6bb89e9810c", size = 222584, upload-time = "2026-05-26T20:39:51.68Z" }, + { url = "https://files.pythonhosted.org/packages/6f/92/fb416fc26d340dcba19518c418d6048e913186e17243982c5e435e41fa7a/coverage-7.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:7af486dabe8954d03b087f0021540897afe084f04e16ff5579e08cc46f871416", size = 223394, upload-time = "2026-05-26T20:39:53.472Z" }, + { url = "https://files.pythonhosted.org/packages/73/c6/02d56e3867972f77d5036de924643f26c056e848f00452cafb4dbc3c29b4/coverage-7.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:2224f89ffd0c5605ccce1ed7a584da162bc7c55f601ab1c946bc9de31a486b42", size = 222015, upload-time = "2026-05-26T20:39:55.374Z" }, + { url = "https://files.pythonhosted.org/packages/4d/9e/fcc77914050df73f7662fa1f00902774c79c075a8388ab334074574bf77e/coverage-7.14.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:de286598cc65d2b489411174b1faec2f5a7775fb3201fd925db2a76b4030f37d", size = 220733, upload-time = "2026-05-26T20:39:57.189Z" }, + { url = "https://files.pythonhosted.org/packages/f7/67/2963cbdaf5cbadec44efa3a1e39eaa1f02df4079585f05387607a221e126/coverage-7.14.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:042c46ded7c288aeb07cf14a28b6c1e10b78fcba40171c3fa1e939377eeef0b5", size = 221086, upload-time = "2026-05-26T20:39:59.019Z" }, + { url = "https://files.pythonhosted.org/packages/c8/c5/8701645574e11881f2f47d8930f98bc48b5d43b25eb5b4430dfc4a2f9f48/coverage-7.14.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f4ddbe407477f04c45115d1a4e5bc480f753553b534d338d4c3358b1cdd0ea52", size = 262381, upload-time = "2026-05-26T20:40:00.822Z" }, + { url = "https://files.pythonhosted.org/packages/7c/28/7a64d73598263e0c5abd5084211a8474488d31b3c552ff531c719dfcff62/coverage-7.14.1-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d13e6725992e2d2fd7d81d4f5241952d13740121dfd501da09201be39b2c003a", size = 264458, upload-time = "2026-05-26T20:40:02.506Z" }, + { url = "https://files.pythonhosted.org/packages/fa/d8/4969179db9f7eb4df218e69540adf829d1c835f59452513d065d15446802/coverage-7.14.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f747dc8edcfe740130f28f32f3995e955494285717e86ee25af51db2219df08a", size = 266884, upload-time = "2026-05-26T20:40:04.421Z" }, + { url = "https://files.pythonhosted.org/packages/a6/78/a45d5794dbc9bafd97afc96a4377c86c7820d78b6cf51b89bc1d4e919275/coverage-7.14.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ced2f09ef276fd58611a1ef502164ad266d2b75174e5a40cabbdb4033f9f6cf2", size = 268022, upload-time = "2026-05-26T20:40:06.298Z" }, + { url = "https://files.pythonhosted.org/packages/21/cb/4f5e354e9e3e67af96bd4e57113e6db6b22298c7168b13eec408a549903d/coverage-7.14.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b84800013769a78ccb9ef4659402e26d06867e337b61ec365f77ad008adea80e", size = 261631, upload-time = "2026-05-26T20:40:08.226Z" }, + { url = "https://files.pythonhosted.org/packages/ec/49/eced49af4cb996d5d8b7e94e736175c513e4facd3398507b89892b4326d8/coverage-7.14.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ea8cd6ca0ee9f616aaef3afc6882e32c2cbf18b00d96313ffd76af650574034d", size = 264443, upload-time = "2026-05-26T20:40:10.137Z" }, + { url = "https://files.pythonhosted.org/packages/f1/d8/5603a88a7c5913a6b54f6cb1a8c46f7b39cbb30f27cd3f492908da09b2d7/coverage-7.14.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:aa5e304a873fabddc11e484e9b6b738bd38bd7bed17b09aa84eecf5332e8b8bb", size = 262069, upload-time = "2026-05-26T20:40:11.999Z" }, + { url = "https://files.pythonhosted.org/packages/f0/59/2ae3cb79da554a06c8619d6c88ea19dd1e4aed4b834b6a83bb1fa243bdc5/coverage-7.14.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:5a1c5215be81035e629d5bc756650634d0bf31991038db7a0eccb90f025ce16d", size = 265780, upload-time = "2026-05-26T20:40:13.858Z" }, + { url = "https://files.pythonhosted.org/packages/af/5f/b130c1dc999031f2648bd25317fbce505ad8d5562079b4ed81e736a84967/coverage-7.14.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:79058c47dae6788504b5effb319961bcd72d7240551464b91d474bc0ed186d69", size = 260970, upload-time = "2026-05-26T20:40:16.142Z" }, + { url = "https://files.pythonhosted.org/packages/87/d1/ec13ccddeb48ec963bdfa72a11224bac2584bd045ba13beca82f8113e9c7/coverage-7.14.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:370c5afae3fa0658e11694a32b24c2778f6bc2d17718121f94ee185e69f26b54", size = 263157, upload-time = "2026-05-26T20:40:18.382Z" }, + { url = "https://files.pythonhosted.org/packages/cf/c2/cd91ead503045161092d3845f7bb95ea2f25131ce96d3e314dd835d91b9c/coverage-7.14.1-cp313-cp313t-win32.whl", hash = "sha256:3758dd0a7f1fa57365ef2e781df0f0731d38b6e3772259d13dae4bd8a958d4b1", size = 223259, upload-time = "2026-05-26T20:40:20.381Z" }, + { url = "https://files.pythonhosted.org/packages/71/9f/1e28d97e6bd2c76b07f38b7c02870f1371255ff6717f54eca578fcbbdd0e/coverage-7.14.1-cp313-cp313t-win_amd64.whl", hash = "sha256:6ff665fb023a77386fe11685190cee1f60a7d635994a30d9b0a061533d470fce", size = 224320, upload-time = "2026-05-26T20:40:22.316Z" }, + { url = "https://files.pythonhosted.org/packages/a9/e0/d936e908f0e1efa55e52b91e01b52f1055cef5e1ab2718493390ed8e2fb8/coverage-7.14.1-cp313-cp313t-win_arm64.whl", hash = "sha256:17a5a241e5997621a956a7f402a7433ef4221e5152809b785bec79e2323799f1", size = 222577, upload-time = "2026-05-26T20:40:24.894Z" }, + { url = "https://files.pythonhosted.org/packages/d6/34/fc2f101b151af3799a101f0550b0454aa008afdc0add677394ec4aa8ea10/coverage-7.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d5ed429d0b8edaac649e889b4ffcedb6c80b06629a3f93050e3dddfb99235bee", size = 220091, upload-time = "2026-05-26T20:40:27.249Z" }, + { url = "https://files.pythonhosted.org/packages/3d/a7/1ebae2ab5b961b5c79bb09fe7b3ac99edb190d8be4a8c510b2cf66f46468/coverage-7.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8011224a62280e50dab346960c03cf47aca1a1e09e608c0fb33fd6e0cc8e9500", size = 220421, upload-time = "2026-05-26T20:40:30.084Z" }, + { url = "https://files.pythonhosted.org/packages/5e/90/92aca9cf0acc95123c96cd1eb1f08917897a7f5dee01e15738922971ec31/coverage-7.14.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:12c42ec1e14f553c4f817e989365982e646e27211f10a0f717855b94a79c8906", size = 251466, upload-time = "2026-05-26T20:40:32.542Z" }, + { url = "https://files.pythonhosted.org/packages/26/2b/78048cbe3b999f6cbf9cc0d90abba6a88a3e0863a8c1c6cbc762f3f8802f/coverage-7.14.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:06144cd511cf2624873a035c5069cf297144f6e77a73ee3d7a55b605ec5efb42", size = 253973, upload-time = "2026-05-26T20:40:34.473Z" }, + { url = "https://files.pythonhosted.org/packages/8e/21/c2e33b29d1cfde484a19d437afc343c6cd30b08d78cbbf9f5aff14e57b2b/coverage-7.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a311d8e1da24be5c1ccf85cbfb06315dbaa1703d5a1eab3f6432c72b837917c8", size = 255318, upload-time = "2026-05-26T20:40:38.154Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ee/aad2f108d63b769121005302f16bf66db8625c88ceaba466942e09a2607e/coverage-7.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c79cead5b5bc584d9c71451cb984d0e3a84e0c0937379c8efcbf27c8d661b851", size = 257633, upload-time = "2026-05-26T20:40:40.164Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f8/11a2c29b4fd76d9849f81d0bb812ec0017a9396df3217214e38934a8c837/coverage-7.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dcbf65f1f66a26cdd88c35cf68fb4729c5d1cd2e88added72420541dfb212034", size = 251488, upload-time = "2026-05-26T20:40:42.631Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b8/9a5820de4b8ac2b71d85e3b5fb49108d7469c665f0e2ad0dd7569023e305/coverage-7.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fd86572566fb40189a8260446158235159bc7a82dfbc87a3b39cf4fb57fcec1c", size = 253329, upload-time = "2026-05-26T20:40:45.208Z" }, + { url = "https://files.pythonhosted.org/packages/6b/ff/f33e4823667e27548e8fd8df44217515303f9808d0ff29817db56f87d990/coverage-7.14.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:7771b601718fdde84832c3a434ca9bbf4ae9adbc49d84198b4110700c3c77c36", size = 251291, upload-time = "2026-05-26T20:40:47.502Z" }, + { url = "https://files.pythonhosted.org/packages/68/9b/489db0ebb209054766b90a9014a45f6d26eb724c02ec21311c3733b5a644/coverage-7.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:39b21e212c55af06fa375e3dbf90a8a8e38792f3a910c580066d23563830ddd5", size = 255564, upload-time = "2026-05-26T20:40:49.372Z" }, + { url = "https://files.pythonhosted.org/packages/27/b5/16bc2d4c2409b23c7737edb68c83bc89e345f378050549fe1d75ac7d34d5/coverage-7.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f2302660e32562a532b442480121aef8aa61a5bdb20b30bf0adab29f10a5a4b4", size = 251107, upload-time = "2026-05-26T20:40:51.677Z" }, + { url = "https://files.pythonhosted.org/packages/7d/0c/2629997469a00cd069d588a41c9dc887610f2775ae89d250c4791e65272a/coverage-7.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03a6f93c1ec3b7f2e77b5dbcc5573a2c21f12529a5c6bbe0f16f72303cc2fa4d", size = 252764, upload-time = "2026-05-26T20:40:54.267Z" }, + { url = "https://files.pythonhosted.org/packages/d2/ee/f78d63c8f079e0d7211c7e2401fa17e311514534ba61bae03e4b287ce4ab/coverage-7.14.1-cp314-cp314-win32.whl", hash = "sha256:8a3ce026d73290f42f08dafecbd82c193a74df280461fbf97300fec51fd133ee", size = 222837, upload-time = "2026-05-26T20:40:56.496Z" }, + { url = "https://files.pythonhosted.org/packages/dc/b9/be539854f93a70dfbeec69117f33ec70dc42ff0b65b5b07ab8d40d04228e/coverage-7.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:114c95ef29302423b87d159075805f4ab973254a2638a5d7d046c94887cc87d7", size = 223650, upload-time = "2026-05-26T20:40:58.351Z" }, + { url = "https://files.pythonhosted.org/packages/fe/9e/24e2842fef40f35ac82ba3a7719c8023d011bf3bf652d0675316a9d088a1/coverage-7.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:a07891c3f4805442b31b71e84ba3cf29ed1aa9a428284e06deeb4b23e5b46343", size = 222218, upload-time = "2026-05-26T20:41:00.321Z" }, + { url = "https://files.pythonhosted.org/packages/0a/1d/ac0a9df5fe31c1e8bdd658074905fc12844a05c1a7e3fdb8417e97c31e23/coverage-7.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:1101a5ebb083aecb625ebb6209d4105b58f647b093cb2dc8122d7b33f743cfe1", size = 220822, upload-time = "2026-05-26T20:41:02.281Z" }, + { url = "https://files.pythonhosted.org/packages/32/cf/f964fd9aff20323f9f1a726c97135f8a76bcd87b92dad141a456a43f3c64/coverage-7.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:851b9e1e4e8a4608e77c79714b2e77c0970d2ed7202a05e92ae407817481887b", size = 221084, upload-time = "2026-05-26T20:41:04.593Z" }, + { url = "https://files.pythonhosted.org/packages/d8/5e/7e5ef2aba844de2b80d678619fcf0841b42e3f37f16411226f3fe4c1016f/coverage-7.14.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d5b89cdfb2ee051b71e8c3c70bd81a9eff81100f736a269136fe1a68efe00474", size = 262454, upload-time = "2026-05-26T20:41:06.641Z" }, + { url = "https://files.pythonhosted.org/packages/64/62/75809bded87015cc4935524218a2a8ed8dd1a8498bfed30a2f4f7a4b4d34/coverage-7.14.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0177614a0370f227888b4e436a7c55686d6a9f90eb1ade2b624ba685a1686e86", size = 264578, upload-time = "2026-05-26T20:41:08.556Z" }, + { url = "https://files.pythonhosted.org/packages/f3/42/d33392dc14633525012d2d504fa1a33b05538bf535f5c1d64675e5754b78/coverage-7.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2d69af5dea2de76fc485a83032a630523f985198b7e25be901ec60181587b01e", size = 266981, upload-time = "2026-05-26T20:41:10.824Z" }, + { url = "https://files.pythonhosted.org/packages/2a/49/0157c4428c2aca7f1e09d5565930586fd5ae36f1655f08b0daa7cf1fcae1/coverage-7.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:35ab22d91de736e8966b980dc355cbcdd2c6dbbcfe275f9a2991bc8a91b3df65", size = 268112, upload-time = "2026-05-26T20:41:12.966Z" }, + { url = "https://files.pythonhosted.org/packages/96/26/86b9ce71f4092b1ed325ce1421698081df1286b833400b6836912834d6e0/coverage-7.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:357d4e32935c36588aaba057d734fa32428c360c9fc2e4442afbf1b646beee6e", size = 261558, upload-time = "2026-05-26T20:41:15Z" }, + { url = "https://files.pythonhosted.org/packages/20/4c/c311210c5472cf5401d8422b0d7812cdd520f24417673afabda6c323faca/coverage-7.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:51bd64741cc6fa065abd300ede1afe5a5291ece9c31da8b24884deda48bcc3f8", size = 264447, upload-time = "2026-05-26T20:41:17.369Z" }, + { url = "https://files.pythonhosted.org/packages/fb/71/59513f8710ed3e6b0ac0a050a5b7e977bb9c9e880354863b5d00d8809256/coverage-7.14.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:9132cd363a68a4c3daa7c8704a654b1e39d3360f6f5b8ddd470608a945236c07", size = 262048, upload-time = "2026-05-26T20:41:19.309Z" }, + { url = "https://files.pythonhosted.org/packages/84/8d/bceed32dc494f5bbf50f775cd2e78ca814953942b5ea28d3c1c3ac316f14/coverage-7.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:07c6290b1697b862c0478eab545eec949a0d0e4d6d03497f446d706da3b4f2de", size = 265781, upload-time = "2026-05-26T20:41:21.559Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c5/9348fe40dbfd4991aaf78df2c6c3098bfb2cc834d1fd362a64b4efef855a/coverage-7.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5ea0c297e27133853b4d8a3eb799bff5a2dbd9f2f41537a240d337ac9b4df890", size = 260896, upload-time = "2026-05-26T20:41:23.428Z" }, + { url = "https://files.pythonhosted.org/packages/ca/92/1ea0f03929da7cf87206b1fa24f4c8e9c158be0455481af29ec0a1f3503f/coverage-7.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:01b7733daad0237daa01ef80fe2dfceffc911e6a17fa7b55d14aa8214eaaaecd", size = 263214, upload-time = "2026-05-26T20:41:25.419Z" }, + { url = "https://files.pythonhosted.org/packages/f6/a9/b2493c054c0e01a643266742ab45e15744e60743f9260cd930c7142b1124/coverage-7.14.1-cp314-cp314t-win32.whl", hash = "sha256:6adc5a36984624a70bf11d7184e20fa0a49aa7c47ffab43804106a1a695ea22e", size = 223624, upload-time = "2026-05-26T20:41:27.795Z" }, + { url = "https://files.pythonhosted.org/packages/fc/bd/3e1e6a57fccd2d7c83fcdf338e93ba98eb85c6e877dd34731ac585375490/coverage-7.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:ddf799247318f34dbcd2efa8c95a8d0642674e926bb1774cf9b63dfd2a389d1c", size = 224728, upload-time = "2026-05-26T20:41:30.098Z" }, + { url = "https://files.pythonhosted.org/packages/bb/d7/31066cf1d2f0c6c797fce911bcfa01dd35642dc6da992a950256097c5860/coverage-7.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:145986fe66647eb489f18d9a997567a3fd358584c4b5a808769113abc07466af", size = 222752, upload-time = "2026-05-26T20:41:32.123Z" }, + { url = "https://files.pythonhosted.org/packages/8a/3c/1a983b9a745d7f83d53f057bcc5bf79ba6a2bbc08266b3f0c7d6fe630c9b/coverage-7.14.1-py3-none-any.whl", hash = "sha256:a252f21c27e38347e60111a3266b03827422a7d5525951aceee313aa68bab1d2", size = 211815, upload-time = "2026-05-26T20:41:34.078Z" }, ] [[package]] name = "cryptography" -version = "46.0.7" +version = "48.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/47/93/ac8f3d5ff04d54bc814e961a43ae5b0b146154c89c61b47bb07557679b18/cryptography-46.0.7.tar.gz", hash = "sha256:e4cfd68c5f3e0bfdad0d38e023239b96a2fe84146481852dffbcca442c245aa5", size = 750652, upload-time = "2026-04-08T01:57:54.692Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/5d/4a8f770695d73be252331e60e526291e3df0c9b27556a90a6b47bccca4c2/cryptography-46.0.7-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:ea42cbe97209df307fdc3b155f1b6fa2577c0defa8f1f7d3be7d31d189108ad4", size = 7179869, upload-time = "2026-04-08T01:56:17.157Z" }, - { url = "https://files.pythonhosted.org/packages/5f/45/6d80dc379b0bbc1f9d1e429f42e4cb9e1d319c7a8201beffd967c516ea01/cryptography-46.0.7-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b36a4695e29fe69215d75960b22577197aca3f7a25b9cf9d165dcfe9d80bc325", size = 4275492, upload-time = "2026-04-08T01:56:19.36Z" }, - { url = "https://files.pythonhosted.org/packages/4a/9a/1765afe9f572e239c3469f2cb429f3ba7b31878c893b246b4b2994ffe2fe/cryptography-46.0.7-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5ad9ef796328c5e3c4ceed237a183f5d41d21150f972455a9d926593a1dcb308", size = 4426670, upload-time = "2026-04-08T01:56:21.415Z" }, - { url = "https://files.pythonhosted.org/packages/8f/3e/af9246aaf23cd4ee060699adab1e47ced3f5f7e7a8ffdd339f817b446462/cryptography-46.0.7-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:73510b83623e080a2c35c62c15298096e2a5dc8d51c3b4e1740211839d0dea77", size = 4280275, upload-time = "2026-04-08T01:56:23.539Z" }, - { url = "https://files.pythonhosted.org/packages/0f/54/6bbbfc5efe86f9d71041827b793c24811a017c6ac0fd12883e4caa86b8ed/cryptography-46.0.7-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cbd5fb06b62bd0721e1170273d3f4d5a277044c47ca27ee257025146c34cbdd1", size = 4928402, upload-time = "2026-04-08T01:56:25.624Z" }, - { url = "https://files.pythonhosted.org/packages/2d/cf/054b9d8220f81509939599c8bdbc0c408dbd2bdd41688616a20731371fe0/cryptography-46.0.7-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:420b1e4109cc95f0e5700eed79908cef9268265c773d3a66f7af1eef53d409ef", size = 4459985, upload-time = "2026-04-08T01:56:27.309Z" }, - { url = "https://files.pythonhosted.org/packages/f9/46/4e4e9c6040fb01c7467d47217d2f882daddeb8828f7df800cb806d8a2288/cryptography-46.0.7-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:24402210aa54baae71d99441d15bb5a1919c195398a87b563df84468160a65de", size = 3990652, upload-time = "2026-04-08T01:56:29.095Z" }, - { url = "https://files.pythonhosted.org/packages/36/5f/313586c3be5a2fbe87e4c9a254207b860155a8e1f3cca99f9910008e7d08/cryptography-46.0.7-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:8a469028a86f12eb7d2fe97162d0634026d92a21f3ae0ac87ed1c4a447886c83", size = 4279805, upload-time = "2026-04-08T01:56:30.928Z" }, - { url = "https://files.pythonhosted.org/packages/69/33/60dfc4595f334a2082749673386a4d05e4f0cf4df8248e63b2c3437585f2/cryptography-46.0.7-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9694078c5d44c157ef3162e3bf3946510b857df5a3955458381d1c7cfc143ddb", size = 4892883, upload-time = "2026-04-08T01:56:32.614Z" }, - { url = "https://files.pythonhosted.org/packages/c7/0b/333ddab4270c4f5b972f980adef4faa66951a4aaf646ca067af597f15563/cryptography-46.0.7-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:42a1e5f98abb6391717978baf9f90dc28a743b7d9be7f0751a6f56a75d14065b", size = 4459756, upload-time = "2026-04-08T01:56:34.306Z" }, - { url = "https://files.pythonhosted.org/packages/d2/14/633913398b43b75f1234834170947957c6b623d1701ffc7a9600da907e89/cryptography-46.0.7-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:91bbcb08347344f810cbe49065914fe048949648f6bd5c2519f34619142bbe85", size = 4410244, upload-time = "2026-04-08T01:56:35.977Z" }, - { url = "https://files.pythonhosted.org/packages/10/f2/19ceb3b3dc14009373432af0c13f46aa08e3ce334ec6eff13492e1812ccd/cryptography-46.0.7-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5d1c02a14ceb9148cc7816249f64f623fbfee39e8c03b3650d842ad3f34d637e", size = 4674868, upload-time = "2026-04-08T01:56:38.034Z" }, - { url = "https://files.pythonhosted.org/packages/1a/bb/a5c213c19ee94b15dfccc48f363738633a493812687f5567addbcbba9f6f/cryptography-46.0.7-cp311-abi3-win32.whl", hash = "sha256:d23c8ca48e44ee015cd0a54aeccdf9f09004eba9fc96f38c911011d9ff1bd457", size = 3026504, upload-time = "2026-04-08T01:56:39.666Z" }, - { url = "https://files.pythonhosted.org/packages/2b/02/7788f9fefa1d060ca68717c3901ae7fffa21ee087a90b7f23c7a603c32ae/cryptography-46.0.7-cp311-abi3-win_amd64.whl", hash = "sha256:397655da831414d165029da9bc483bed2fe0e75dde6a1523ec2fe63f3c46046b", size = 3488363, upload-time = "2026-04-08T01:56:41.893Z" }, - { url = "https://files.pythonhosted.org/packages/7b/56/15619b210e689c5403bb0540e4cb7dbf11a6bf42e483b7644e471a2812b3/cryptography-46.0.7-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:d151173275e1728cf7839aaa80c34fe550c04ddb27b34f48c232193df8db5842", size = 7119671, upload-time = "2026-04-08T01:56:44Z" }, - { url = "https://files.pythonhosted.org/packages/74/66/e3ce040721b0b5599e175ba91ab08884c75928fbeb74597dd10ef13505d2/cryptography-46.0.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:db0f493b9181c7820c8134437eb8b0b4792085d37dbb24da050476ccb664e59c", size = 4268551, upload-time = "2026-04-08T01:56:46.071Z" }, - { url = "https://files.pythonhosted.org/packages/03/11/5e395f961d6868269835dee1bafec6a1ac176505a167f68b7d8818431068/cryptography-46.0.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ebd6daf519b9f189f85c479427bbd6e9c9037862cf8fe89ee35503bd209ed902", size = 4408887, upload-time = "2026-04-08T01:56:47.718Z" }, - { url = "https://files.pythonhosted.org/packages/40/53/8ed1cf4c3b9c8e611e7122fb56f1c32d09e1fff0f1d77e78d9ff7c82653e/cryptography-46.0.7-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b7b412817be92117ec5ed95f880defe9cf18a832e8cafacf0a22337dc1981b4d", size = 4271354, upload-time = "2026-04-08T01:56:49.312Z" }, - { url = "https://files.pythonhosted.org/packages/50/46/cf71e26025c2e767c5609162c866a78e8a2915bbcfa408b7ca495c6140c4/cryptography-46.0.7-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:fbfd0e5f273877695cb93baf14b185f4878128b250cc9f8e617ea0c025dfb022", size = 4905845, upload-time = "2026-04-08T01:56:50.916Z" }, - { url = "https://files.pythonhosted.org/packages/c0/ea/01276740375bac6249d0a971ebdf6b4dc9ead0ee0a34ef3b5a88c1a9b0d4/cryptography-46.0.7-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:ffca7aa1d00cf7d6469b988c581598f2259e46215e0140af408966a24cf086ce", size = 4444641, upload-time = "2026-04-08T01:56:52.882Z" }, - { url = "https://files.pythonhosted.org/packages/3d/4c/7d258f169ae71230f25d9f3d06caabcff8c3baf0978e2b7d65e0acac3827/cryptography-46.0.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:60627cf07e0d9274338521205899337c5d18249db56865f943cbe753aa96f40f", size = 3967749, upload-time = "2026-04-08T01:56:54.597Z" }, - { url = "https://files.pythonhosted.org/packages/b5/2a/2ea0767cad19e71b3530e4cad9605d0b5e338b6a1e72c37c9c1ceb86c333/cryptography-46.0.7-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:80406c3065e2c55d7f49a9550fe0c49b3f12e5bfff5dedb727e319e1afb9bf99", size = 4270942, upload-time = "2026-04-08T01:56:56.416Z" }, - { url = "https://files.pythonhosted.org/packages/41/3d/fe14df95a83319af25717677e956567a105bb6ab25641acaa093db79975d/cryptography-46.0.7-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:c5b1ccd1239f48b7151a65bc6dd54bcfcc15e028c8ac126d3fada09db0e07ef1", size = 4871079, upload-time = "2026-04-08T01:56:58.31Z" }, - { url = "https://files.pythonhosted.org/packages/9c/59/4a479e0f36f8f378d397f4eab4c850b4ffb79a2f0d58704b8fa0703ddc11/cryptography-46.0.7-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:d5f7520159cd9c2154eb61eb67548ca05c5774d39e9c2c4339fd793fe7d097b2", size = 4443999, upload-time = "2026-04-08T01:57:00.508Z" }, - { url = "https://files.pythonhosted.org/packages/28/17/b59a741645822ec6d04732b43c5d35e4ef58be7bfa84a81e5ae6f05a1d33/cryptography-46.0.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fcd8eac50d9138c1d7fc53a653ba60a2bee81a505f9f8850b6b2888555a45d0e", size = 4399191, upload-time = "2026-04-08T01:57:02.654Z" }, - { url = "https://files.pythonhosted.org/packages/59/6a/bb2e166d6d0e0955f1e9ff70f10ec4b2824c9cfcdb4da772c7dd69cc7d80/cryptography-46.0.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:65814c60f8cc400c63131584e3e1fad01235edba2614b61fbfbfa954082db0ee", size = 4655782, upload-time = "2026-04-08T01:57:04.592Z" }, - { url = "https://files.pythonhosted.org/packages/95/b6/3da51d48415bcb63b00dc17c2eff3a651b7c4fed484308d0f19b30e8cb2c/cryptography-46.0.7-cp314-cp314t-win32.whl", hash = "sha256:fdd1736fed309b4300346f88f74cd120c27c56852c3838cab416e7a166f67298", size = 3002227, upload-time = "2026-04-08T01:57:06.91Z" }, - { url = "https://files.pythonhosted.org/packages/32/a8/9f0e4ed57ec9cebe506e58db11ae472972ecb0c659e4d52bbaee80ca340a/cryptography-46.0.7-cp314-cp314t-win_amd64.whl", hash = "sha256:e06acf3c99be55aa3b516397fe42f5855597f430add9c17fa46bf2e0fb34c9bb", size = 3475332, upload-time = "2026-04-08T01:57:08.807Z" }, - { url = "https://files.pythonhosted.org/packages/a7/7f/cd42fc3614386bc0c12f0cb3c4ae1fc2bbca5c9662dfed031514911d513d/cryptography-46.0.7-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:462ad5cb1c148a22b2e3bcc5ad52504dff325d17daf5df8d88c17dda1f75f2a4", size = 7165618, upload-time = "2026-04-08T01:57:10.645Z" }, - { url = "https://files.pythonhosted.org/packages/a5/d0/36a49f0262d2319139d2829f773f1b97ef8aef7f97e6e5bd21455e5a8fb5/cryptography-46.0.7-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:84d4cced91f0f159a7ddacad249cc077e63195c36aac40b4150e7a57e84fffe7", size = 4270628, upload-time = "2026-04-08T01:57:12.885Z" }, - { url = "https://files.pythonhosted.org/packages/8a/6c/1a42450f464dda6ffbe578a911f773e54dd48c10f9895a23a7e88b3e7db5/cryptography-46.0.7-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:128c5edfe5e5938b86b03941e94fac9ee793a94452ad1365c9fc3f4f62216832", size = 4415405, upload-time = "2026-04-08T01:57:14.923Z" }, - { url = "https://files.pythonhosted.org/packages/9a/92/4ed714dbe93a066dc1f4b4581a464d2d7dbec9046f7c8b7016f5286329e2/cryptography-46.0.7-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5e51be372b26ef4ba3de3c167cd3d1022934bc838ae9eaad7e644986d2a3d163", size = 4272715, upload-time = "2026-04-08T01:57:16.638Z" }, - { url = "https://files.pythonhosted.org/packages/b7/e6/a26b84096eddd51494bba19111f8fffe976f6a09f132706f8f1bf03f51f7/cryptography-46.0.7-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cdf1a610ef82abb396451862739e3fc93b071c844399e15b90726ef7470eeaf2", size = 4918400, upload-time = "2026-04-08T01:57:19.021Z" }, - { url = "https://files.pythonhosted.org/packages/c7/08/ffd537b605568a148543ac3c2b239708ae0bd635064bab41359252ef88ed/cryptography-46.0.7-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1d25aee46d0c6f1a501adcddb2d2fee4b979381346a78558ed13e50aa8a59067", size = 4450634, upload-time = "2026-04-08T01:57:21.185Z" }, - { url = "https://files.pythonhosted.org/packages/16/01/0cd51dd86ab5b9befe0d031e276510491976c3a80e9f6e31810cce46c4ad/cryptography-46.0.7-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:cdfbe22376065ffcf8be74dc9a909f032df19bc58a699456a21712d6e5eabfd0", size = 3985233, upload-time = "2026-04-08T01:57:22.862Z" }, - { url = "https://files.pythonhosted.org/packages/92/49/819d6ed3a7d9349c2939f81b500a738cb733ab62fbecdbc1e38e83d45e12/cryptography-46.0.7-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:abad9dac36cbf55de6eb49badd4016806b3165d396f64925bf2999bcb67837ba", size = 4271955, upload-time = "2026-04-08T01:57:24.814Z" }, - { url = "https://files.pythonhosted.org/packages/80/07/ad9b3c56ebb95ed2473d46df0847357e01583f4c52a85754d1a55e29e4d0/cryptography-46.0.7-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:935ce7e3cfdb53e3536119a542b839bb94ec1ad081013e9ab9b7cfd478b05006", size = 4879888, upload-time = "2026-04-08T01:57:26.88Z" }, - { url = "https://files.pythonhosted.org/packages/b8/c7/201d3d58f30c4c2bdbe9b03844c291feb77c20511cc3586daf7edc12a47b/cryptography-46.0.7-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:35719dc79d4730d30f1c2b6474bd6acda36ae2dfae1e3c16f2051f215df33ce0", size = 4449961, upload-time = "2026-04-08T01:57:29.068Z" }, - { url = "https://files.pythonhosted.org/packages/a5/ef/649750cbf96f3033c3c976e112265c33906f8e462291a33d77f90356548c/cryptography-46.0.7-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7bbc6ccf49d05ac8f7d7b5e2e2c33830d4fe2061def88210a126d130d7f71a85", size = 4401696, upload-time = "2026-04-08T01:57:31.029Z" }, - { url = "https://files.pythonhosted.org/packages/41/52/a8908dcb1a389a459a29008c29966c1d552588d4ae6d43f3a1a4512e0ebe/cryptography-46.0.7-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a1529d614f44b863a7b480c6d000fe93b59acee9c82ffa027cfadc77521a9f5e", size = 4664256, upload-time = "2026-04-08T01:57:33.144Z" }, - { url = "https://files.pythonhosted.org/packages/4b/fa/f0ab06238e899cc3fb332623f337a7364f36f4bb3f2534c2bb95a35b132c/cryptography-46.0.7-cp38-abi3-win32.whl", hash = "sha256:f247c8c1a1fb45e12586afbb436ef21ff1e80670b2861a90353d9b025583d246", size = 3013001, upload-time = "2026-04-08T01:57:34.933Z" }, - { url = "https://files.pythonhosted.org/packages/d2/f1/00ce3bde3ca542d1acd8f8cfa38e446840945aa6363f9b74746394b14127/cryptography-46.0.7-cp38-abi3-win_amd64.whl", hash = "sha256:506c4ff91eff4f82bdac7633318a526b1d1309fc07ca76a3ad182cb5b686d6d3", size = 3472985, upload-time = "2026-04-08T01:57:36.714Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/9f/a9/db8f313fdcd85d767d4973515e1db101f9c71f95fced83233de224673757/cryptography-48.0.0.tar.gz", hash = "sha256:5c3932f4436d1cccb036cb0eaef46e6e2db91035166f1ad6505c3c9d5a635920", size = 832984, upload-time = "2026-05-04T22:59:38.133Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/3d/01f6dd9190170a5a241e0e98c2d04be3664a9e6f5b9b872cde63aff1c3dd/cryptography-48.0.0-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:0c558d2cdffd8f4bbb30fc7134c74d2ca9a476f830bb053074498fbc86f41ed6", size = 8001587, upload-time = "2026-05-04T22:57:36.803Z" }, + { url = "https://files.pythonhosted.org/packages/b2/6e/e90527eef33f309beb811cf7c982c3aeffcce8e3edb178baa4ca3ae4a6fa/cryptography-48.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5333311663ea94f75dd408665686aaf426563556bb5283554a3539177e03b8c", size = 4690433, upload-time = "2026-05-04T22:57:40.373Z" }, + { url = "https://files.pythonhosted.org/packages/90/04/673510ed51ddff56575f306cf1617d80411ee76831ccd3097599140efdfe/cryptography-48.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7995ef305d7165c3f11ae07f2517e5a4f1d5c18da1376a0a9ed496336b69e5f3", size = 4710620, upload-time = "2026-05-04T22:57:42.935Z" }, + { url = "https://files.pythonhosted.org/packages/14/d5/e9c4ef932c8d800490c34d8bd589d64a31d5890e27ec9e9ad532be893294/cryptography-48.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:40ba1f85eaa6959837b1d51c9767e230e14612eea4ef110ee8854ada22da1bf5", size = 4696283, upload-time = "2026-05-04T22:57:45.294Z" }, + { url = "https://files.pythonhosted.org/packages/0c/29/174b9dfb60b12d59ecfc6cfa04bc88c21b42a54f01b8aae09bb6e51e4c7f/cryptography-48.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:369a6348999f94bbd53435c894377b20ab95f25a9065c283570e70150d8abc3c", size = 5296573, upload-time = "2026-05-04T22:57:47.933Z" }, + { url = "https://files.pythonhosted.org/packages/95/38/0d29a6fd7d0d1373f0c0c88a04ba20e359b257753ac497564cd660fc1d55/cryptography-48.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a0e692c683f4df67815a2d258b324e66f4738bd7a96a218c826dce4f4bd05d8f", size = 4743677, upload-time = "2026-05-04T22:57:50.067Z" }, + { url = "https://files.pythonhosted.org/packages/30/be/eef653013d5c63b6a490529e0316f9ac14a37602965d4903efed1399f32b/cryptography-48.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:18349bbc56f4743c8b12dc32e2bccb2cf83ee8b69a3bba74ef8ae857e26b3d25", size = 4330808, upload-time = "2026-05-04T22:57:52.301Z" }, + { url = "https://files.pythonhosted.org/packages/84/9e/500463e87abb7a0a0f9f256ec21123ecde0a7b5541a15e840ea54551fd81/cryptography-48.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:7e8eac43dfca5c4cccc6dad9a80504436fca53bb9bc3100a2386d730fbe6b602", size = 4695941, upload-time = "2026-05-04T22:57:54.603Z" }, + { url = "https://files.pythonhosted.org/packages/e3/dc/7303087450c2ec9e7fbb750e17c2abfbc658f23cbd0e54009509b7cc4091/cryptography-48.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9ccdac7d40688ecb5a3b4a604b8a88c8002e3442d6c60aead1db2a89a041560c", size = 5252579, upload-time = "2026-05-04T22:57:57.207Z" }, + { url = "https://files.pythonhosted.org/packages/d0/c0/7101d3b7215edcdc90c45da544961fd8ed2d6448f77577460fa75a8443f7/cryptography-48.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:bd72e68b06bb1e96913f97dd4901119bc17f39d4586a5adf2d3e47bc2b9d58b5", size = 4743326, upload-time = "2026-05-04T22:57:59.535Z" }, + { url = "https://files.pythonhosted.org/packages/ac/d8/5b833bad13016f562ab9d063d68199a4bd121d18458e439515601d3357ec/cryptography-48.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:59baa2cb386c4f0b9905bd6eb4c2a79a69a128408fd31d32ca4d7102d4156321", size = 4826672, upload-time = "2026-05-04T22:58:01.996Z" }, + { url = "https://files.pythonhosted.org/packages/98/e1/7074eb8bf3c135558c73fc2bcf0f5633f912e6fb87e868a55c454080ef09/cryptography-48.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9249e3cd978541d665967ac2cb2787fd6a62bddf1e75b3e347a594d7dacf4f74", size = 4972574, upload-time = "2026-05-04T22:58:03.968Z" }, + { url = "https://files.pythonhosted.org/packages/04/70/e5a1b41d325f797f39427aa44ef8baf0be500065ab6d8e10369d850d4a4f/cryptography-48.0.0-cp311-abi3-win32.whl", hash = "sha256:9c459db21422be75e2809370b829a87eb37f74cd785fc4aa9ea1e5f43b47cda4", size = 3294868, upload-time = "2026-05-04T22:58:06.467Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ac/8ac51b4a5fc5932eb7ee5c517ba7dc8cd834f0048962b6b352f00f41ebf9/cryptography-48.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:5b012212e08b8dd5edc78ef54da83dd9892fd9105323b3993eff6bea65dc21d7", size = 3817107, upload-time = "2026-05-04T22:58:08.845Z" }, + { url = "https://files.pythonhosted.org/packages/6b/84/70e3feea9feea87fd7cbe77efb2712ae1e3e6edf10749dc6e95f4e60e455/cryptography-48.0.0-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:3cb07a3ed6431663cd321ea8a000a1314c74211f823e4177fefa2255e057d1ec", size = 7986556, upload-time = "2026-05-04T22:58:11.172Z" }, + { url = "https://files.pythonhosted.org/packages/89/6e/18e07a618bb5442ba10cf4df16e99c071365528aa570dfcb8c02e25a303b/cryptography-48.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c7378637d7d88016fa6791c159f698b3d3eed28ebf844ac36b9dc04a14dae18", size = 4684776, upload-time = "2026-05-04T22:58:13.712Z" }, + { url = "https://files.pythonhosted.org/packages/be/6a/4ea3b4c6c6759794d5ee2103c304a5076dc4b19ae1f9fe47dba439e159e9/cryptography-48.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc90c0b39b2e3c65ef52c804b72e3c58f8a04ab2a1871272798e5f9572c17d20", size = 4698121, upload-time = "2026-05-04T22:58:16.448Z" }, + { url = "https://files.pythonhosted.org/packages/2f/59/6ff6ad6cae03bb887da2a5860b2c9805f8dac969ef01ce563336c49bd1d1/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:76341972e1eff8b4bea859f09c0d3e64b96ce931b084f9b9b7db8ef364c30eff", size = 4690042, upload-time = "2026-05-04T22:58:18.544Z" }, + { url = "https://files.pythonhosted.org/packages/ca/b4/fc334ed8cfd705aca282fe4d8f5ae64a8e0f74932e9feecb344610cf6e4d/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:55b7718303bf06a5753dcdccf2f3945cf18ad7bffde41b61226e4db31ab89a9c", size = 5282526, upload-time = "2026-05-04T22:58:20.75Z" }, + { url = "https://files.pythonhosted.org/packages/11/08/9f8c5386cc4cd90d8255c7cdd0f5baf459a08502a09de30dc51f553d38dc/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:a64697c641c7b1b2178e573cbc31c7c6684cd56883a478d75143dbb7118036db", size = 4733116, upload-time = "2026-05-04T22:58:23.627Z" }, + { url = "https://files.pythonhosted.org/packages/b8/77/99307d7574045699f8805aa500fa0fb83422d115b5400a064ddd306d7750/cryptography-48.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:561215ea3879cb1cbbf272867e2efda62476f240fb58c64de6b393ae19246741", size = 4316030, upload-time = "2026-05-04T22:58:25.581Z" }, + { url = "https://files.pythonhosted.org/packages/fd/36/a608b98337af3cb2aff4818e406649d30572b7031918b04c87d979495348/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ad64688338ed4bc1a6618076ba75fd7194a5f1797ac60b47afe926285adb3166", size = 4689640, upload-time = "2026-05-04T22:58:27.747Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a6/825010a291b4438aecc1f568bc428189fc1175515223632477c07dc0a6df/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:906cbf0670286c6e0044156bc7d4af9cbb0ef6db9f73e52c3ec56ba6bdde5336", size = 5237657, upload-time = "2026-05-04T22:58:29.848Z" }, + { url = "https://files.pythonhosted.org/packages/b9/09/4e76a09b4caa29aad535ddc806f5d4c5d01885bd978bd984fbc6ca032cae/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:ea8990436d914540a40ab24b6a77c0969695ed52f4a4874c5137ccf7045a7057", size = 4732362, upload-time = "2026-05-04T22:58:32.009Z" }, + { url = "https://files.pythonhosted.org/packages/18/78/444fa04a77d0cb95f417dda20d450e13c56ba8e5220fc892a1658f44f882/cryptography-48.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c18684a7f0cc9a3cb60328f496b8e3372def7c5d2df39ac267878b05565aaaae", size = 4819580, upload-time = "2026-05-04T22:58:34.254Z" }, + { url = "https://files.pythonhosted.org/packages/38/85/ea67067c70a1fd4be2c63d35eeed82658023021affccc7b17705f8527dd2/cryptography-48.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9be5aafa5736574f8f15f262adc81b2a9869e2cfe9014d52a44633905b40d52c", size = 4963283, upload-time = "2026-05-04T22:58:36.376Z" }, + { url = "https://files.pythonhosted.org/packages/75/54/cc6d0f3deac3e81c7f847e8a189a12b6cdd65059b43dad25d4316abd849a/cryptography-48.0.0-cp314-cp314t-win32.whl", hash = "sha256:c17dfe85494deaeddc5ce251aebd1d60bbe6afc8b62071bb0b469431a000124f", size = 3270954, upload-time = "2026-05-04T22:58:38.791Z" }, + { url = "https://files.pythonhosted.org/packages/49/67/cc947e288c0758a4e5473d1dcb743037ab7785541265a969240b8885441a/cryptography-48.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27241b1dc9962e056062a8eef1991d02c3a24569c95975bd2322a8a52c6e5e12", size = 3797313, upload-time = "2026-05-04T22:58:40.746Z" }, + { url = "https://files.pythonhosted.org/packages/f2/63/61d4a4e1c6b6bab6ce1e213cd36a24c415d90e76d78c5eb8577c5541d2e8/cryptography-48.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:58d00498e8933e4a194f3076aee1b4a97dfec1a6da444535755822fe5d8b0b86", size = 7983482, upload-time = "2026-05-04T22:58:43.769Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ac/f5b5995b87770c693e2596559ffafe195b4033a57f14a82268a2842953f3/cryptography-48.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:614d0949f4790582d2cc25553abd09dd723025f0c0e7c67376a1d77196743d6e", size = 4683266, upload-time = "2026-05-04T22:58:46.064Z" }, + { url = "https://files.pythonhosted.org/packages/ec/c6/8b14f67e18338fbc4adb76f66c001f5c3610b3e2d1837f268f47a347dbbb/cryptography-48.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7ce4bfae76319a532a2dc68f82cc32f5676ee792a983187dac07183690e5c66f", size = 4696228, upload-time = "2026-05-04T22:58:48.22Z" }, + { url = "https://files.pythonhosted.org/packages/ea/73/f808fbae9514bd91b47875b003f13e284c8c6bdfd904b7944e803937eec1/cryptography-48.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:2eb992bbd4661238c5a397594c83f5b4dc2bc5b848c365c8f991b6780efcc5c7", size = 4689097, upload-time = "2026-05-04T22:58:50.9Z" }, + { url = "https://files.pythonhosted.org/packages/93/01/d86632d7d28db8ae83221995752eeb6639ffb374c2d22955648cf8d52797/cryptography-48.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:22a5cb272895dce158b2cacdfdc3debd299019659f42947dbdac6f32d68fe832", size = 5283582, upload-time = "2026-05-04T22:58:53.017Z" }, + { url = "https://files.pythonhosted.org/packages/02/e1/50edc7a50334807cc4791fc4a0ce7468b4a1416d9138eab358bfc9a3d70b/cryptography-48.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2b4d59804e8408e2fea7d1fbaf218e5ec984325221db76e6a241a9abd6cdd95c", size = 4730479, upload-time = "2026-05-04T22:58:55.611Z" }, + { url = "https://files.pythonhosted.org/packages/6f/af/99a582b1b1641ff5911ac559beb45097cf79efd4ead4657f578ef1af2d47/cryptography-48.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:984a20b0f62a26f48a3396c72e4bc34c66e356d356bf370053066b3b6d54634a", size = 4326481, upload-time = "2026-05-04T22:58:57.607Z" }, + { url = "https://files.pythonhosted.org/packages/90/ee/89aa26a06ef0a7d7611788ffd571a7c50e368cc6a4d5eef8b4884e866edb/cryptography-48.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5a5ed8fde7a1d09376ca0b40e68cd59c69fe23b1f9768bd5824f54681626032a", size = 4688713, upload-time = "2026-05-04T22:59:00.077Z" }, + { url = "https://files.pythonhosted.org/packages/70/ba/bcb1b0bb7a33d4c7c0c4d4c7874b4a62ae4f56113a5f4baefa362dfb1f0f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:8cd666227ef7af430aa5914a9910e0ddd703e75f039cef0825cd0da71b6b711a", size = 5238165, upload-time = "2026-05-04T22:59:02.317Z" }, + { url = "https://files.pythonhosted.org/packages/c9/70/ca4003b1ce5ca3dc3186ada51908c8a9b9ff7d5cab83cc0d43ee14ec144f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9071196d81abc88b3516ac8cdfad32e2b66dd4a5393a8e68a961e9161ddc6239", size = 4729947, upload-time = "2026-05-04T22:59:05.255Z" }, + { url = "https://files.pythonhosted.org/packages/44/a0/4ec7cf774207905aef1a8d11c3750d5a1db805eb380ee4e16df317870128/cryptography-48.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e2d54c8be6152856a36f0882ab231e70f8ec7f14e93cf87db8a2ed056bf160c", size = 4822059, upload-time = "2026-05-04T22:59:07.802Z" }, + { url = "https://files.pythonhosted.org/packages/1e/75/a2e55f99c16fcac7b5d6c1eb19ad8e00799854d6be5ca845f9259eae1681/cryptography-48.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a5da777e32ffed6f85a7b2b3f7c5cbc88c146bfcd0a1d7baf5fcc6c52ee35dd4", size = 4960575, upload-time = "2026-05-04T22:59:09.851Z" }, + { url = "https://files.pythonhosted.org/packages/b8/23/6e6f32143ab5d8b36ca848a502c4bcd477ae75b9e1677e3530d669062578/cryptography-48.0.0-cp39-abi3-win32.whl", hash = "sha256:77a2ccbbe917f6710e05ba9adaa25fb5075620bf3ea6fb751997875aff4ae4bd", size = 3279117, upload-time = "2026-05-04T22:59:12.019Z" }, + { url = "https://files.pythonhosted.org/packages/9d/9a/0fea98a70cf1749d41d738836f6349d97945f7c89433a259a6c2642eefeb/cryptography-48.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:16cd65b9330583e4619939b3a3843eec1e6e789744bb01e7c7e2e62e33c239c8", size = 3792100, upload-time = "2026-05-04T22:59:14.884Z" }, ] [[package]] @@ -837,11 +849,11 @@ wheels = [ [[package]] name = "decorator" -version = "5.2.1" +version = "5.3.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/fa/6d96a0978d19e17b68d634497769987b16c8f4cd0a7a05048bec693caa6b/decorator-5.2.1.tar.gz", hash = "sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360", size = 56711, upload-time = "2025-02-24T04:41:34.073Z" } +sdist = { url = "https://files.pythonhosted.org/packages/60/8b/32f9823da46cde7df2087faa08cd98d01b908f8dcab982cdba9c84e85355/decorator-5.3.1.tar.gz", hash = "sha256:4cbcdd55a6efadb9dbea26b858f4fb3264567b52d69ca0d25b721b553f60ea82", size = 58084, upload-time = "2026-05-18T06:03:28.057Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a", size = 9190, upload-time = "2025-02-24T04:41:32.565Z" }, + { url = "https://files.pythonhosted.org/packages/05/7f/798705f5296a58ca505d600456748d1be48078eac8a7050d8a98bc9edb89/decorator-5.3.1-py3-none-any.whl", hash = "sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c", size = 10365, upload-time = "2026-05-18T06:03:26.517Z" }, ] [[package]] @@ -855,15 +867,15 @@ wheels = [ [[package]] name = "delta-spark" -version = "4.2.0" +version = "4.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "importlib-metadata" }, { name = "pyspark" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/37/e4/dcf0afa219bc454d5edab65a4defd4dc90e196fdb8c6fb5156ec6d7fc468/delta_spark-4.2.0.tar.gz", hash = "sha256:198079cf74ee40788895a73111900095ebaed2c631205315868b4406c521e0d5", size = 45896, upload-time = "2026-04-10T23:31:30.102Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/09/d394015eb956c4475f6a949fb5fccedf7af19f97e981acbc81629e868a5e/delta_spark-4.1.0.tar.gz", hash = "sha256:98f73c2744f972919e0472974467f85d157810b617341ebf586374d91b8eadc7", size = 36808, upload-time = "2026-02-20T18:37:59.8Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0a/86/766c7cb36a5209c09222be74bad9c047da22690c8e562fab4be29b6a5776/delta_spark-4.2.0-py3-none-any.whl", hash = "sha256:46327eaeb50915aec3b66bb4ab8fb8db7f90774f2a3bb65bb859333cedc6ef8f", size = 53929, upload-time = "2026-04-10T23:31:28.884Z" }, + { url = "https://files.pythonhosted.org/packages/b7/03/2e440efd4a49c8ecfbcb665dea7db94583cf33a365d8480e47fb0aa0dc39/delta_spark-4.1.0-py3-none-any.whl", hash = "sha256:d80f6ebca542df48f257f2535f9c8c21bbfda65771b6ec21be843c0087e1dece", size = 43877, upload-time = "2026-02-20T18:37:58.15Z" }, ] [[package]] @@ -907,7 +919,7 @@ wheels = [ [[package]] name = "dlt" -version = "1.27.0" +version = "1.27.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -936,9 +948,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "tzdata" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bb/80/4cc5a1af42e2b9aa3b4a7a8285f1f28fd889a2270bf7c76f812295bbec5b/dlt-1.27.0.tar.gz", hash = "sha256:cec18a34386150ac1620b2f1b5aa160f08e171ec0609af5da573792277f5218c", size = 1069777, upload-time = "2026-05-18T21:07:19.941Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/df/79ef0330010c40ad94db093846df5c3a67b07eae8b991bd91e3ad1ea9c50/dlt-1.27.1.tar.gz", hash = "sha256:b4cee83b07b417a27450eea5b9d16f1e6042374635673cee39ad858d406a33e1", size = 1070230, upload-time = "2026-05-27T20:17:04.823Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/76/23/26e6dcc639bd74025e6b54bb5af140a0a7582047e6b848d072cb947b271a/dlt-1.27.0-py3-none-any.whl", hash = "sha256:69ac35c56f9b319ab3ae5378df2bc199bf746073df287fdd17994f44efd27ad0", size = 1347146, upload-time = "2026-05-18T21:07:23.631Z" }, + { url = "https://files.pythonhosted.org/packages/d8/3b/1424f5cf91947af89ae9ff1bb14a8cd2c08d1c0de2161c4c916f264a6bac/dlt-1.27.1-py3-none-any.whl", hash = "sha256:b3af5996f742b05537e755664e6343faf58fc1349f4ebf3d4fc06632dfff4905", size = 1347625, upload-time = "2026-05-27T20:17:08.144Z" }, ] [package.optional-dependencies] @@ -1370,23 +1382,23 @@ wheels = [ [[package]] name = "idna" -version = "3.11" +version = "3.17" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/28/99c51f664567218d824af024c0251650fb27e4ca066df188dab0769c5b91/idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f", size = 196048, upload-time = "2026-05-28T14:32:38.55Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, + { url = "https://files.pythonhosted.org/packages/de/a7/f76514cc40ad6234098ecdebda08732d75964776c51a42845b7da10649e2/idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c", size = 65316, upload-time = "2026-05-28T14:32:37.035Z" }, ] [[package]] name = "importlib-metadata" -version = "8.7.1" +version = "9.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "zipp" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f3/49/3b30cad09e7771a4982d9975a8cbf64f00d4a1ececb53297f1d9a7be1b10/importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb", size = 57107, upload-time = "2025-12-21T10:00:19.278Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/5e/f8e9a1d23b9c20a551a8a02ea3637b4642e22c2626e3a13a9a29cdea99eb/importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151", size = 27865, upload-time = "2025-12-21T10:00:18.329Z" }, + { url = "https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl", hash = "sha256:2d21d1cc5a017bd0559e36150c21c830ab1dc304dedd1b7ea85d20f45ef3edd7", size = 27789, upload-time = "2026-03-20T06:42:55.665Z" }, ] [[package]] @@ -1433,7 +1445,7 @@ wheels = [ [[package]] name = "ipython" -version = "9.12.0" +version = "9.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, @@ -1443,13 +1455,14 @@ dependencies = [ { name = "matplotlib-inline" }, { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, { name = "prompt-toolkit" }, + { name = "psutil" }, { name = "pygments" }, { name = "stack-data" }, { name = "traitlets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3a/73/7114f80a8f9cabdb13c27732dce24af945b2923dcab80723602f7c8bc2d8/ipython-9.12.0.tar.gz", hash = "sha256:01daa83f504b693ba523b5a407246cabde4eb4513285a3c6acaff11a66735ee4", size = 4428879, upload-time = "2026-03-27T09:42:45.312Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/87cda5842cf5c31837c06ddb588e11c3c35d8ece89b7a0108c06b8c9b00a/ipython-9.13.0.tar.gz", hash = "sha256:7e834b6afc99f020e3f05966ced34792f40267d64cb1ea9043886dab0dde5967", size = 4430549, upload-time = "2026-04-24T12:24:55.221Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/59/22/906c8108974c673ebef6356c506cebb6870d48cedea3c41e949e2dd556bb/ipython-9.12.0-py3-none-any.whl", hash = "sha256:0f2701e8ee86e117e37f50563205d36feaa259d2e08d4a6bc6b6d74b18ce128d", size = 625661, upload-time = "2026-03-27T09:42:42.831Z" }, + { url = "https://files.pythonhosted.org/packages/b9/86/3060e8029b7cc505cce9a0137431dda81d0a3fde93a8f0f50ee0bf37a795/ipython-9.13.0-py3-none-any.whl", hash = "sha256:57f9d4639e20818d328d287c7b549af3d05f12486ea8f2e7f73e52a36ec4d201", size = 627274, upload-time = "2026-04-24T12:24:53.038Z" }, ] [[package]] @@ -1500,14 +1513,14 @@ wheels = [ [[package]] name = "jedi" -version = "0.19.2" +version = "0.20.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "parso" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/72/3a/79a912fbd4d8dd6fbb02bf69afd3bb72cf0c729bb3063c6f4498603db17a/jedi-0.19.2.tar.gz", hash = "sha256:4770dc3de41bde3966b02eb84fbcf557fb33cce26ad23da12c742fb50ecb11f0", size = 1231287, upload-time = "2024-11-11T01:41:42.873Z" } +sdist = { url = "https://files.pythonhosted.org/packages/46/b7/a3635f6a2d7cf5b5dd98064fc1d5fbbafcb25477bcea204a3a92145d158b/jedi-0.20.0.tar.gz", hash = "sha256:c3f4ccbd276696f4b19c54618d4fb18f9fc24b0aef02acf704b23f487daa1011", size = 3119416, upload-time = "2026-05-01T23:38:47.814Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl", hash = "sha256:a8ef22bde8490f57fe5c7681a3c83cb58874daf72b4784de3cce5b6ef6edb5b9", size = 1572278, upload-time = "2024-11-11T01:41:40.175Z" }, + { url = "https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl", hash = "sha256:7bdd9c2634f56713299976f4cbd59cb3fa92165cc5e05ea811fb253480728b67", size = 4884812, upload-time = "2026-05-01T23:38:43.919Z" }, ] [[package]] @@ -1728,16 +1741,16 @@ wheels = [ [[package]] name = "langchain-anthropic" -version = "1.4.3" +version = "1.4.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anthropic" }, { name = "langchain-core" }, { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fa/e3/d2f9dec95602524b1cfb4be2747ba5bc38d32501b2a56cb4bcb76e80bb45/langchain_anthropic-1.4.3.tar.gz", hash = "sha256:f8a2442463c0629b1b3110eaeaa56fdbdc87df2a802f8c7f5ecf611eb4874ec8", size = 685219, upload-time = "2026-05-03T17:33:27.118Z" } +sdist = { url = "https://files.pythonhosted.org/packages/03/f6/113cf54064b5847367b5a535d80ca2bee34edaea9fddb56ac4466de5e595/langchain_anthropic-1.4.4.tar.gz", hash = "sha256:9ea39ed345f8b13f13bb37549130eedcec2d032eb08b4942642e758c5ba3ece5", size = 687917, upload-time = "2026-05-28T20:20:20.342Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d3/55/482a1968c95275e8be6d8c1e53b54f0f7be0b8b155ce1608c947a95cf543/langchain_anthropic-1.4.3-py3-none-any.whl", hash = "sha256:65466e0f2f95909a009708f2958e917dfdbfab79c612b4484a30866a85e1f291", size = 50389, upload-time = "2026-05-03T17:33:25.671Z" }, + { url = "https://files.pythonhosted.org/packages/c5/e6/d13340529db7fc52ffb1436b18e988df29ebfdd207a2944975136cc82093/langchain_anthropic-1.4.4-py3-none-any.whl", hash = "sha256:10a5d2915e8d4fd8a9f169774e760b089c7f14de044e13259db5f1c2d3d59d9b", size = 51263, upload-time = "2026-05-28T20:20:18.757Z" }, ] [[package]] @@ -1844,14 +1857,14 @@ wheels = [ [[package]] name = "langchain-protocol" -version = "0.0.15" +version = "0.0.16" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4f/24/9777489d6fbbee64af0c8f96d4f840239c408cf694f3394672807dafc490/langchain_protocol-0.0.15.tar.gz", hash = "sha256:9ab2d11ee73944754f10e037e717098d3a6796f0e58afa9cadda6154e7655ade", size = 5862, upload-time = "2026-05-01T22:30:04.748Z" } +sdist = { url = "https://files.pythonhosted.org/packages/36/e7/8300ba22d968653051fd06e3117d783872dddf3dcebdd6b1d386836eb43c/langchain_protocol-0.0.16.tar.gz", hash = "sha256:806c7cdd951b1c4f692fa40fce60821ff0f221d4360e27673ddf2c2b99c2b7ff", size = 5969, upload-time = "2026-05-28T23:05:11.121Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/7a/9c97a7b9cbe4c5dc6a44cdb1545450c28f0c8ce89b9c1f0ee7fbad896263/langchain_protocol-0.0.15-py3-none-any.whl", hash = "sha256:461eb794358f83d5e42635a5797799ffec7b4702314e34edf73ac21e75d3ef79", size = 6982, upload-time = "2026-05-01T22:30:03.877Z" }, + { url = "https://files.pythonhosted.org/packages/1f/9c/06dfcc88d02a6364e8d864c421ddd3736305cb0a6c853f75c302c80fe17c/langchain_protocol-0.0.16-py3-none-any.whl", hash = "sha256:3658c142c5d0fb3a023a4be442ce4c15c6d626aab6135eb79a76dc64ad19c3c3", size = 7037, upload-time = "2026-05-28T23:05:10.163Z" }, ] [[package]] @@ -1924,7 +1937,7 @@ wheels = [ [[package]] name = "langsmith" -version = "0.8.5" +version = "0.8.7" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, @@ -1934,74 +1947,75 @@ dependencies = [ { name = "requests" }, { name = "requests-toolbelt" }, { name = "uuid-utils" }, + { name = "websockets" }, { name = "xxhash" }, { name = "zstandard" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/eb/8883d1158c743d0aac350f09df7880714d27283497e8c80bb9fe3480f165/langsmith-0.8.5.tar.gz", hash = "sha256:3615243d99c12f4047f13042bdc05a373dce232d106a6511b3ca7b48c5af1c2c", size = 4462348, upload-time = "2026-05-15T21:31:41.093Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/8a/9c4a27612f08b607fad56c1c093334d61d3e65d6b1783205118deb1da4bd/langsmith-0.8.7.tar.gz", hash = "sha256:1d0f2b4bcfbf26e9e37bf978dfe23e50d4c90bf1a0f26717879d56f941465a85", size = 4467388, upload-time = "2026-05-29T12:40:16.044Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/23/85/968c88a63e32a59b3e5c68afd2fe114ce0708a125db0be1a85efc25fb2ea/langsmith-0.8.5-py3-none-any.whl", hash = "sha256:efc779f9d450dcaf9d97bc8894f4926276509d6e730e05289af9a64debce06ae", size = 399564, upload-time = "2026-05-15T21:31:39.046Z" }, + { url = "https://files.pythonhosted.org/packages/c6/6e/a99e22455925eccb45aa2c5de1c4773309420bcc0ee1f8fec5bb20e70d9d/langsmith-0.8.7-py3-none-any.whl", hash = "sha256:bfc9bd3276c75201edbf85d12367502aed60752e2da720daee46dccb96f0c3cb", size = 402326, upload-time = "2026-05-29T12:40:13.623Z" }, ] [[package]] name = "lxml" -version = "6.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/28/30/9abc9e34c657c33834eaf6cd02124c61bdf5944d802aa48e69be8da3585d/lxml-6.1.0.tar.gz", hash = "sha256:bfd57d8008c4965709a919c3e9a98f76c2c7cb319086b3d26858250620023b13", size = 4197006, upload-time = "2026-04-18T04:32:51.613Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/08/03/69347590f1cf4a6d5a4944bb6099e6d37f334784f16062234e1f892fdb1d/lxml-6.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a0092f2b107b69601adf562a57c956fbb596e05e3e6651cabd3054113b007e45", size = 8559689, upload-time = "2026-04-18T04:31:57.785Z" }, - { url = "https://files.pythonhosted.org/packages/3f/58/25e00bb40b185c974cfe156c110474d9a8a8390d5f7c92a4e328189bb60e/lxml-6.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fc7140d7a7386e6b545d41b7358f4d02b656d4053f5fa6859f92f4b9c2572c4d", size = 4617892, upload-time = "2026-04-18T04:32:01.78Z" }, - { url = "https://files.pythonhosted.org/packages/f5/54/92ad98a94ac318dc4f97aaac22ff8d1b94212b2ae8af5b6e9b354bf825f7/lxml-6.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:419c58fc92cc3a2c3fa5f78c63dbf5da70c1fa9c1b25f25727ecee89a96c7de2", size = 4923489, upload-time = "2026-04-18T04:33:31.401Z" }, - { url = "https://files.pythonhosted.org/packages/15/3b/a20aecfab42bdf4f9b390590d345857ad3ffd7c51988d1c89c53a0c73faf/lxml-6.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:37fabd1452852636cf38ecdcc9dd5ca4bba7a35d6c53fa09725deeb894a87491", size = 5082162, upload-time = "2026-04-18T04:33:34.262Z" }, - { url = "https://files.pythonhosted.org/packages/45/26/2cdb3d281ac1bd175603e290cbe4bad6eff127c0f8de90bafd6f8548f0fd/lxml-6.1.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2853c8b2170cc6cd54a6b4d50d2c1a8a7aeca201f23804b4898525c7a152cfc", size = 4993247, upload-time = "2026-04-18T04:33:36.674Z" }, - { url = "https://files.pythonhosted.org/packages/f6/05/d735aef963740022a08185c84821f689fc903acb3d50326e6b1e9886cc22/lxml-6.1.0-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8e369cbd690e788c8d15e56222d91a09c6a417f49cbc543040cba0fe2e25a79e", size = 5613042, upload-time = "2026-04-18T04:33:39.205Z" }, - { url = "https://files.pythonhosted.org/packages/ee/b8/ead7c10efff731738c72e59ed6eb5791854879fbed7ae98781a12006263a/lxml-6.1.0-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e69aa6805905807186eb00e66c6d97a935c928275182eb02ee40ba00da9623b2", size = 5228304, upload-time = "2026-04-18T04:33:41.647Z" }, - { url = "https://files.pythonhosted.org/packages/6b/10/e9842d2ec322ea65f0a7270aa0315a53abed06058b88ef1b027f620e7a5f/lxml-6.1.0-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:4bd1bdb8a9e0e2dd229de19b5f8aebac80e916921b4b2c6ef8a52bc131d0c1f9", size = 5341578, upload-time = "2026-04-18T04:33:44.596Z" }, - { url = "https://files.pythonhosted.org/packages/89/54/40d9403d7c2775fa7301d3ddd3464689bfe9ba71acc17dfff777071b4fdc/lxml-6.1.0-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:cbd7b79cdcb4986ad78a2662625882747f09db5e4cd7b2ae178a88c9c51b3dfe", size = 4700209, upload-time = "2026-04-18T04:33:47.552Z" }, - { url = "https://files.pythonhosted.org/packages/85/b2/bbdcc2cf45dfc7dfffef4fd97e5c47b15919b6a365247d95d6f684ef5e82/lxml-6.1.0-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:43e4d297f11080ec9d64a4b1ad7ac02b4484c9f0e2179d9c4ef78e886e747b88", size = 5232365, upload-time = "2026-04-18T04:33:50.249Z" }, - { url = "https://files.pythonhosted.org/packages/48/5a/b06875665e53aaba7127611a7bed3b7b9658e20b22bc2dd217a0b7ab0091/lxml-6.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cc16682cc987a3da00aa56a3aa3075b08edb10d9b1e476938cfdbee8f3b67181", size = 5043654, upload-time = "2026-04-18T04:33:52.71Z" }, - { url = "https://files.pythonhosted.org/packages/e9/9c/e71a069d09641c1a7abeb30e693f828c7c90a41cbe3d650b2d734d876f85/lxml-6.1.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:d6d8efe71429635f0559579092bb5e60560d7b9115ee38c4adbea35632e7fa24", size = 4769326, upload-time = "2026-04-18T04:33:55.244Z" }, - { url = "https://files.pythonhosted.org/packages/cc/06/7a9cd84b3d4ed79adf35f874750abb697dec0b4a81a836037b36e47c091a/lxml-6.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7e39ab3a28af7784e206d8606ec0e4bcad0190f63a492bca95e94e5a4aef7f6e", size = 5635879, upload-time = "2026-04-18T04:33:58.509Z" }, - { url = "https://files.pythonhosted.org/packages/cc/f0/9d57916befc1e54c451712c7ee48e9e74e80ae4d03bdce49914e0aee42cd/lxml-6.1.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:9eb667bf50856c4a58145f8ca2d5e5be160191e79eb9e30855a476191b3c3495", size = 5224048, upload-time = "2026-04-18T04:34:00.943Z" }, - { url = "https://files.pythonhosted.org/packages/99/75/90c4eefda0c08c92221fe0753db2d6699a4c628f76ff4465ec20dea84cc1/lxml-6.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7f4a77d6f7edf9230cee3e1f7f6764722a41604ee5681844f18db9a81ea0ec33", size = 5250241, upload-time = "2026-04-18T04:34:03.365Z" }, - { url = "https://files.pythonhosted.org/packages/5e/73/16596f7e4e38fa33084b9ccbccc22a15f82a290a055126f2c1541236d2ff/lxml-6.1.0-cp313-cp313-win32.whl", hash = "sha256:28902146ffbe5222df411c5d19e5352490122e14447e98cd118907ee3fd6ee62", size = 3596938, upload-time = "2026-04-18T04:31:56.206Z" }, - { url = "https://files.pythonhosted.org/packages/8e/63/981401c5680c1eb30893f00a19641ac80db5d1e7086c62cb4b13ed813038/lxml-6.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:4a1503c56e4e2b38dc76f2f2da7bae69670c0f1933e27cfa34b2fa5876410b16", size = 3995728, upload-time = "2026-04-18T04:31:58.763Z" }, - { url = "https://files.pythonhosted.org/packages/e7/e8/c358a38ac3e541d16a1b527e4e9cb78c0419b0506a070ace11777e5e8404/lxml-6.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:e0af85773850417d994d019741239b901b22c6680206f46a34766926e466141d", size = 3658372, upload-time = "2026-04-18T04:32:03.629Z" }, - { url = "https://files.pythonhosted.org/packages/eb/45/cee4cf203ef0bab5c52afc118da61d6b460c928f2893d40023cfa27e0b80/lxml-6.1.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:ab863fd37458fed6456525f297d21239d987800c46e67da5ef04fc6b3dd93ac8", size = 8576713, upload-time = "2026-04-18T04:32:06.831Z" }, - { url = "https://files.pythonhosted.org/packages/8a/a7/eda05babeb7e046839204eaf254cd4d7c9130ce2bbf0d9e90ea41af5654d/lxml-6.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:6fd8b1df8254ff4fd93fd31da1fc15770bde23ac045be9bb1f87425702f61cc9", size = 4623874, upload-time = "2026-04-18T04:32:10.755Z" }, - { url = "https://files.pythonhosted.org/packages/e7/e9/db5846de9b436b91890a62f29d80cd849ea17948a49bf532d5278ee69a9e/lxml-6.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:47024feaae386a92a146af0d2aeed65229bf6fff738e6a11dda6b0015fb8fd03", size = 4949535, upload-time = "2026-04-18T04:34:06.657Z" }, - { url = "https://files.pythonhosted.org/packages/5a/ba/0d3593373dcae1d68f40dc3c41a5a92f2544e68115eb2f62319a4c2a6500/lxml-6.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3f00972f84450204cd5d93a5395965e348956aaceaadec693a22ec743f8ae3eb", size = 5086881, upload-time = "2026-04-18T04:34:09.556Z" }, - { url = "https://files.pythonhosted.org/packages/43/76/759a7484539ad1af0d125a9afe9c3fb5f82a8779fd1f5f56319d9e4ea2fd/lxml-6.1.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97faa0860e13b05b15a51fb4986421ef7a30f0b3334061c416e0981e9450ca4c", size = 5031305, upload-time = "2026-04-18T04:34:12.336Z" }, - { url = "https://files.pythonhosted.org/packages/dc/b9/c1f0daf981a11e47636126901fd4ab82429e18c57aeb0fc3ad2940b42d8b/lxml-6.1.0-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:972a6451204798675407beaad97b868d0c733d9a74dafefc63120b81b8c2de28", size = 5647522, upload-time = "2026-04-18T04:34:14.89Z" }, - { url = "https://files.pythonhosted.org/packages/31/e6/1f533dcd205275363d9ba3511bcec52fa2df86abf8abe6a5f2c599f0dc31/lxml-6.1.0-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fe022f20bc4569ec66b63b3fb275a3d628d9d32da6326b2982584104db6d3086", size = 5239310, upload-time = "2026-04-18T04:34:17.652Z" }, - { url = "https://files.pythonhosted.org/packages/c3/8c/4175fb709c78a6e315ed814ed33be3defd8b8721067e70419a6cf6f971da/lxml-6.1.0-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:75c4c7c619a744f972f4451bf5adf6d0fb00992a1ffc9fd78e13b0bc817cc99f", size = 5350799, upload-time = "2026-04-18T04:34:20.529Z" }, - { url = "https://files.pythonhosted.org/packages/fd/77/6ffdebc5994975f0dde4acb59761902bd9d9bb84422b9a0bd239a7da9ca8/lxml-6.1.0-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:3648f20d25102a22b6061c688beb3a805099ea4beb0a01ce62975d926944d292", size = 4697693, upload-time = "2026-04-18T04:34:23.541Z" }, - { url = "https://files.pythonhosted.org/packages/f8/f1/565f36bd5c73294602d48e04d23f81ff4c8736be6ba5e1d1ec670ac9be80/lxml-6.1.0-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:77b9f99b17cbf14026d1e618035077060fc7195dd940d025149f3e2e830fbfcb", size = 5250708, upload-time = "2026-04-18T04:34:26.001Z" }, - { url = "https://files.pythonhosted.org/packages/5a/11/a68ab9dd18c5c499404deb4005f4bc4e0e88e5b72cd755ad96efec81d18d/lxml-6.1.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:32662519149fd7a9db354175aa5e417d83485a8039b8aaa62f873ceee7ea4cad", size = 5084737, upload-time = "2026-04-18T04:34:28.32Z" }, - { url = "https://files.pythonhosted.org/packages/ab/78/e8f41e2c74f4af564e6a0348aea69fb6daaefa64bc071ef469823d22cc18/lxml-6.1.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:73d658216fc173cf2c939e90e07b941c5e12736b0bf6a99e7af95459cfe8eabb", size = 4737817, upload-time = "2026-04-18T04:34:30.784Z" }, - { url = "https://files.pythonhosted.org/packages/06/2d/aa4e117aa2ce2f3b35d9ff246be74a2f8e853baba5d2a92c64744474603a/lxml-6.1.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ac4db068889f8772a4a698c5980ec302771bb545e10c4b095d4c8be26749616f", size = 5670753, upload-time = "2026-04-18T04:34:33.675Z" }, - { url = "https://files.pythonhosted.org/packages/08/f5/dd745d50c0409031dbfcc4881740542a01e54d6f0110bd420fa7782110b8/lxml-6.1.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:45e9dfbd1b661eb64ba0d4dbe762bd210c42d86dd1e5bd2bdf89d634231beb43", size = 5238071, upload-time = "2026-04-18T04:34:36.12Z" }, - { url = "https://files.pythonhosted.org/packages/3e/74/ad424f36d0340a904665867dab310a3f1f4c96ff4039698de83b77f44c1f/lxml-6.1.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:89e8d73d09ac696a5ba42ec69787913d53284f12092f651506779314f10ba585", size = 5264319, upload-time = "2026-04-18T04:34:39.035Z" }, - { url = "https://files.pythonhosted.org/packages/53/36/a15d8b3514ec889bfd6aa3609107fcb6c9189f8dc347f1c0b81eded8d87c/lxml-6.1.0-cp314-cp314-win32.whl", hash = "sha256:ebe33f4ec1b2de38ceb225a1749a2965855bffeef435ba93cd2d5d540783bf2f", size = 3657139, upload-time = "2026-04-18T04:32:20.006Z" }, - { url = "https://files.pythonhosted.org/packages/1a/a4/263ebb0710851a3c6c937180a9a86df1206fdfe53cc43005aa2237fd7736/lxml-6.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:398443df51c538bd578529aa7e5f7afc6c292644174b47961f3bf87fe5741120", size = 4064195, upload-time = "2026-04-18T04:32:23.876Z" }, - { url = "https://files.pythonhosted.org/packages/80/68/2000f29d323b6c286de077ad20b429fc52272e44eae6d295467043e56012/lxml-6.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:8c8984e1d8c4b3949e419158fda14d921ff703a9ed8a47236c6eb7a2b6cb4946", size = 3741870, upload-time = "2026-04-18T04:32:27.922Z" }, - { url = "https://files.pythonhosted.org/packages/30/e9/21383c7c8d43799f0da90224c0d7c921870d476ec9b3e01e1b2c0b8237c5/lxml-6.1.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:1081dd10bc6fa437db2500e13993abf7cc30716d0a2f40e65abb935f02ec559c", size = 8827548, upload-time = "2026-04-18T04:32:15.094Z" }, - { url = "https://files.pythonhosted.org/packages/a5/01/c6bc11cd587030dd4f719f65c5657960649fe3e19196c844c75bf32cd0d6/lxml-6.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:dabecc48db5f42ba348d1f5d5afdc54c6c4cc758e676926c7cd327045749517d", size = 4735866, upload-time = "2026-04-18T04:32:18.924Z" }, - { url = "https://files.pythonhosted.org/packages/f3/01/757132fff5f4acf25463b5298f1a46099f3a94480b806547b29ce5e385de/lxml-6.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e3dd5fe19c9e0ac818a9c7f132a5e43c1339ec1cbbfecb1a938bd3a47875b7c9", size = 4969476, upload-time = "2026-04-18T04:34:41.889Z" }, - { url = "https://files.pythonhosted.org/packages/fd/fb/1bc8b9d27ed64be7c8903db6c89e74dc8c2cd9ec630a7462e4654316dc5b/lxml-6.1.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9e7b0a4ca6dcc007a4cef00a761bba2dea959de4bd2df98f926b33c92ca5dfb9", size = 5103719, upload-time = "2026-04-18T04:34:44.797Z" }, - { url = "https://files.pythonhosted.org/packages/d5/e7/5bf82fa28133536a54601aae633b14988e89ed61d4c1eb6b899b023233aa/lxml-6.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d27bbe326c6b539c64b42638b18bc6003a8d88f76213a97ac9ed4f885efeab7", size = 5027890, upload-time = "2026-04-18T04:34:47.634Z" }, - { url = "https://files.pythonhosted.org/packages/2d/20/e048db5d4b4ea0366648aa595f26bb764b2670903fc585b87436d0a5032c/lxml-6.1.0-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4e425db0c5445ef0ad56b0eec54f89b88b2d884656e536a90b2f52aecb4ca86", size = 5596008, upload-time = "2026-04-18T04:34:51.503Z" }, - { url = "https://files.pythonhosted.org/packages/9a/c2/d10807bc8da4824b39e5bd01b5d05c077b6fd01bd91584167edf6b269d22/lxml-6.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b89b098105b8599dc57adac95d1813409ac476d3c948a498775d3d0c6124bfb", size = 5224451, upload-time = "2026-04-18T04:34:54.263Z" }, - { url = "https://files.pythonhosted.org/packages/3c/15/2ebea45bea427e7f0057e9ce7b2d62c5aba20c6b001cca89ed0aadb3ad41/lxml-6.1.0-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:c4a699432846df86cc3de502ee85f445ebad748a1c6021d445f3e514d2cd4b1c", size = 5312135, upload-time = "2026-04-18T04:34:56.818Z" }, - { url = "https://files.pythonhosted.org/packages/31/e2/87eeae151b0be2a308d49a7ec444ff3eb192b14251e62addb29d0bf3778f/lxml-6.1.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:30e7b2ed63b6c8e97cca8af048589a788ab5c9c905f36d9cf1c2bb549f450d2f", size = 4639126, upload-time = "2026-04-18T04:34:59.704Z" }, - { url = "https://files.pythonhosted.org/packages/a3/51/8a3f6a20902ad604dd746ec7b4000311b240d389dac5e9d95adefd349e0c/lxml-6.1.0-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:022981127642fe19866d2907d76241bb07ed21749601f727d5d5dd1ce5d1b773", size = 5232579, upload-time = "2026-04-18T04:35:02.658Z" }, - { url = "https://files.pythonhosted.org/packages/6d/d2/650d619bdbe048d2c3f2c31edb00e35670a5e2d65b4fe3b61bce37b19121/lxml-6.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:23cad0cc86046d4222f7f418910e46b89971c5a45d3c8abfad0f64b7b05e4a9b", size = 5084206, upload-time = "2026-04-18T04:35:05.175Z" }, - { url = "https://files.pythonhosted.org/packages/dd/8a/672ca1a3cbeabd1f511ca275a916c0514b747f4b85bdaae103b8fa92f307/lxml-6.1.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:21c3302068f50d1e8728c67c87ba92aa87043abee517aa2576cca1855326b405", size = 4758906, upload-time = "2026-04-18T04:35:08.098Z" }, - { url = "https://files.pythonhosted.org/packages/be/f1/ef4b691da85c916cb2feb1eec7414f678162798ac85e042fa164419ac05c/lxml-6.1.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:be10838781cb3be19251e276910cd508fe127e27c3242e50521521a0f3781690", size = 5620553, upload-time = "2026-04-18T04:35:11.23Z" }, - { url = "https://files.pythonhosted.org/packages/59/17/94e81def74107809755ac2782fdad4404420f1c92ca83433d117a6d5acf0/lxml-6.1.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:2173a7bffe97667bbf0767f8a99e587740a8c56fdf3befac4b09cb29a80276fd", size = 5229458, upload-time = "2026-04-18T04:35:14.254Z" }, - { url = "https://files.pythonhosted.org/packages/21/55/c4be91b0f830a871fc1b0d730943d56013b683d4671d5198260e2eae722b/lxml-6.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c6854e9cf99c84beb004eecd7d3a3868ef1109bf2b1df92d7bc11e96a36c2180", size = 5247861, upload-time = "2026-04-18T04:35:17.006Z" }, - { url = "https://files.pythonhosted.org/packages/c2/ca/77123e4d77df3cb1e968ade7b1f808f5d3a5c1c96b18a33895397de292c1/lxml-6.1.0-cp314-cp314t-win32.whl", hash = "sha256:00750d63ef0031a05331b9223463b1c7c02b9004cef2346a5b2877f0f9494dd2", size = 3897377, upload-time = "2026-04-18T04:32:07.656Z" }, - { url = "https://files.pythonhosted.org/packages/64/ce/3554833989d074267c063209bae8b09815e5656456a2d332b947806b05ff/lxml-6.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:80410c3a7e3c617af04de17caa9f9f20adaa817093293d69eae7d7d0522836f5", size = 4392701, upload-time = "2026-04-18T04:32:12.113Z" }, - { url = "https://files.pythonhosted.org/packages/2b/a0/9b916c68c0e57752c07f8f64b30138d9d4059dbeb27b90274dedbea128ff/lxml-6.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:26dd9f57ee3bd41e7d35b4c98a2ffd89ed11591649f421f0ec19f67d50ec67ac", size = 3817120, upload-time = "2026-04-18T04:32:15.803Z" }, +version = "6.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/3b/aab6728cae887456f409b4d75e8a01856e4f04bd510de38052a47768b680/lxml-6.1.1.tar.gz", hash = "sha256:ba96ae44888e0185281e937633a743ea90d5a196c6000f82565ebb0580012d40", size = 4197430, upload-time = "2026-05-18T19:19:06.424Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/eb/7e6f37c5584ccbb2ff267f56fd0339016938c1c8684cfefab9b33ffc2f36/lxml-6.1.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:68a9198d0fc122d14bb76837de9aa80cf84caed990b5b237f532ed87d3706736", size = 8559780, upload-time = "2026-05-18T19:17:57.661Z" }, + { url = "https://files.pythonhosted.org/packages/a1/36/587c2521cf23a2cd6c9c22108aa7528f683a1f195ed7ccd23a4b1786ad36/lxml-6.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7d47866cb32fb503450b6edc9df355d10dc49836af2e89901bd6ac6b0896d9d9", size = 4618006, upload-time = "2026-05-18T19:18:04.452Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ca/ab7bfe2bf4c972af5e7878262845ead3a24a929a9b04bc11c7c1ece6c82a/lxml-6.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb7c9811bfaa8b1ed5ed319f5d370dfbcaa59d52ea64be2a5a85e18195930354", size = 4924139, upload-time = "2026-05-18T19:19:04.873Z" }, + { url = "https://files.pythonhosted.org/packages/6b/55/a0c72851dfee5ecc689f949723a73dea457758912542cb955b108eaf0d8f/lxml-6.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:762ff394d5bd56da0cf034a23dcce4e13923f15321a2adfa2ac00201dc6d3fca", size = 5082329, upload-time = "2026-05-18T19:19:09.728Z" }, + { url = "https://files.pythonhosted.org/packages/f0/b6/0608f7d61a3b96cc67e5648a3d906e31a5082093e10e7be65b3886289938/lxml-6.1.1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a088f287f7d8275a33c07f2cac6c50b9319309a0200a39e7e75d80c707723099", size = 4993564, upload-time = "2026-05-18T19:19:13.608Z" }, + { url = "https://files.pythonhosted.org/packages/4c/66/ae227524b066d29d55bf0b453d93d2d793c40218657d643dcbbca13b8faf/lxml-6.1.1-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e902da4b04e6b52e5893900d4b8ab46068f75f3561f01bf1080957f9fd932ed6", size = 5613467, upload-time = "2026-05-18T19:19:16.228Z" }, + { url = "https://files.pythonhosted.org/packages/a6/76/dbe4a00b50385e40194231dcfe5a12c059de7cf90e89c83407d2b085b719/lxml-6.1.1-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1d4962d4c66bf830a7e59ed6cfc17d148149898a3aefa8ec6e59763e6e3ed085", size = 5228304, upload-time = "2026-05-18T19:19:19.354Z" }, + { url = "https://files.pythonhosted.org/packages/1c/01/00b1b8442ed2041793336868ba0b9ea4b13d7da7c085c6404c207a63bf79/lxml-6.1.1-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:581d4c8ae690a6609e64862dd6b7c2489635c2d13907fc2b20f2bc200ff1d21e", size = 5341607, upload-time = "2026-05-18T19:19:22.297Z" }, + { url = "https://files.pythonhosted.org/packages/63/36/1ad29931e9a4638bb707869f01d423a6c815f82152138d1a40dfcfde2b95/lxml-6.1.1-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:876e1ff5930ed8bf295ec5ef9a8155e9b6b1876bbf1deed8b3a8069311875a8f", size = 4700168, upload-time = "2026-05-18T19:19:25.133Z" }, + { url = "https://files.pythonhosted.org/packages/3c/d1/a9536cecf9be18a0dc72d32bead283a2332d1ffebd2dd3ac70ce444686e5/lxml-6.1.1-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9eb9b5a968f6e0f6d640092a567e14529ff8cea2e29d00da6f78a79fa49f013c", size = 5232487, upload-time = "2026-05-18T19:19:28.603Z" }, + { url = "https://files.pythonhosted.org/packages/0e/77/b4fb1e03bf5d130e879214d3100092e386418807fb74dd0adc4b0a48f351/lxml-6.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:aa49e06d94aba782c6a02eecb7e507969e7e7a41b267f1b359bb35585f295d5b", size = 5044231, upload-time = "2026-05-18T19:18:42.246Z" }, + { url = "https://files.pythonhosted.org/packages/26/4c/d00daeeb0a5530c4028a9232aa1b93db3ef4ed2158c116ea73c79a9765b3/lxml-6.1.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:70cdfd80589d59e43e18005dd7244e8895e93db8ab6a620b7e23df5445a4e3d2", size = 4769450, upload-time = "2026-05-18T19:18:48.013Z" }, + { url = "https://files.pythonhosted.org/packages/ed/6a/715a3a8d156ce42f29cf014706f5410c2ff3b02267774110fc23266409fe/lxml-6.1.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:aad9aa39483ed8ec44d6d2e59e5b98a0d80676ef0d92f44bfc374836111f62f5", size = 5635874, upload-time = "2026-05-18T19:18:51.914Z" }, + { url = "https://files.pythonhosted.org/packages/45/37/0544bc21dde2a88f3a17b504e6fc79c0e01d25a33c2f6079724e9e72b9c7/lxml-6.1.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:d49514be2f28d895c38cf9d2b72d7b9a07d00314519f456c0b50b53cfcf4c785", size = 5223987, upload-time = "2026-05-18T19:18:59.715Z" }, + { url = "https://files.pythonhosted.org/packages/4d/f8/f6a5e8185bcb28c2befae3d31f8e3df3b811cb0f47746517a81279fcafe1/lxml-6.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:47402e62c52ff5988c1e8c6c63177f5708bccf48e366dea4e3dcf1e645e04947", size = 5250276, upload-time = "2026-05-18T19:19:03.834Z" }, + { url = "https://files.pythonhosted.org/packages/c7/f2/1a2b9f1b7a49d45495369be7ef9ad05b262930f2eab3e3145706fca8083f/lxml-6.1.1-cp313-cp313-win32.whl", hash = "sha256:3483644525531e1d5762b0c44a8e18b6efba321b6dcf8a8952de10b037618bca", size = 3596903, upload-time = "2026-05-18T19:17:29.863Z" }, + { url = "https://files.pythonhosted.org/packages/e6/99/f4ffb024f238eec2131aaa09f3278fb6129cf892741bf68e1fc1afb8c100/lxml-6.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:a10bd2fd62e8ce916ececb342f348f190724a098c1faa056fdfb2a22ad5e8660", size = 3995869, upload-time = "2026-05-18T19:18:02.596Z" }, + { url = "https://files.pythonhosted.org/packages/d1/53/70eb8c5c6037f27448f1e3c54ebede9545a801ae63f0a7254afca4fe8e45/lxml-6.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:424aa57aca0897eb922aef34395bd1289b3b6f04e6bae20ea123c0c7e333cffc", size = 3658490, upload-time = "2026-05-19T19:22:53.846Z" }, + { url = "https://files.pythonhosted.org/packages/13/e2/2e325795566de01d0d7c3bb57d3c370616b2d07b01214e84eec5d3b10963/lxml-6.1.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:19b7ab10b210b0b3ad7985d9ac4eb66ab09a90b20fe6e2f7ba55d01a234345d0", size = 8577146, upload-time = "2026-05-18T19:18:17.765Z" }, + { url = "https://files.pythonhosted.org/packages/93/cf/5630b5e4be7d2e6bee8efe83865c925221103cf0221303b104ce134b01e2/lxml-6.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c08e5c694306507275f2290073350c4f32e383db15213b2c69e7ff39c1193840", size = 4623866, upload-time = "2026-05-18T19:18:30.669Z" }, + { url = "https://files.pythonhosted.org/packages/d2/51/3904907c063451cf8d4a5c9fe0cad95fa1f4ec57f4e3884fa0731bd7a305/lxml-6.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:74a9717fd0d82effef5c2854f0d917231d5324b5a3eb7275c43ac9fa32f97a14", size = 4950022, upload-time = "2026-05-18T19:19:31.958Z" }, + { url = "https://files.pythonhosted.org/packages/94/cd/9c7611a51c37a2830928405817cc5d56a97f64fab83cc3f628748b135749/lxml-6.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:efe0374196335f93b53269acd811b944f2e6bdc88e8894f214bd636455484909", size = 5086695, upload-time = "2026-05-18T19:19:34.764Z" }, + { url = "https://files.pythonhosted.org/packages/da/d6/24e3b5906abb0b674ff2ae195bc3ce59708df2bcd17cf17703b2d7dd643a/lxml-6.1.1-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac931cdc9442c1763b8a8f6cd62c0c938737eafc5be75eff88df55fc73bc0d00", size = 5031642, upload-time = "2026-05-18T19:19:37.771Z" }, + { url = "https://files.pythonhosted.org/packages/2d/db/6ec54f99019838bff54785c51da07f189eb4676861c5f2730962b0d8d665/lxml-6.1.1-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:aee395f5d0927f947758b4ec119fd5fc8ec71f07a1c5c52077b30b04c0fa6955", size = 5647338, upload-time = "2026-05-18T19:19:40.553Z" }, + { url = "https://files.pythonhosted.org/packages/42/3d/ef4dcfffd22d27a61805d8ed9f7fb888495bc6aa88648fa07c1eaa5586b6/lxml-6.1.1-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9395002973c827b3ed67db77e6ec09f092919a587022174554096a269378fb13", size = 5239528, upload-time = "2026-05-18T19:19:43.657Z" }, + { url = "https://files.pythonhosted.org/packages/62/bb/37fb3f0dff146bdcfa78eec47879273820b2a0bf350ec236ce14bd0b1c26/lxml-6.1.1-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:73bc2086f141224ebddb7fc5c6a36ca58b31b94b561e1dfe8e073e3270fad1e7", size = 5350730, upload-time = "2026-05-18T19:19:46.307Z" }, + { url = "https://files.pythonhosted.org/packages/90/42/43253f168388df4fae1f38c01df36ddb9bee39e2048167b54cdcbae85ea3/lxml-6.1.1-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:3779def59032b81e44a5f70096ef6bf2082f8d901937dca354474ba09782e245", size = 4697530, upload-time = "2026-05-18T19:19:49.889Z" }, + { url = "https://files.pythonhosted.org/packages/eb/a8/c5a8504f81bbdfc8e7094c2c850cdb4ed6777fc4d5ddd9e5ab819f3b0d54/lxml-6.1.1-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:86c89b9d55ebf820ad7c90bc533410f0d098054f293351f10603c0c46ff598f5", size = 5250670, upload-time = "2026-05-18T19:19:53.199Z" }, + { url = "https://files.pythonhosted.org/packages/77/b7/c7e76ab18744d75e21f320ebf9ff9d1ceae2b54dd431ea5a64caf26c9672/lxml-6.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19607c6bbff2a44cf3fe8250abccd20942d3462473e0a721d01d379ed017e462", size = 5084485, upload-time = "2026-05-18T19:19:08.422Z" }, + { url = "https://files.pythonhosted.org/packages/31/31/b35c53f8ef7b7c31cacd23d3638652fff7bcd1deb6eedb709ab43b685908/lxml-6.1.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:c6ed5141a5c7507cf3ee76bd363b0d6f801e3321adc35b5d825a23115faa5465", size = 4737635, upload-time = "2026-05-18T19:19:12.321Z" }, + { url = "https://files.pythonhosted.org/packages/d9/06/31f23c813a7fe8e0cb1b175e915b08c9bf4e86d225b210feadbdbe519667/lxml-6.1.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:62aeb7e85b5d60320b9d77eef2e773994e2c0ce10121b277e0a19804e1654a5a", size = 5670681, upload-time = "2026-05-18T19:19:15.001Z" }, + { url = "https://files.pythonhosted.org/packages/1a/bc/ce619bccc89b1fd9ad8a8e1330ee3f3beff9f2ff95b712d7bbcdd6e22fc3/lxml-6.1.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b1b963fd8f5caa68e99dfae060d54de1fe9cba899b8718b44a00cdca53c3e590", size = 5238229, upload-time = "2026-05-18T19:19:18.131Z" }, + { url = "https://files.pythonhosted.org/packages/2f/5d/b329acbbedc0b619ebc2be6cf7ee9ed07e80892c88d4dfd612c33805789a/lxml-6.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:63876be28efefa04a1df615b46770e82042cce445cfdce55160522f57b231ccb", size = 5264191, upload-time = "2026-05-18T19:19:21.118Z" }, + { url = "https://files.pythonhosted.org/packages/d6/85/be36fb1425b30db3c3f9df75fe86343ebffb79e6320bd7f588e25bfeac39/lxml-6.1.1-cp314-cp314-win32.whl", hash = "sha256:7f7a92e8583f06b1fd49d01158143b8461cfcd135dcb10ec807270a3051bd603", size = 3657202, upload-time = "2026-05-18T19:17:39.509Z" }, + { url = "https://files.pythonhosted.org/packages/b8/ce/3cf9a827342269f54d405a6202397de63f07c69cbd6ce7d183a3f0cba1e9/lxml-6.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:b2d444f2e66624d68e9c6b211e28a76e22fff5fcabcfff4deac18b529b7d4137", size = 4064497, upload-time = "2026-05-18T19:18:14.662Z" }, + { url = "https://files.pythonhosted.org/packages/d9/3e/1a957bde8f0760039e627f94699f82caa782c9d838d86c3d28245ee67212/lxml-6.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:3fd9728a2735fda14f4e8235830c86b539e9661e849665bf926d3f867943b4bf", size = 3741991, upload-time = "2026-05-19T19:22:59.111Z" }, + { url = "https://files.pythonhosted.org/packages/78/b2/00ed55b3a2efa4658fb795c38d1090ec9b3e8a6c3683d4441fa517f09c3b/lxml-6.1.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:787b2496d0dbe8cd180984e8d29e3a6f76e7ea34db781cb3bd55e4ba1ef8b4ee", size = 8827545, upload-time = "2026-05-18T19:18:41.193Z" }, + { url = "https://files.pythonhosted.org/packages/c0/73/74573db19baa618d5f266f2407898b087ff6927115b00b71e5fc1b700847/lxml-6.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2c8daa471358dc2d6fcf02165e80ec68f77871a286df95bc5cc3816153b0fd2c", size = 4735736, upload-time = "2026-05-18T19:18:46.761Z" }, + { url = "https://files.pythonhosted.org/packages/16/02/6f7061f4f95f51e545d48e87647c54791d204a4e881be4156e7a26ba5338/lxml-6.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:acd7d70b64c0aae0c7922cca83d288a16f5f6da523637697872253415269baef", size = 4970291, upload-time = "2026-05-18T19:19:56.215Z" }, + { url = "https://files.pythonhosted.org/packages/b0/02/55fc057d8283427dea7d6edb102e7a840239c77a64a983d92f62a304c0e9/lxml-6.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4f0dd2f01f9f8a89f565d000e03abcf0a13d692a346c8d22f628d49af098777a", size = 5102822, upload-time = "2026-05-18T19:19:59.223Z" }, + { url = "https://files.pythonhosted.org/packages/e4/48/8e1cf78d89d66850121d9255a2a24414c98f775da93b90cf976956c24b14/lxml-6.1.1-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b7e8a14c8634bf6f7a568634cb395305a6d964aeb5b7ee32248094bed3a7e2c", size = 5027923, upload-time = "2026-05-18T19:20:01.549Z" }, + { url = "https://files.pythonhosted.org/packages/ed/00/0632a0647612c8af24d26997b3b961397daa9d5b2581444805933629a4cb/lxml-6.1.1-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:86281fbdd6a8162756f8d603f37e3435bfa38043adb79c6dc6a2dfee065e7525", size = 5595843, upload-time = "2026-05-18T19:20:03.93Z" }, + { url = "https://files.pythonhosted.org/packages/bc/86/ab008a7dc360711b66858d61c80a5979a70a09f2aa2b05d9698df80b803d/lxml-6.1.1-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5d7152ec39ca7c402d8fb9bad86140a15b9503bd0c54484e3f1bbe3dd37ceca", size = 5224515, upload-time = "2026-05-18T19:20:06.381Z" }, + { url = "https://files.pythonhosted.org/packages/75/c6/2702ff375e728e34f56d9a45339a9cf7e4427e917f542225242d63a05afa/lxml-6.1.1-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:88d8cb75b9d82858497a5393e3c63cfbf03035225e4b35a49ed7ccb151e4dc0e", size = 5312511, upload-time = "2026-05-18T19:20:09.308Z" }, + { url = "https://files.pythonhosted.org/packages/b7/57/a5807c98f87a86f10ef9ffab35516df7c0f0c4b6d5d33e9f608ab9c04a31/lxml-6.1.1-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f64ec5397ea6a41fc1b4af0380d79b44a755b5531dcaccd9940fb260dca93038", size = 4639206, upload-time = "2026-05-18T19:20:11.704Z" }, + { url = "https://files.pythonhosted.org/packages/1f/e1/8a0a2c35734812395f4da4eaf33748a7e5705bfb2a58b128da764339d5ec/lxml-6.1.1-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d34bbf07dbc7ca5970671b1512e928991fb5e9d95365636c9b2d8b4f53af405e", size = 5232404, upload-time = "2026-05-18T19:20:14.064Z" }, + { url = "https://files.pythonhosted.org/packages/c2/e2/0e6a4dd5ad84d01d99aa7bae7cfefd4a760a0e0f8176818241de17d9b6c0/lxml-6.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:17e0e18d4ad8adbd0399291bc44845b69d9dd68439a3cdebdf35ff902ec05072", size = 5083769, upload-time = "2026-05-18T19:19:23.758Z" }, + { url = "https://files.pythonhosted.org/packages/a0/7e/161f33d463f6ffc1c7679104b65086dea120080d49dde4d238f015aaee2f/lxml-6.1.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:3ab541146f1f6968c462d6c2ac495148e8cdba2f8347700b2141b6ec5a75bf52", size = 4758936, upload-time = "2026-05-18T19:19:27.256Z" }, + { url = "https://files.pythonhosted.org/packages/f1/fb/2369825e3f6ca99305bf9f7b7085fda91c8b0922a89e54d900974aa3ef85/lxml-6.1.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2a0217714657e023ef4293500f65aa20fce6164c8fd6b08fa5bd4a859fb14b9b", size = 5620296, upload-time = "2026-05-18T19:19:29.993Z" }, + { url = "https://files.pythonhosted.org/packages/30/90/d61e383146f74c5ab683947ea14dc7b82778838ab9b95ea73a23b60d0191/lxml-6.1.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:05a82eb6e1530a64f26225b55cbd178113bd0b5af1c2b625f25e5296742c26d2", size = 5228598, upload-time = "2026-05-18T19:19:33.523Z" }, + { url = "https://files.pythonhosted.org/packages/76/2d/2dafd8149e94b05bb070690efd5bb2680720681e03ff03fc57d2b70a1105/lxml-6.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9e36f163528fc50cbef305f02a5fd66d404edf7049cdaff211dbc2cba5a7013e", size = 5247845, upload-time = "2026-05-18T19:19:36.649Z" }, + { url = "https://files.pythonhosted.org/packages/ce/68/b30e913340c380ddac9580c6e6230991fc37240ec4f64704833e4f3e2769/lxml-6.1.1-cp314-cp314t-win32.whl", hash = "sha256:649dda677cf3bd6ac9ae14007ba0c824ded8ce5808b53fc7431d9140399118c1", size = 3897345, upload-time = "2026-05-18T19:17:33.562Z" }, + { url = "https://files.pythonhosted.org/packages/3c/4e/9eb2af5335545f9fbcd7af57bcf87c6025d31eaa31b14ec184a6c8675328/lxml-6.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:793033d6c5cdf33a573f910d9bea14ef8f5771820411d118da8e1182edb53d5e", size = 4393350, upload-time = "2026-05-18T19:18:10.076Z" }, + { url = "https://files.pythonhosted.org/packages/7f/2c/0f1e93c636720e8a3eb59af2bfda99d98b55891e1c53bc30c2e0e865f01b/lxml-6.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:58bb955caba94e467d2a96da17660d2d704e0675894cba21ab8a775b8621fd1c", size = 3817223, upload-time = "2026-05-19T19:22:56.823Z" }, ] [[package]] @@ -2050,11 +2064,11 @@ wheels = [ [[package]] name = "marko" -version = "2.2.2" +version = "2.2.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e3/2f/050b6d485f052ddf17d76a41f9334d6fb2a8a85df35347a12d97ed3bc5c1/marko-2.2.2.tar.gz", hash = "sha256:6940308e655f63733ca518c47a68ec9510279dbb916c83616e4c4b5829f052e8", size = 143641, upload-time = "2026-01-05T11:04:41.935Z" } +sdist = { url = "https://files.pythonhosted.org/packages/58/cc/01b80dc58e4d44fe039403ef1ac0008bcb9375364ccd246a4b8bfec29b46/marko-2.2.3.tar.gz", hash = "sha256:e31ec2875383bc62f9093d16babed5a2c2cde601c00d834ea935a2222120ec19", size = 144531, upload-time = "2026-05-28T02:07:39.479Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/83/f8/36d79bac5701e6786f9880c61bbe57574760a13c1af84ab71e5ed21faecc/marko-2.2.2-py3-none-any.whl", hash = "sha256:f064ae8c10416285ad1d96048dc11e98ef04e662d3342ae416f662b70aa7959e", size = 42701, upload-time = "2026-01-05T11:04:40.75Z" }, + { url = "https://files.pythonhosted.org/packages/97/50/0a8fab45fa374820c27cc4c3178c4914c60902ba9d6404a692a979e20dbc/marko-2.2.3-py3-none-any.whl", hash = "sha256:8e1d7a0387281e59dfbc52a381b58c570156970e36b2bbe047f8a3a2f368cacc", size = 42951, upload-time = "2026-05-28T02:07:38.373Z" }, ] [[package]] @@ -2111,14 +2125,14 @@ wheels = [ [[package]] name = "matplotlib-inline" -version = "0.2.1" +version = "0.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "traitlets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c7/74/97e72a36efd4ae2bccb3463284300f8953f199b5ffbc04cbbb0ec78f74b1/matplotlib_inline-0.2.1.tar.gz", hash = "sha256:e1ee949c340d771fc39e241ea75683deb94762c8fa5f2927ec57c83c4dffa9fe", size = 8110, upload-time = "2025-10-23T09:00:22.126Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/c0/9f7c9a46090390368a4d7bcb76bb87a4a36c421e4c0792cdb53486ffac7a/matplotlib_inline-0.2.2.tar.gz", hash = "sha256:72f3fe8fce36b70d4a5b612f899090cd0401deddc4ea90e1572b9f4bfb058c79", size = 8150, upload-time = "2026-05-08T17:33:33.49Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl", hash = "sha256:d56ce5156ba6085e00a9d54fead6ed29a9c47e215cd1bba2e976ef39f5710a76", size = 9516, upload-time = "2025-10-23T09:00:20.675Z" }, + { url = "https://files.pythonhosted.org/packages/41/09/5b161152e2d90f7b87f781c2e1267494aef9c32498df793f73ad0a0a494a/matplotlib_inline-0.2.2-py3-none-any.whl", hash = "sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6", size = 9534, upload-time = "2026-05-08T17:33:32.055Z" }, ] [[package]] @@ -2196,22 +2210,20 @@ wheels = [ [[package]] name = "moto" -version = "5.1.22" +version = "5.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "boto3" }, { name = "botocore" }, { name = "cryptography" }, - { name = "jinja2" }, - { name = "python-dateutil" }, { name = "requests" }, { name = "responses" }, { name = "werkzeug" }, { name = "xmltodict" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b2/3d/1765accbf753dc1ae52f26a2e2ed2881d78c2eb9322c178e45312472e4a0/moto-5.1.22.tar.gz", hash = "sha256:e5b2c378296e4da50ce5a3c355a1743c8d6d396ea41122f5bb2a40f9b9a8cc0e", size = 8547792, upload-time = "2026-03-08T21:06:43.731Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/e9/c38202162db2e76623176be9f1dbc9aa41228ffa91ee8da2d3986082c3e3/moto-5.2.1.tar.gz", hash = "sha256:ccb2f3e1dfa82e50e054bda98b0be708d244d2668364dcc1d45e8d3de6091bde", size = 8634437, upload-time = "2026-05-10T19:11:57.286Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/46/4f/8812a01e3e0bd6be3e13b90432fb5c696af9a720af3f00e6eba5ad748345/moto-5.1.22-py3-none-any.whl", hash = "sha256:d9f20ae3cf29c44f93c1f8f06c8f48d5560e5dc027816ef1d0d2059741ffcfbe", size = 6617400, upload-time = "2026-03-08T21:06:41.093Z" }, + { url = "https://files.pythonhosted.org/packages/15/79/8085b7c1ecd48d0535c3c8444a1d8df2926e457dce8e55fabc332a382c9c/moto-5.2.1-py3-none-any.whl", hash = "sha256:19d2fbd6e613aa5b4e364c52cd5d3cea371643a0f4210689a703227bd2924c5c", size = 6671379, upload-time = "2026-05-10T19:11:53.543Z" }, ] [package.optional-dependencies] @@ -2310,15 +2322,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, ] -[[package]] -name = "nest-asyncio" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, -] - [[package]] name = "numpy" version = "2.4.6" @@ -2480,11 +2483,11 @@ wheels = [ [[package]] name = "packaging" -version = "25.0" +version = "26.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, ] [[package]] @@ -2533,11 +2536,11 @@ wheels = [ [[package]] name = "parso" -version = "0.8.6" +version = "0.8.7" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/81/76/a1e769043c0c0c9fe391b702539d594731a4362334cdf4dc25d0c09761e7/parso-0.8.6.tar.gz", hash = "sha256:2b9a0332696df97d454fa67b81618fd69c35a7b90327cbe6ba5c92d2c68a7bfd", size = 401621, upload-time = "2026-02-09T15:45:24.425Z" } +sdist = { url = "https://files.pythonhosted.org/packages/30/4b/90c937815137d43ce71ba043cd3566221e9df6b9c805f24b5d138c9d40a7/parso-0.8.7.tar.gz", hash = "sha256:eaaac4c9fdd5e9e8852dc778d2d7405897ec510f2a298071453e5e3a07914bb1", size = 401824, upload-time = "2026-05-01T23:13:02.138Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/61/fae042894f4296ec49e3f193aff5d7c18440da9e48102c3315e1bc4519a7/parso-0.8.6-py2.py3-none-any.whl", hash = "sha256:2c549f800b70a5c4952197248825584cb00f033b29c692671d3bf08bf380baff", size = 106894, upload-time = "2026-02-09T15:45:21.391Z" }, + { url = "https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl", hash = "sha256:a8926eb2a1b915486941fdbd31e86a4baf88fe8c210f25f2f35ecec5b574ca1c", size = 107025, upload-time = "2026-05-01T23:12:58.867Z" }, ] [[package]] @@ -2596,7 +2599,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess", marker = "(platform_python_implementation != 'PyPy' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "ptyprocess" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ @@ -2605,11 +2608,11 @@ wheels = [ [[package]] name = "platformdirs" -version = "4.9.6" +version = "4.10.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9f/4a/0883b8e3802965322523f0b200ecf33d31f10991d0401162f4b23c698b42/platformdirs-4.9.6.tar.gz", hash = "sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a", size = 29400, upload-time = "2026-04-09T00:04:10.812Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/47/e4501f49c178ae1d9f4a75073fda4204f52647993f075a9db4d14930e0c5/platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7", size = 31224, upload-time = "2026-05-28T03:32:53.587Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917", size = 21348, upload-time = "2026-04-09T00:04:09.463Z" }, + { url = "https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a", size = 22743, upload-time = "2026-05-28T03:32:52.175Z" }, ] [[package]] @@ -2988,12 +2991,12 @@ crypto = [ [[package]] name = "pyspark" -version = "4.1.1" +version = "4.1.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "py4j" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/19/bf/58ee13add151469c25825b7125bbf62c3bdcec05eec4d458fcb5c5516066/pyspark-4.1.1.tar.gz", hash = "sha256:77f78984aa84fbe865c717dd37b49913b4e5c97d76ef6824f932f1aefa6621ec", size = 455359625, upload-time = "2026-01-09T09:38:38.28Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/71/4dd20c69332a2a4bf7ece8a655c9da98e4bd9b6bcea235349c1a00399d57/pyspark-4.1.2.tar.gz", hash = "sha256:fa5d6159f700d0990a07f4f62df1b7449401dccee9cd7d5d6df8957530841602", size = 455428043, upload-time = "2026-05-21T14:49:21.785Z" } [package.optional-dependencies] connect = [ @@ -3037,14 +3040,14 @@ wheels = [ [[package]] name = "pytest-asyncio" -version = "1.3.0" +version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pytest" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/90/2c/8af215c0f776415f3590cac4f9086ccefd6fd463befeae41cd4d3f193e5a/pytest_asyncio-1.3.0.tar.gz", hash = "sha256:d7f52f36d231b80ee124cd216ffb19369aa168fc10095013c6b014a34d3ee9e5", size = 50087, upload-time = "2025-11-10T16:07:47.256Z" } +sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, + { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, ] [[package]] @@ -3317,7 +3320,7 @@ wheels = [ [[package]] name = "requests" -version = "2.33.1" +version = "2.34.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, @@ -3325,9 +3328,9 @@ dependencies = [ { name = "idna" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5f/a4/98b9c7c6428a668bf7e42ebb7c79d576a1c3c1e3ae2d47e674b468388871/requests-2.33.1.tar.gz", hash = "sha256:18817f8c57c6263968bc123d237e3b8b08ac046f5456bd1e307ee8f4250d3517", size = 134120, upload-time = "2026-03-30T16:09:15.531Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/8e/7540e8a2036f79a125c1d2ebadf69ed7901608859186c856fa0388ef4197/requests-2.33.1-py3-none-any.whl", hash = "sha256:4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a", size = 64947, upload-time = "2026-03-30T16:09:13.83Z" }, + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] [[package]] @@ -3356,16 +3359,16 @@ wheels = [ [[package]] name = "responses" -version = "0.26.0" +version = "0.26.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyyaml" }, { name = "requests" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9f/b4/b7e040379838cc71bf5aabdb26998dfbe5ee73904c92c1c161faf5de8866/responses-0.26.0.tar.gz", hash = "sha256:c7f6923e6343ef3682816ba421c006626777893cb0d5e1434f674b649bac9eb4", size = 81303, upload-time = "2026-02-19T14:38:05.574Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c2/58/1fb6de3503428196df78638f991ec8095274f1ee9723e272ee4d9ff0092b/responses-0.26.1.tar.gz", hash = "sha256:2eb3218553cc8f79b57d257bac23af5e1bf381f5b9390b1767816f0843e01dc2", size = 83088, upload-time = "2026-05-21T19:56:39.747Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/04/7f73d05b556da048923e31a0cc878f03be7c5425ed1f268082255c75d872/responses-0.26.0-py3-none-any.whl", hash = "sha256:03ec4409088cd5c66b71ecbbbd27fe2c58ddfad801c66203457b3e6a04868c37", size = 35099, upload-time = "2026-02-19T14:38:03.847Z" }, + { url = "https://files.pythonhosted.org/packages/3a/31/6a620b4427d546b9e7cca8b3b8c5f0559d9cef2bb9eedcda7f73c1473c19/responses-0.26.1-py3-none-any.whl", hash = "sha256:8aacc4586eb08fb2208ef64a9eb4258d9b0c6e6f4260845f2f018ab847495345", size = 35502, upload-time = "2026-05-21T19:56:38.046Z" }, ] [[package]] @@ -3404,93 +3407,122 @@ wheels = [ [[package]] name = "rpds-py" -version = "0.30.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/20/af/3f2f423103f1113b36230496629986e0ef7e199d2aa8392452b484b38ced/rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84", size = 69469, upload-time = "2025-11-30T20:24:38.837Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ed/dc/d61221eb88ff410de3c49143407f6f3147acf2538c86f2ab7ce65ae7d5f9/rpds_py-0.30.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f83424d738204d9770830d35290ff3273fbb02b41f919870479fab14b9d303b2", size = 374887, upload-time = "2025-11-30T20:22:41.812Z" }, - { url = "https://files.pythonhosted.org/packages/fd/32/55fb50ae104061dbc564ef15cc43c013dc4a9f4527a1f4d99baddf56fe5f/rpds_py-0.30.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7536cd91353c5273434b4e003cbda89034d67e7710eab8761fd918ec6c69cf8", size = 358904, upload-time = "2025-11-30T20:22:43.479Z" }, - { url = "https://files.pythonhosted.org/packages/58/70/faed8186300e3b9bdd138d0273109784eea2396c68458ed580f885dfe7ad/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2771c6c15973347f50fece41fc447c054b7ac2ae0502388ce3b6738cd366e3d4", size = 389945, upload-time = "2025-11-30T20:22:44.819Z" }, - { url = "https://files.pythonhosted.org/packages/bd/a8/073cac3ed2c6387df38f71296d002ab43496a96b92c823e76f46b8af0543/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0a59119fc6e3f460315fe9d08149f8102aa322299deaa5cab5b40092345c2136", size = 407783, upload-time = "2025-11-30T20:22:46.103Z" }, - { url = "https://files.pythonhosted.org/packages/77/57/5999eb8c58671f1c11eba084115e77a8899d6e694d2a18f69f0ba471ec8b/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:76fec018282b4ead0364022e3c54b60bf368b9d926877957a8624b58419169b7", size = 515021, upload-time = "2025-11-30T20:22:47.458Z" }, - { url = "https://files.pythonhosted.org/packages/e0/af/5ab4833eadc36c0a8ed2bc5c0de0493c04f6c06de223170bd0798ff98ced/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:692bef75a5525db97318e8cd061542b5a79812d711ea03dbc1f6f8dbb0c5f0d2", size = 414589, upload-time = "2025-11-30T20:22:48.872Z" }, - { url = "https://files.pythonhosted.org/packages/b7/de/f7192e12b21b9e9a68a6d0f249b4af3fdcdff8418be0767a627564afa1f1/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9027da1ce107104c50c81383cae773ef5c24d296dd11c99e2629dbd7967a20c6", size = 394025, upload-time = "2025-11-30T20:22:50.196Z" }, - { url = "https://files.pythonhosted.org/packages/91/c4/fc70cd0249496493500e7cc2de87504f5aa6509de1e88623431fec76d4b6/rpds_py-0.30.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:9cf69cdda1f5968a30a359aba2f7f9aa648a9ce4b580d6826437f2b291cfc86e", size = 408895, upload-time = "2025-11-30T20:22:51.87Z" }, - { url = "https://files.pythonhosted.org/packages/58/95/d9275b05ab96556fefff73a385813eb66032e4c99f411d0795372d9abcea/rpds_py-0.30.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a4796a717bf12b9da9d3ad002519a86063dcac8988b030e405704ef7d74d2d9d", size = 422799, upload-time = "2025-11-30T20:22:53.341Z" }, - { url = "https://files.pythonhosted.org/packages/06/c1/3088fc04b6624eb12a57eb814f0d4997a44b0d208d6cace713033ff1a6ba/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d4c2aa7c50ad4728a094ebd5eb46c452e9cb7edbfdb18f9e1221f597a73e1e7", size = 572731, upload-time = "2025-11-30T20:22:54.778Z" }, - { url = "https://files.pythonhosted.org/packages/d8/42/c612a833183b39774e8ac8fecae81263a68b9583ee343db33ab571a7ce55/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba81a9203d07805435eb06f536d95a266c21e5b2dfbf6517748ca40c98d19e31", size = 599027, upload-time = "2025-11-30T20:22:56.212Z" }, - { url = "https://files.pythonhosted.org/packages/5f/60/525a50f45b01d70005403ae0e25f43c0384369ad24ffe46e8d9068b50086/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:945dccface01af02675628334f7cf49c2af4c1c904748efc5cf7bbdf0b579f95", size = 563020, upload-time = "2025-11-30T20:22:58.2Z" }, - { url = "https://files.pythonhosted.org/packages/0b/5d/47c4655e9bcd5ca907148535c10e7d489044243cc9941c16ed7cd53be91d/rpds_py-0.30.0-cp313-cp313-win32.whl", hash = "sha256:b40fb160a2db369a194cb27943582b38f79fc4887291417685f3ad693c5a1d5d", size = 223139, upload-time = "2025-11-30T20:23:00.209Z" }, - { url = "https://files.pythonhosted.org/packages/f2/e1/485132437d20aa4d3e1d8b3fb5a5e65aa8139f1e097080c2a8443201742c/rpds_py-0.30.0-cp313-cp313-win_amd64.whl", hash = "sha256:806f36b1b605e2d6a72716f321f20036b9489d29c51c91f4dd29a3e3afb73b15", size = 240224, upload-time = "2025-11-30T20:23:02.008Z" }, - { url = "https://files.pythonhosted.org/packages/24/95/ffd128ed1146a153d928617b0ef673960130be0009c77d8fbf0abe306713/rpds_py-0.30.0-cp313-cp313-win_arm64.whl", hash = "sha256:d96c2086587c7c30d44f31f42eae4eac89b60dabbac18c7669be3700f13c3ce1", size = 230645, upload-time = "2025-11-30T20:23:03.43Z" }, - { url = "https://files.pythonhosted.org/packages/ff/1b/b10de890a0def2a319a2626334a7f0ae388215eb60914dbac8a3bae54435/rpds_py-0.30.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:eb0b93f2e5c2189ee831ee43f156ed34e2a89a78a66b98cadad955972548be5a", size = 364443, upload-time = "2025-11-30T20:23:04.878Z" }, - { url = "https://files.pythonhosted.org/packages/0d/bf/27e39f5971dc4f305a4fb9c672ca06f290f7c4e261c568f3dea16a410d47/rpds_py-0.30.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:922e10f31f303c7c920da8981051ff6d8c1a56207dbdf330d9047f6d30b70e5e", size = 353375, upload-time = "2025-11-30T20:23:06.342Z" }, - { url = "https://files.pythonhosted.org/packages/40/58/442ada3bba6e8e6615fc00483135c14a7538d2ffac30e2d933ccf6852232/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdc62c8286ba9bf7f47befdcea13ea0e26bf294bda99758fd90535cbaf408000", size = 383850, upload-time = "2025-11-30T20:23:07.825Z" }, - { url = "https://files.pythonhosted.org/packages/14/14/f59b0127409a33c6ef6f5c1ebd5ad8e32d7861c9c7adfa9a624fc3889f6c/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47f9a91efc418b54fb8190a6b4aa7813a23fb79c51f4bb84e418f5476c38b8db", size = 392812, upload-time = "2025-11-30T20:23:09.228Z" }, - { url = "https://files.pythonhosted.org/packages/b3/66/e0be3e162ac299b3a22527e8913767d869e6cc75c46bd844aa43fb81ab62/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f3587eb9b17f3789ad50824084fa6f81921bbf9a795826570bda82cb3ed91f2", size = 517841, upload-time = "2025-11-30T20:23:11.186Z" }, - { url = "https://files.pythonhosted.org/packages/3d/55/fa3b9cf31d0c963ecf1ba777f7cf4b2a2c976795ac430d24a1f43d25a6ba/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39c02563fc592411c2c61d26b6c5fe1e51eaa44a75aa2c8735ca88b0d9599daa", size = 408149, upload-time = "2025-11-30T20:23:12.864Z" }, - { url = "https://files.pythonhosted.org/packages/60/ca/780cf3b1a32b18c0f05c441958d3758f02544f1d613abf9488cd78876378/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a1234d8febafdfd33a42d97da7a43f5dcb120c1060e352a3fbc0c6d36e2083", size = 383843, upload-time = "2025-11-30T20:23:14.638Z" }, - { url = "https://files.pythonhosted.org/packages/82/86/d5f2e04f2aa6247c613da0c1dd87fcd08fa17107e858193566048a1e2f0a/rpds_py-0.30.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:eb2c4071ab598733724c08221091e8d80e89064cd472819285a9ab0f24bcedb9", size = 396507, upload-time = "2025-11-30T20:23:16.105Z" }, - { url = "https://files.pythonhosted.org/packages/4b/9a/453255d2f769fe44e07ea9785c8347edaf867f7026872e76c1ad9f7bed92/rpds_py-0.30.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bdfdb946967d816e6adf9a3d8201bfad269c67efe6cefd7093ef959683c8de0", size = 414949, upload-time = "2025-11-30T20:23:17.539Z" }, - { url = "https://files.pythonhosted.org/packages/a3/31/622a86cdc0c45d6df0e9ccb6becdba5074735e7033c20e401a6d9d0e2ca0/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c77afbd5f5250bf27bf516c7c4a016813eb2d3e116139aed0096940c5982da94", size = 565790, upload-time = "2025-11-30T20:23:19.029Z" }, - { url = "https://files.pythonhosted.org/packages/1c/5d/15bbf0fb4a3f58a3b1c67855ec1efcc4ceaef4e86644665fff03e1b66d8d/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:61046904275472a76c8c90c9ccee9013d70a6d0f73eecefd38c1ae7c39045a08", size = 590217, upload-time = "2025-11-30T20:23:20.885Z" }, - { url = "https://files.pythonhosted.org/packages/6d/61/21b8c41f68e60c8cc3b2e25644f0e3681926020f11d06ab0b78e3c6bbff1/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4c5f36a861bc4b7da6516dbdf302c55313afa09b81931e8280361a4f6c9a2d27", size = 555806, upload-time = "2025-11-30T20:23:22.488Z" }, - { url = "https://files.pythonhosted.org/packages/f9/39/7e067bb06c31de48de3eb200f9fc7c58982a4d3db44b07e73963e10d3be9/rpds_py-0.30.0-cp313-cp313t-win32.whl", hash = "sha256:3d4a69de7a3e50ffc214ae16d79d8fbb0922972da0356dcf4d0fdca2878559c6", size = 211341, upload-time = "2025-11-30T20:23:24.449Z" }, - { url = "https://files.pythonhosted.org/packages/0a/4d/222ef0b46443cf4cf46764d9c630f3fe4abaa7245be9417e56e9f52b8f65/rpds_py-0.30.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f14fc5df50a716f7ece6a80b6c78bb35ea2ca47c499e422aa4463455dd96d56d", size = 225768, upload-time = "2025-11-30T20:23:25.908Z" }, - { url = "https://files.pythonhosted.org/packages/86/81/dad16382ebbd3d0e0328776d8fd7ca94220e4fa0798d1dc5e7da48cb3201/rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:68f19c879420aa08f61203801423f6cd5ac5f0ac4ac82a2368a9fcd6a9a075e0", size = 362099, upload-time = "2025-11-30T20:23:27.316Z" }, - { url = "https://files.pythonhosted.org/packages/2b/60/19f7884db5d5603edf3c6bce35408f45ad3e97e10007df0e17dd57af18f8/rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ec7c4490c672c1a0389d319b3a9cfcd098dcdc4783991553c332a15acf7249be", size = 353192, upload-time = "2025-11-30T20:23:29.151Z" }, - { url = "https://files.pythonhosted.org/packages/bf/c4/76eb0e1e72d1a9c4703c69607cec123c29028bff28ce41588792417098ac/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f251c812357a3fed308d684a5079ddfb9d933860fc6de89f2b7ab00da481e65f", size = 384080, upload-time = "2025-11-30T20:23:30.785Z" }, - { url = "https://files.pythonhosted.org/packages/72/87/87ea665e92f3298d1b26d78814721dc39ed8d2c74b86e83348d6b48a6f31/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac98b175585ecf4c0348fd7b29c3864bda53b805c773cbf7bfdaffc8070c976f", size = 394841, upload-time = "2025-11-30T20:23:32.209Z" }, - { url = "https://files.pythonhosted.org/packages/77/ad/7783a89ca0587c15dcbf139b4a8364a872a25f861bdb88ed99f9b0dec985/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3e62880792319dbeb7eb866547f2e35973289e7d5696c6e295476448f5b63c87", size = 516670, upload-time = "2025-11-30T20:23:33.742Z" }, - { url = "https://files.pythonhosted.org/packages/5b/3c/2882bdac942bd2172f3da574eab16f309ae10a3925644e969536553cb4ee/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e7fc54e0900ab35d041b0601431b0a0eb495f0851a0639b6ef90f7741b39a18", size = 408005, upload-time = "2025-11-30T20:23:35.253Z" }, - { url = "https://files.pythonhosted.org/packages/ce/81/9a91c0111ce1758c92516a3e44776920b579d9a7c09b2b06b642d4de3f0f/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47e77dc9822d3ad616c3d5759ea5631a75e5809d5a28707744ef79d7a1bcfcad", size = 382112, upload-time = "2025-11-30T20:23:36.842Z" }, - { url = "https://files.pythonhosted.org/packages/cf/8e/1da49d4a107027e5fbc64daeab96a0706361a2918da10cb41769244b805d/rpds_py-0.30.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b4dc1a6ff022ff85ecafef7979a2c6eb423430e05f1165d6688234e62ba99a07", size = 399049, upload-time = "2025-11-30T20:23:38.343Z" }, - { url = "https://files.pythonhosted.org/packages/df/5a/7ee239b1aa48a127570ec03becbb29c9d5a9eb092febbd1699d567cae859/rpds_py-0.30.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4559c972db3a360808309e06a74628b95eaccbf961c335c8fe0d590cf587456f", size = 415661, upload-time = "2025-11-30T20:23:40.263Z" }, - { url = "https://files.pythonhosted.org/packages/70/ea/caa143cf6b772f823bc7929a45da1fa83569ee49b11d18d0ada7f5ee6fd6/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0ed177ed9bded28f8deb6ab40c183cd1192aa0de40c12f38be4d59cd33cb5c65", size = 565606, upload-time = "2025-11-30T20:23:42.186Z" }, - { url = "https://files.pythonhosted.org/packages/64/91/ac20ba2d69303f961ad8cf55bf7dbdb4763f627291ba3d0d7d67333cced9/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ad1fa8db769b76ea911cb4e10f049d80bf518c104f15b3edb2371cc65375c46f", size = 591126, upload-time = "2025-11-30T20:23:44.086Z" }, - { url = "https://files.pythonhosted.org/packages/21/20/7ff5f3c8b00c8a95f75985128c26ba44503fb35b8e0259d812766ea966c7/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:46e83c697b1f1c72b50e5ee5adb4353eef7406fb3f2043d64c33f20ad1c2fc53", size = 553371, upload-time = "2025-11-30T20:23:46.004Z" }, - { url = "https://files.pythonhosted.org/packages/72/c7/81dadd7b27c8ee391c132a6b192111ca58d866577ce2d9b0ca157552cce0/rpds_py-0.30.0-cp314-cp314-win32.whl", hash = "sha256:ee454b2a007d57363c2dfd5b6ca4a5d7e2c518938f8ed3b706e37e5d470801ed", size = 215298, upload-time = "2025-11-30T20:23:47.696Z" }, - { url = "https://files.pythonhosted.org/packages/3e/d2/1aaac33287e8cfb07aab2e6b8ac1deca62f6f65411344f1433c55e6f3eb8/rpds_py-0.30.0-cp314-cp314-win_amd64.whl", hash = "sha256:95f0802447ac2d10bcc69f6dc28fe95fdf17940367b21d34e34c737870758950", size = 228604, upload-time = "2025-11-30T20:23:49.501Z" }, - { url = "https://files.pythonhosted.org/packages/e8/95/ab005315818cc519ad074cb7784dae60d939163108bd2b394e60dc7b5461/rpds_py-0.30.0-cp314-cp314-win_arm64.whl", hash = "sha256:613aa4771c99f03346e54c3f038e4cc574ac09a3ddfb0e8878487335e96dead6", size = 222391, upload-time = "2025-11-30T20:23:50.96Z" }, - { url = "https://files.pythonhosted.org/packages/9e/68/154fe0194d83b973cdedcdcc88947a2752411165930182ae41d983dcefa6/rpds_py-0.30.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:7e6ecfcb62edfd632e56983964e6884851786443739dbfe3582947e87274f7cb", size = 364868, upload-time = "2025-11-30T20:23:52.494Z" }, - { url = "https://files.pythonhosted.org/packages/83/69/8bbc8b07ec854d92a8b75668c24d2abcb1719ebf890f5604c61c9369a16f/rpds_py-0.30.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a1d0bc22a7cdc173fedebb73ef81e07faef93692b8c1ad3733b67e31e1b6e1b8", size = 353747, upload-time = "2025-11-30T20:23:54.036Z" }, - { url = "https://files.pythonhosted.org/packages/ab/00/ba2e50183dbd9abcce9497fa5149c62b4ff3e22d338a30d690f9af970561/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d08f00679177226c4cb8c5265012eea897c8ca3b93f429e546600c971bcbae7", size = 383795, upload-time = "2025-11-30T20:23:55.556Z" }, - { url = "https://files.pythonhosted.org/packages/05/6f/86f0272b84926bcb0e4c972262f54223e8ecc556b3224d281e6598fc9268/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5965af57d5848192c13534f90f9dd16464f3c37aaf166cc1da1cae1fd5a34898", size = 393330, upload-time = "2025-11-30T20:23:57.033Z" }, - { url = "https://files.pythonhosted.org/packages/cb/e9/0e02bb2e6dc63d212641da45df2b0bf29699d01715913e0d0f017ee29438/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a4e86e34e9ab6b667c27f3211ca48f73dba7cd3d90f8d5b11be56e5dbc3fb4e", size = 518194, upload-time = "2025-11-30T20:23:58.637Z" }, - { url = "https://files.pythonhosted.org/packages/ee/ca/be7bca14cf21513bdf9c0606aba17d1f389ea2b6987035eb4f62bd923f25/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d3e6b26f2c785d65cc25ef1e5267ccbe1b069c5c21b8cc724efee290554419", size = 408340, upload-time = "2025-11-30T20:24:00.2Z" }, - { url = "https://files.pythonhosted.org/packages/c2/c7/736e00ebf39ed81d75544c0da6ef7b0998f8201b369acf842f9a90dc8fce/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:626a7433c34566535b6e56a1b39a7b17ba961e97ce3b80ec62e6f1312c025551", size = 383765, upload-time = "2025-11-30T20:24:01.759Z" }, - { url = "https://files.pythonhosted.org/packages/4a/3f/da50dfde9956aaf365c4adc9533b100008ed31aea635f2b8d7b627e25b49/rpds_py-0.30.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:acd7eb3f4471577b9b5a41baf02a978e8bdeb08b4b355273994f8b87032000a8", size = 396834, upload-time = "2025-11-30T20:24:03.687Z" }, - { url = "https://files.pythonhosted.org/packages/4e/00/34bcc2565b6020eab2623349efbdec810676ad571995911f1abdae62a3a0/rpds_py-0.30.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe5fa731a1fa8a0a56b0977413f8cacac1768dad38d16b3a296712709476fbd5", size = 415470, upload-time = "2025-11-30T20:24:05.232Z" }, - { url = "https://files.pythonhosted.org/packages/8c/28/882e72b5b3e6f718d5453bd4d0d9cf8df36fddeb4ddbbab17869d5868616/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:74a3243a411126362712ee1524dfc90c650a503502f135d54d1b352bd01f2404", size = 565630, upload-time = "2025-11-30T20:24:06.878Z" }, - { url = "https://files.pythonhosted.org/packages/3b/97/04a65539c17692de5b85c6e293520fd01317fd878ea1995f0367d4532fb1/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3e8eeb0544f2eb0d2581774be4c3410356eba189529a6b3e36bbbf9696175856", size = 591148, upload-time = "2025-11-30T20:24:08.445Z" }, - { url = "https://files.pythonhosted.org/packages/85/70/92482ccffb96f5441aab93e26c4d66489eb599efdcf96fad90c14bbfb976/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dbd936cde57abfee19ab3213cf9c26be06d60750e60a8e4dd85d1ab12c8b1f40", size = 556030, upload-time = "2025-11-30T20:24:10.956Z" }, - { url = "https://files.pythonhosted.org/packages/20/53/7c7e784abfa500a2b6b583b147ee4bb5a2b3747a9166bab52fec4b5b5e7d/rpds_py-0.30.0-cp314-cp314t-win32.whl", hash = "sha256:dc824125c72246d924f7f796b4f63c1e9dc810c7d9e2355864b3c3a73d59ade0", size = 211570, upload-time = "2025-11-30T20:24:12.735Z" }, - { url = "https://files.pythonhosted.org/packages/d0/02/fa464cdfbe6b26e0600b62c528b72d8608f5cc49f96b8d6e38c95d60c676/rpds_py-0.30.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27f4b0e92de5bfbc6f86e43959e6edd1425c33b5e69aab0984a72047f2bcf1e3", size = 226532, upload-time = "2025-11-30T20:24:14.634Z" }, +version = "2026.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2e/43/25a8dcd3feedd735039a8f0b5b7e3b118232b5eae288c4fd9ab200d41094/rpds_py-2026.5.1.tar.gz", hash = "sha256:07b24fea40541e28570e5b795a4a38fbdcd12550c06bd0748005ecc8116ca256", size = 64459, upload-time = "2026-05-28T12:02:13.232Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/32/14c961ad295f490eb0849ada8b79683e93a59b9de3afdd983eaf55fa6867/rpds_py-2026.5.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:efef4ac29c6ff495531eb17ee705b62841ecaa291b7c7077e848ea03e237164d", size = 352787, upload-time = "2026-05-28T11:59:33.655Z" }, + { url = "https://files.pythonhosted.org/packages/ca/bb/d1b85117967c11191441a7274ae616c65d93901d082c588f89a50a8da5ae/rpds_py-2026.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c39f5b67a8a2e67179ada2a954227d670fe65fa9098457f698f56ddf248709b3", size = 345179, upload-time = "2026-05-28T11:59:35Z" }, + { url = "https://files.pythonhosted.org/packages/7c/46/d84105f062e626a1b233f863907288a4708c2d833b8b4c6fb2764bc080c0/rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5c30f3f04eef4fbd362226a6f31d7c8895ca4fbb6e0b790f6890a98d8da8559", size = 376173, upload-time = "2026-05-28T11:59:36.43Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/469d7959ce5b1201e1de135dc735b86db3b35dd0d1734f6a44246d5f061c/rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:277f6c82f0580848796c7ecc8a7173aa3bfb928e4ff831261c2f60a81dc270db", size = 383162, upload-time = "2026-05-28T11:59:37.995Z" }, + { url = "https://files.pythonhosted.org/packages/dc/a2/57853d31a1116a561aa072794602ad3f6341e18d70a8523f1bd5b9fc1e5a/rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:63c2c4c213f1a4e3f3de28ecab029dbdee976324e729c0d7a55211be72576b02", size = 495093, upload-time = "2026-05-28T11:59:39.453Z" }, + { url = "https://files.pythonhosted.org/packages/99/63/3a8eabcad9314b7daf5c65f451d2c33d989235cd8a5762186cf2c3f5a4f8/rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3350ec808fb538fe71a1f94dfaa0e29c598dfad805ce49f0caec5ae3183c652b", size = 389829, upload-time = "2026-05-28T11:59:40.896Z" }, + { url = "https://files.pythonhosted.org/packages/4b/25/05678d97fc25e2622df14dc530fb82023174ecfff6733991ed0d78f167bd/rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1b964e3ab599e718dc46c018d104b1ebc007cbc6567d827c94a687fca56d77e", size = 374786, upload-time = "2026-05-28T11:59:42.626Z" }, + { url = "https://files.pythonhosted.org/packages/88/d1/8c90b6431e80a3b91b284a5c7c8c0c4f9c006444d90477a740d6e0f9c694/rpds_py-2026.5.1-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:19cb09fab7b7fc96b2a6e28f2e34b72a3705ff27b37edb77455316e5d3f3dc9b", size = 386920, upload-time = "2026-05-28T11:59:44.124Z" }, + { url = "https://files.pythonhosted.org/packages/ff/99/4638f672ab356682d633ee0da9255f5b67ce6efd0b85eb94ad3e255e65a5/rpds_py-2026.5.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:abe76bcdba31e576cb83eeb8797aa0d882b738fef6dc65d0601fc753806a5b46", size = 405059, upload-time = "2026-05-28T11:59:47.177Z" }, + { url = "https://files.pythonhosted.org/packages/66/3f/3546524b6eb4cc2e1f363a3d638fa52f6c24faae3500c25fb488b02f1740/rpds_py-2026.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8bff7073db3899158fff55ebf57b113a67030af26f80a18978f9f0aa60250ddf", size = 553030, upload-time = "2026-05-28T11:59:48.603Z" }, + { url = "https://files.pythonhosted.org/packages/c6/c3/7b3388c796fcf471bd17194242d4dc1a7608567c0fa422bcc1c5e79f9c1e/rpds_py-2026.5.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:8ba264fa49be666cd9cc56bf34ec7002fb3d27a4aee5bcb4d43d0d18feb1bb6f", size = 618975, upload-time = "2026-05-28T11:59:50.314Z" }, + { url = "https://files.pythonhosted.org/packages/61/1e/a3cb07f2795075d1d88efddae2f541359fde5f08c81ee114c29c2949c90a/rpds_py-2026.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4860b603ddda0475a8885499b3729e90229d480105b42651962a5397d995fa89", size = 581178, upload-time = "2026-05-28T11:59:51.673Z" }, + { url = "https://files.pythonhosted.org/packages/a1/74/e758c03a5ef46f04c37f2651a2893db846d569ba8a7bca469d4b58939bcd/rpds_py-2026.5.1-cp313-cp313-win32.whl", hash = "sha256:7944270ae71383f6e2657dd7d5ce4eeb4ac2d0059a6738f0510583d462ab4842", size = 212481, upload-time = "2026-05-28T11:59:53.148Z" }, + { url = "https://files.pythonhosted.org/packages/70/ec/a2aca432db9c7359b40fa393eeeaa0d166c2f70175be956e75fa24197c44/rpds_py-2026.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:88647f43a73c4e01be19b04ceef0c8d3a1958153604d13c773becd8016f2a0cf", size = 228519, upload-time = "2026-05-28T11:59:54.505Z" }, + { url = "https://files.pythonhosted.org/packages/29/60/a73bfdd45b096574556acf303bbd9fa9eed36ca8a818b514e2a5d5fe2b9d/rpds_py-2026.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:453895624ecf7db7063b1004e44037522bbaef9ff6a945e59bc71662d7a03abd", size = 223446, upload-time = "2026-05-28T11:59:56.081Z" }, + { url = "https://files.pythonhosted.org/packages/18/e2/408105fd611823f00882aea810f3989a30d26b1bab8b6beb20f98c724e0e/rpds_py-2026.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:b4e4bc98639ec915f512fde3aa7a95e0041d95d9c3cc86eea841fa63cb1e8600", size = 355287, upload-time = "2026-05-28T11:59:57.448Z" }, + { url = "https://files.pythonhosted.org/packages/8d/58/5c4a43436843c90d0f6d19f82c200c80e3843ca9fa07b237623327f6d384/rpds_py-2026.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:cacedb7a6e167680acba45ad5716e89067d225dc80da0d7040cae8c81d4572fa", size = 347033, upload-time = "2026-05-28T11:59:58.881Z" }, + { url = "https://files.pythonhosted.org/packages/fb/c2/1a71acdacaf4e259b10278fb87b039ded3cf80041bcd89dd8a3ea702ded6/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68700371c5d7ae1412862ddfa719090925c93ecf351c566d66f09d04b136ea00", size = 376891, upload-time = "2026-05-28T12:00:00.516Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c8/535f3d9b65addd8e28aa87b83c6e526799c3717a88273db8ea795beeef7a/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:296c799becfa849c779c8725494fe9ed94959ed886787df4364b058465bad7f0", size = 385646, upload-time = "2026-05-28T12:00:02.394Z" }, + { url = "https://files.pythonhosted.org/packages/1c/91/dc033f313345c354ade914dbe73cdb90b615a4409ea02430d5356794f3d8/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d3858b908218ee108d0bbfb2095ccc237648053c9bf98affad7cb079acaf1d97", size = 498830, upload-time = "2026-05-28T12:00:04.189Z" }, + { url = "https://files.pythonhosted.org/packages/27/fc/90fcbea459dbb8ddc18a2e0fd1de9412b48bc84ffff2db771cf714bacfd6/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4fb8d2e7cb2f850b169806d61d1b991738acec96500a75c30f49caf064ce7cef", size = 392830, upload-time = "2026-05-28T12:00:05.797Z" }, + { url = "https://files.pythonhosted.org/packages/b2/1d/46cd11a228c9750684a798d98f878be6f614aa762438da7378f035e79e35/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27b74c10ed6a8f190f4287f53bcfea348b92a84a9c9f70d30183d1e6172d580d", size = 379613, upload-time = "2026-05-28T12:00:07.433Z" }, + { url = "https://files.pythonhosted.org/packages/24/4a/d9b0c6af3a1de03eb93741bbe8be2bdce84d8fda8224f3005451d86df389/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:b9a6528956191c48c52294a592dbd4a8386d7048bdb25c0efcb6b966466c6d83", size = 388183, upload-time = "2026-05-28T12:00:09.227Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b4/db7aaabdda6d020afc87d981bcc2f57a434c7dec60ecfc2ab3dd50b20351/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:af03e34e860047bc7a352b842856fcf78798fbb81132cc98bd2f907ab4eb9cd2", size = 408578, upload-time = "2026-05-28T12:00:10.779Z" }, + { url = "https://files.pythonhosted.org/packages/08/d6/070f6a41cbb343e2ac4171859bf3f3623e0ab002f72619d6d505313ec2de/rpds_py-2026.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:fea6e836d10abbe191d557d33bd58bd5987725fe63aa1eefe557d230209855bd", size = 553573, upload-time = "2026-05-28T12:00:12.443Z" }, + { url = "https://files.pythonhosted.org/packages/75/ab/1a71ea3589c4345dac0a0518f0e6a031cb42689277851b683c46d27463a5/rpds_py-2026.5.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:fc0c0f878ea770a0a8a462456c5ad36fc9fe6358e6b76fdadc7f17575e0b8bf1", size = 620861, upload-time = "2026-05-28T12:00:14.09Z" }, + { url = "https://files.pythonhosted.org/packages/8a/22/9bf80a56069c0c443fcfefac639a86a744550a2898817a6dfd3e26654924/rpds_py-2026.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e0b360f316d966b048b085857630b3cc51f3db2f07b06f440eac8f695374d1e3", size = 585633, upload-time = "2026-05-28T12:00:15.66Z" }, + { url = "https://files.pythonhosted.org/packages/da/68/3b2c0a75c9e04125696f84ebdbbf304acf5a40b58ba4481cdb98a922c3ba/rpds_py-2026.5.1-cp313-cp313t-win32.whl", hash = "sha256:a2999883eedf72fdfb7520b92c7d4ec2572a71ff40239377aa604cc529eecafc", size = 210074, upload-time = "2026-05-28T12:00:17.291Z" }, + { url = "https://files.pythonhosted.org/packages/e7/8b/609157d5a25d37d4f29f92840ba531f416907c34ae5c5739dd21fc2bef98/rpds_py-2026.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:e07be2a9d7122bd6e82dea89814ef8dc893feb1aae97fec1630f3263bbb30e55", size = 228635, upload-time = "2026-05-28T12:00:18.73Z" }, + { url = "https://files.pythonhosted.org/packages/d4/6f/19c1918a4b590d8de87e712e4abe4b3875771eff60216fb6153cf6665c68/rpds_py-2026.5.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:1f2c391c3059798093b65df23aca2cac150460ae9c630d99dec83d703d9485b9", size = 349756, upload-time = "2026-05-28T12:00:20.217Z" }, + { url = "https://files.pythonhosted.org/packages/e5/60/a06fe7da34eca79dacbf958a2ba0c6eea85bc2b29de20080bf40f72f66fa/rpds_py-2026.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:413b424f7c4ee65ab5e5be91f5731be0f8b41a1ee2b12dfe810d716312e95a78", size = 343831, upload-time = "2026-05-28T12:00:21.711Z" }, + { url = "https://files.pythonhosted.org/packages/bf/ec/b2333b97b90e2a6ef6ca8ad386ee284968e74bcfe113b3f1a8d9036429a9/rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c595a1d9255dce0599e13130d1440ab2506654f2b50294226ee06402f8fef63", size = 375127, upload-time = "2026-05-28T12:00:23.326Z" }, + { url = "https://files.pythonhosted.org/packages/14/7f/e00aae54067f2b488c4637961d5f58204d470795fc791085fa3f15060d2e/rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1c27c5f6102eac8c03e7595a00827a53b271ba40a53b59ff8709170e0855ea4a", size = 379034, upload-time = "2026-05-28T12:00:24.89Z" }, + { url = "https://files.pythonhosted.org/packages/be/cc/423999bbb8ae8dc93c77fc1d5e984ade5eb89d237d3bb884ccfa72ae2890/rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c7fcf61d44cacecaf3aea542b0e053db77972a4573e7ceda16fb2b399161195", size = 490823, upload-time = "2026-05-28T12:00:26.676Z" }, + { url = "https://files.pythonhosted.org/packages/0f/aa/c671bf660f12e68d3c52ff86c7066ed1372df5a0f4f2ff584e419b8207e7/rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2c817a189d4ee14290420e5ff051e4dd6baa13f3edf84685071dee07a6d538ee", size = 388144, upload-time = "2026-05-28T12:00:28.577Z" }, + { url = "https://files.pythonhosted.org/packages/19/c8/d63bb75b68afe77b229e3021c6031bcaf01da5db5b0e69d0d10f9ba679a7/rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21846aac0ed2e0589f38c12dc44e77bb64e494b771eadbcf169cba00566ba7ba", size = 371959, upload-time = "2026-05-28T12:00:30.304Z" }, + { url = "https://files.pythonhosted.org/packages/82/35/c51122014d8274ff37dc606d60049c3db7d83da02b5b282511e5a906a9a6/rpds_py-2026.5.1-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b317c87a13f769a4e787819bd508aaa5d69aa09b0880de9af6d3a8a54571cdec", size = 383558, upload-time = "2026-05-28T12:00:31.764Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f9/2790cb99c136a5363acdeacf5c27c56f3de0d4118a1f48fca83404c99c89/rpds_py-2026.5.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ce87129d9f2c14fa6c4a8601fb80eb4488c80d38a20cd13758ef11123e14995d", size = 402789, upload-time = "2026-05-28T12:00:33.247Z" }, + { url = "https://files.pythonhosted.org/packages/e5/1b/e4fb584f8c75d35c38150ff6a332cda949e6f97acba1f4fd123b14ab56fe/rpds_py-2026.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9cdddb6c1207d284d94fd1530adf57fbd797fe7c4b8704ba85f49414f2557e7d", size = 551405, upload-time = "2026-05-28T12:00:34.819Z" }, + { url = "https://files.pythonhosted.org/packages/d8/f7/a6731b4216cb3793ea1af5391da240f5683dacc0d13e034fe5fc3503f240/rpds_py-2026.5.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:4e237e139f94d3c036fd28eb9f564c99055476ff4ff05cd42be55ce349b5aa02", size = 616975, upload-time = "2026-05-28T12:00:36.268Z" }, + { url = "https://files.pythonhosted.org/packages/2c/ea/2e051a81d95d8e63f4b35a1c463a87e8766bc3d083c067c5dfb6bf220747/rpds_py-2026.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ed0954b524873214369184a9c82b0eaa45a3fbb9a798cd95b17e0d98499e7ea0", size = 578701, upload-time = "2026-05-28T12:00:37.82Z" }, + { url = "https://files.pythonhosted.org/packages/65/56/b5f6fdb2083e32bca8a8993d89e70db114b4756c9e2c38421328126689d2/rpds_py-2026.5.1-cp314-cp314-win32.whl", hash = "sha256:2d88621d6a7d4dfa633d21abe90f280bb205274e16b1d1e61c6ad4640b2453b7", size = 209806, upload-time = "2026-05-28T12:00:39.492Z" }, + { url = "https://files.pythonhosted.org/packages/fb/80/65a5aa96c155e611d1ed844e4e1f57f3e36b021f396d9f8585d756e6b90d/rpds_py-2026.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:cef8ac28d26f4dda3533060c20fbf80a325458fa9fd23ea72a73cdfa8e978838", size = 225985, upload-time = "2026-05-28T12:00:40.94Z" }, + { url = "https://files.pythonhosted.org/packages/27/7c/ad185212e87b05f196daef92bc5f3caf07298eb47c295b5585c3dd3093ac/rpds_py-2026.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:eaaea962c68cdc68d4a533ba985ab8e9484277910bbfaa2ab3ef7732667bfed8", size = 221219, upload-time = "2026-05-28T12:00:43.15Z" }, + { url = "https://files.pythonhosted.org/packages/23/58/e14ae18759020334646b031e708ab4158d653a938822bfb7b95ef2e93aa3/rpds_py-2026.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:21942f52dbbd5f8758bf021213d28bd45c39e873e65e2407faf5f1846f5761ad", size = 352148, upload-time = "2026-05-28T12:00:44.638Z" }, + { url = "https://files.pythonhosted.org/packages/31/9b/5f4a1e2f960bca3ac5d052b139dd31eed97b259f9d909173821760d542e8/rpds_py-2026.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f414556f6e3958300ff941e40c9f97e3dc9774ddd1b3434c475d73dd354bbed3", size = 345196, upload-time = "2026-05-28T12:00:46.14Z" }, + { url = "https://files.pythonhosted.org/packages/1a/71/1d9574d6a2fa20ab60eaa55c7467f5aa20cbc770f341a05f09c0876f59e2/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef1013a8625c74043210190b246f5b1551e09757c1f356c6e4160ef96c5bc081", size = 374981, upload-time = "2026-05-28T12:00:47.531Z" }, + { url = "https://files.pythonhosted.org/packages/0c/9a/37e99f4915a80aa71670263c1267f7ae0af95f53a3f61e6c3bdc016d4515/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cc68e231a77a5f0d774ae278a1f8e55c0456501820847c1e4efb3829f3441df6", size = 379961, upload-time = "2026-05-28T12:00:49.216Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ff/6e73f74b89d2e0715e0fc86b7dde893f9a61ae2f9b256ff3bdfe41ac4e94/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9baffb505aff33acc69b422a19f77806680f3c8632227d79f48de8a810d1c2c5", size = 495965, upload-time = "2026-05-28T12:00:51.111Z" }, + { url = "https://files.pythonhosted.org/packages/ea/e0/425faba25f59d74d4638b267f7c7a80e8649d2ef4db10a19b0c4a71e6e6f/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8d2f912928d426e8cfa396f7f3f8d29a59e6689c86dcca3c420730c1096322b", size = 389526, upload-time = "2026-05-28T12:00:52.77Z" }, + { url = "https://files.pythonhosted.org/packages/c6/76/7a41960e3fddae47fab43a28684d5da981401dffd88253de0944148654cb/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90f628283be835db980c941767d41c9a27b5239e54ba0a9c1335247e82406964", size = 376190, upload-time = "2026-05-28T12:00:54.215Z" }, + { url = "https://files.pythonhosted.org/packages/27/60/5f38dc70824fc6951b51d35377e577a3a3a4c81a6769cc5a2de25ebe0ad1/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:1ebb2f0ab7e16132995a72de805170e0203df0c3dd22e1ef1cd1fdd90bd7a131", size = 383921, upload-time = "2026-05-28T12:00:55.673Z" }, + { url = "https://files.pythonhosted.org/packages/60/1a/d60a38caa1505f4b9483c3fbbde12c94e1079154f4f401a6da96f7e77621/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f3df3d16ded76f1f8c9cdebd0e1ea55fdf4c23b812de189814da7cf229c22a81", size = 404766, upload-time = "2026-05-28T12:00:57.518Z" }, + { url = "https://files.pythonhosted.org/packages/87/ff/602fd3f174d6425f0bce05ad0dfbec0e96b38d0f7d08a79af5aa20083885/rpds_py-2026.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9af8905b8f854990e40d5206aa5ac58d9b0fe0b7f351ff2bb086c20f6c8c6a47", size = 551343, upload-time = "2026-05-28T12:00:58.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/c1/1be13327acdbead3eca1fde03b6a34dbb011f1e864e217f0d32cc1779a7f/rpds_py-2026.5.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:036a36a87fb1cd3b214d11c4b3c4f7d2ddad933625dca1c900b56a057c07740a", size = 618502, upload-time = "2026-05-28T12:01:00.656Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d7/afb49b49d7f2be8b7ba1a9f0977fa5168003437b93086726f066544e8351/rpds_py-2026.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:62ae3853454fe9ef283a03c96c2d835d39e84b14643a9d62c82ef0fb87d702ca", size = 581916, upload-time = "2026-05-28T12:01:02.22Z" }, + { url = "https://files.pythonhosted.org/packages/25/d1/dbef8c1f8a10f07beb62b5f054e20099fd9924b3ec001b8f0b6ac7813a85/rpds_py-2026.5.1-cp314-cp314t-win32.whl", hash = "sha256:6c3d771a46ec18b12af06ce36243a9a80b07a5d0515236332d90863ca8bb326a", size = 207855, upload-time = "2026-05-28T12:01:03.821Z" }, + { url = "https://files.pythonhosted.org/packages/2a/72/bfa4e61ab8e7dc1c8adf397e05e6cbdd4239357bd72b248d3de662f23915/rpds_py-2026.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c93c629be4636cf54337bd5f06c104d55e42ced54d681f6fe21ae510a65116f6", size = 225422, upload-time = "2026-05-28T12:01:05.194Z" }, + { url = "https://files.pythonhosted.org/packages/27/3a/7b5da92b640f67b6717ccafc83cdd06bfa7ff2395c3685c68922bb54d703/rpds_py-2026.5.1-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:3574b55c604b8f75dacb007136508bbc0db406e626301778096a133327e7f2fb", size = 349576, upload-time = "2026-05-28T12:01:06.722Z" }, + { url = "https://files.pythonhosted.org/packages/d7/8a/2aafd7ad355a1bd48ca76e2262b74b15e6432b5a1efe150efd4d779cd55d/rpds_py-2026.5.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:94068eb3ae6d43f5a786b7db96a406a34e6d5c24489feef32fd6e8946ea7b291", size = 343640, upload-time = "2026-05-28T12:01:08.441Z" }, + { url = "https://files.pythonhosted.org/packages/f7/7d/6c9523c1abbe840a1b7fba3c516d48e1d3487cc80fea4366c4071cf56784/rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3a5b10e8ce894825f380a8f1b6444cf73c294dfea62afbb2d13e3a9e630cec1", size = 375322, upload-time = "2026-05-28T12:01:09.934Z" }, + { url = "https://files.pythonhosted.org/packages/5a/5d/0b7b03fb1dc509321f01de3149784ab773e34c8573022029af8076afcb9c/rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fc09f82e63d4bcd58149572f857a431bae851dc747e313c3b5bdf7abb907fda8", size = 379066, upload-time = "2026-05-28T12:01:11.48Z" }, + { url = "https://files.pythonhosted.org/packages/d7/e2/8ef6012999ebf1cb1c22f876d9ce5e63d960fd4631d2af3202d3f480aa25/rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e10464d17df3b582745c25cec695cb9558bca2cb6ddb631aee1787fc72c767b2", size = 494586, upload-time = "2026-05-28T12:01:13.051Z" }, + { url = "https://files.pythonhosted.org/packages/80/af/1eeb029bec67582c226b7809172207cd005073af4ebd906e65ff494f4983/rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ba05adbf15d994c38ec0b7ab32e858e5110c21e9009a00a86545fd220f84e038", size = 388415, upload-time = "2026-05-28T12:01:14.631Z" }, + { url = "https://files.pythonhosted.org/packages/18/23/ffbe10711c4d766c1cab0557d6906c074f795814863c67b351355d29354a/rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77c004fdc7b891967106f78ddfd7b076bfe6813c6139c6fff6aed3bcaa960b26", size = 372427, upload-time = "2026-05-28T12:01:16.153Z" }, + { url = "https://files.pythonhosted.org/packages/bd/3a/30ba4a6ad457e5b070c18d742a33fb77d8d922b565cc881f8a5313d63bfe/rpds_py-2026.5.1-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:83bcf894486c9d78dd290d3c0124ff6dd8875d3025e2090a8ec49fcc37c55fdd", size = 383615, upload-time = "2026-05-28T12:01:17.809Z" }, + { url = "https://files.pythonhosted.org/packages/d3/69/62e242b53ce39c0814bd24e1a6e6eba6c92be716277745f317f9540a2e7b/rpds_py-2026.5.1-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c3df104083952a0e0c6f10de33e440eabe98fb6317d23e1a58c68f6df08d01b9", size = 402786, upload-time = "2026-05-28T12:01:19.419Z" }, + { url = "https://files.pythonhosted.org/packages/38/c1/a770b9c186928a1ed0f7e6d7ae50e7f3950ed23e3f9e366dbc8e38cb55de/rpds_py-2026.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:980450826cf22e133c57e0835070bdd0dd3f73b9b708c3ce223def2cb9469e14", size = 551583, upload-time = "2026-05-28T12:01:21.013Z" }, + { url = "https://files.pythonhosted.org/packages/21/7c/68e8579b95375b70d2a963103c42e705856cdb98569258bd807f4423891c/rpds_py-2026.5.1-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:205dde846f24332ab0c1188699a043b8d165b79bb84529ce272c45048ff6be01", size = 616941, upload-time = "2026-05-28T12:01:22.548Z" }, + { url = "https://files.pythonhosted.org/packages/70/a1/a6135aed5730ff03ab957182259987ac11e55fb392a28dc6f0592048a280/rpds_py-2026.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:3966b82dd563176396df030f3dd52a6e54cb69b718e95e78bd555ed3d1e0185d", size = 578349, upload-time = "2026-05-28T12:01:24.118Z" }, + { url = "https://files.pythonhosted.org/packages/09/6e/f24201a76a84e6c49d0bdfdfcb735210e21701e9b21c5bfc0ba497dd62f6/rpds_py-2026.5.1-cp315-cp315-win32.whl", hash = "sha256:7818f8d0a415be74d2be3590b0a1c1f463a642f4d0217e7d10602dceef5b79aa", size = 209922, upload-time = "2026-05-28T12:01:25.522Z" }, + { url = "https://files.pythonhosted.org/packages/9e/e4/966bc240bb0485fc265278f6de44d05834bf0b3618886e0b22e33d54c49a/rpds_py-2026.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:b3cc20c0d800af78fd0fac68086e28c1856cec51ea528bb81ea851aa40d39325", size = 226003, upload-time = "2026-05-28T12:01:27.062Z" }, + { url = "https://files.pythonhosted.org/packages/5c/5c/a15a59269cd5e74472734516c73795c15eccfc841b3d4b0228c3f53f19d0/rpds_py-2026.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:3609e9939a8a76cd904cf98a3f1f13b5dc7e150adeaee89e0ea09652ea213e16", size = 221245, upload-time = "2026-05-28T12:01:28.51Z" }, + { url = "https://files.pythonhosted.org/packages/e0/22/135ce03804e179a71ceb13be095deda4a279bc88f7a6b8fa161c5ad44e12/rpds_py-2026.5.1-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:5d333a7127d4b307601ac37792bee01bb95c867cbfacf21b6375b804d6bbd723", size = 352015, upload-time = "2026-05-28T12:01:30.214Z" }, + { url = "https://files.pythonhosted.org/packages/3b/5f/f1f6d2652eb9d848f6eb369d8db83a2da6249bb49ad2c2a48f45d54538d3/rpds_py-2026.5.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:b5f077b44a4f7808520f66dae234988d867deb9aed9be5da057ce9ba831b2a41", size = 345016, upload-time = "2026-05-28T12:01:31.656Z" }, + { url = "https://files.pythonhosted.org/packages/88/66/b74182775691ea2290c99e52ac8d5db844e56fbec90ce421f107658c8314/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55d8f9b7b78c9538fc9e04e82ec0e888ff0c3cffcfad152c77e57cd09351a98a", size = 374775, upload-time = "2026-05-28T12:01:33.136Z" }, + { url = "https://files.pythonhosted.org/packages/ff/8f/15e5a61d9f0a43902d36561d4f07cae6ae9f4716be825159fd72717f33af/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e3a8ae58895ac107ed934a6bf51e5846f95c53b9b940c2c6d310838fd5846358", size = 380270, upload-time = "2026-05-28T12:01:34.574Z" }, + { url = "https://files.pythonhosted.org/packages/02/c3/f859b12763a80540cdf2af0f15b19904cf756a71d7bdd3f82ff3e5b1bbf9/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0957cf3c2b8632ec7aaebffebea8005b353cc2a237b6e2ae3c2cac0820704cfb", size = 495285, upload-time = "2026-05-28T12:01:36.127Z" }, + { url = "https://files.pythonhosted.org/packages/1c/c7/ff27c2ac8411d30b03b1829fd88cae8dad1a4d0da48dd25e57c4038042e6/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c396c1304de421050b3681ea70f371874b54d41b0151e96109758144c231e30b", size = 389581, upload-time = "2026-05-28T12:01:37.635Z" }, + { url = "https://files.pythonhosted.org/packages/6e/67/fe92ee32a6cc05c77228a2f8b1762e7124f386ec20ff83d0757b762d58d0/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad1bff7f666b9598e573815affd666aac6a13a585dde336f843e33350c7fadc", size = 376041, upload-time = "2026-05-28T12:01:39.307Z" }, + { url = "https://files.pythonhosted.org/packages/f8/91/b4d6685c27aba55bd82f25b278be8237038117d05f9659a6213ad3408130/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:656a042550878f12d45752452d47094b7cfe5ad1e9d7b87b5a22ad3ae5ff8015", size = 383946, upload-time = "2026-05-28T12:01:41.043Z" }, + { url = "https://files.pythonhosted.org/packages/bd/79/2c1d832a53c8e0f8e98fc970ec257b950fecd4f62be2ab7182b500a0cbc8/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:73c4bd4f70294737b5206a3e8e30ccadbf8a60301831c8ea23eec5dbeea1ecfa", size = 405526, upload-time = "2026-05-28T12:01:43.032Z" }, + { url = "https://files.pythonhosted.org/packages/78/c4/c98117b03c6a8581ab2c2dfccfe9a5ad82bd8128a3c28b46a6ad2d97c393/rpds_py-2026.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:43bca78665423cabae77146f2fe7ce55272b6c8d55d82cca83effd42c7e13972", size = 551165, upload-time = "2026-05-28T12:01:44.648Z" }, + { url = "https://files.pythonhosted.org/packages/3b/c1/bc479ca069200af730881b1bd525e3114b2b391a351509fcb1b772f28086/rpds_py-2026.5.1-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:42d0f20e85e549c870749d0e247f0c10d318a45b7e9676d575d2dcb04a1b2e66", size = 618778, upload-time = "2026-05-28T12:01:46.337Z" }, + { url = "https://files.pythonhosted.org/packages/77/65/38ab2f90df44c2febfb63cc10ced40763d9b4bc94d173e734528663fe7f5/rpds_py-2026.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:b1be5c35683684d5331b93600c210e8367c254683d8a6df6bd21bd2da3a334fb", size = 581839, upload-time = "2026-05-28T12:01:48.109Z" }, + { url = "https://files.pythonhosted.org/packages/15/2d/ce1f605fe036aadd460e5822e578c6c7ec3a860936cca37d6e0f299daa77/rpds_py-2026.5.1-cp315-cp315t-win32.whl", hash = "sha256:75808f6c38ce7749bb68cc2770161aae5045e6c6f6781a9782e74b93304399df", size = 207866, upload-time = "2026-05-28T12:01:49.648Z" }, + { url = "https://files.pythonhosted.org/packages/79/cb/966040123eb102371559746908ef2c9471f4d43e17ec9a645a2258dab64b/rpds_py-2026.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:90bd6630002a1c7f09e7843dd79f0d24f3d2897cc25a753480917865d14f15b3", size = 225441, upload-time = "2026-05-28T12:01:51.408Z" }, ] [[package]] name = "ruff" -version = "0.15.14" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/dc/8a/8bce2894573e9dae6ff4d77fe34ad727d79b9e6238ad288c5638990d90f6/ruff-0.15.14.tar.gz", hash = "sha256:48e866b165be4a9bdbf310f7d3c9a07edef2fe8cd63ffeb4e00bb590506ebf9f", size = 4700910, upload-time = "2026-05-21T14:34:55.177Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b9/c8/74a92c6ff9fcfb4f1f947126d3ebee8389276e161ecc85de5bda7cda51bd/ruff-0.15.14-py3-none-linux_armv6l.whl", hash = "sha256:8dd2db9416e487c8d4b01fa7056bb02c4d05969d4f8d17a08c229c2f4ff3c108", size = 10739177, upload-time = "2026-05-21T14:34:37.332Z" }, - { url = "https://files.pythonhosted.org/packages/45/91/254a35c20acc38a7223c9d2d594af12e794432464f2cdeb52af1dc4a892d/ruff-0.15.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:be4ff55af755bd71a00ab3dc6bd7ffc467bd76e0df6881e286c2e3d23e8fb43b", size = 11144969, upload-time = "2026-05-21T14:34:43.978Z" }, - { url = "https://files.pythonhosted.org/packages/56/9e/d13e40f83b8d0a94430e6778ce1d94a43b38cf2efe63278bdd2b4c65abbf/ruff-0.15.14-py3-none-macosx_11_0_arm64.whl", hash = "sha256:48d5909d7d06276ce7dde6d32bfa4b0d4cb2651145cd8ee4b440722cbc77832f", size = 10478207, upload-time = "2026-05-21T14:34:48.378Z" }, - { url = "https://files.pythonhosted.org/packages/8d/f1/b15a7839fa4f332f8acec78e20564f26bb2d866e3d21710b877fd0263000/ruff-0.15.14-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca8cbfa94c4f90984a67561978602746d4cd27103568f745fa90eee3f0d4107d", size = 10818459, upload-time = "2026-05-21T14:34:22.318Z" }, - { url = "https://files.pythonhosted.org/packages/45/33/53d651177f84f94b400a0e27f8824eeada3dddc9d5ee8aeb048f4352a520/ruff-0.15.14-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9a6bbc0333f1ab053423bcbf6226477d266ca7cec7738c4c8e3f55647803f3c4", size = 10541800, upload-time = "2026-05-21T14:34:20.209Z" }, - { url = "https://files.pythonhosted.org/packages/b8/a6/868f87e0bf9786ed24b5d0d0ad8676b8a94fd1912f42cddf9cfc7857818a/ruff-0.15.14-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a24a4f7605d7003a6674d4387651effd939dead3fddd0f36561eb77a9a2e542", size = 11342149, upload-time = "2026-05-21T14:34:46.365Z" }, - { url = "https://files.pythonhosted.org/packages/a7/8b/38cd5c19faffdcc05a408d2b78edccc69492ab9720eadb49ea15ef80d768/ruff-0.15.14-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:049b5326e53ed80978f2fc041a280603f69dd6b0c95464342a2bb4572d9d9e2f", size = 12212563, upload-time = "2026-05-21T14:34:28.579Z" }, - { url = "https://files.pythonhosted.org/packages/3e/4d/a3c5b874a556d5731e3e657aaf04311bb76f0a5c3ec220ed43051be6b64b/ruff-0.15.14-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4ed42e6696c8dfa5f06728e6441993901f548eb92d73bc472cb5a38d1395fbf", size = 11493299, upload-time = "2026-05-21T14:34:41.836Z" }, - { url = "https://files.pythonhosted.org/packages/1e/c0/56472c251d09858a53e51efbd485b09e1995d8731668b76d52e5dd6ee0f1/ruff-0.15.14-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:715c543cf450c4888251f91c52f1942a800541d9bddd7ac060aa4e6b77ae7cba", size = 11455931, upload-time = "2026-05-21T14:34:57.276Z" }, - { url = "https://files.pythonhosted.org/packages/2c/4a/e2e7b4d8dbf233d4eace59c75bc3435fa6d8bd3bae82d351d4e4300c0fd1/ruff-0.15.14-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:72ebab6013ec887d439d8b7593737a0a4ffb06d45d209d4e4bf2e92813082d3f", size = 11400794, upload-time = "2026-05-21T14:34:39.773Z" }, - { url = "https://files.pythonhosted.org/packages/97/c7/83c0539fe34c3e09136204d1e75d6052492364e0b3cb05e9465423f567d7/ruff-0.15.14-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:49072d36abdbe97a8dd7f480afe9c675699c0c495d4c84076e2c1203c4550581", size = 10804759, upload-time = "2026-05-21T14:34:31.045Z" }, - { url = "https://files.pythonhosted.org/packages/86/a6/18f2bfc095a2ab4a78745644e428205532ce6653a5d0fa8501572891534d/ruff-0.15.14-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:958522aee105068640c2c2ceae08f413ae44d922f52a1374ac13d6a96032fc93", size = 10539517, upload-time = "2026-05-21T14:34:53.064Z" }, - { url = "https://files.pythonhosted.org/packages/54/3a/5a8b3b69c654d4e4bf1d246ac5b49cbcdac6eaab6905925f8915f31e3b80/ruff-0.15.14-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f3707da619a143a2e8830e2abab8224478d69ace2d28cb6c20543ae97c36bf61", size = 11065169, upload-time = "2026-05-21T14:34:24.484Z" }, - { url = "https://files.pythonhosted.org/packages/ed/c5/8864e4e7925b836ea354b31d57641ec03830564e281a8b6f061f8c3e0ec1/ruff-0.15.14-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:bb01d645694e3ec0102105d07ef2d53703970407d59c04e59d3ba0b7a1d53553", size = 11560214, upload-time = "2026-05-21T14:34:50.975Z" }, - { url = "https://files.pythonhosted.org/packages/36/38/012bf76752e1f89ed50b77b99532d90f3a3e287bc7918e1fc0948ac866ac/ruff-0.15.14-py3-none-win32.whl", hash = "sha256:6d0c1ad2a0ab718d39b6d8fd2217981ce4d625cd96a720095f798fb47d8b13e6", size = 10805548, upload-time = "2026-05-21T14:34:33.453Z" }, - { url = "https://files.pythonhosted.org/packages/d1/b7/4ea2c170f10ad760fff2a5250beb18897719dc8b52b53a24cddbb9dd3f19/ruff-0.15.14-py3-none-win_amd64.whl", hash = "sha256:802342981e056db3851a7836e5b070f8f15f67d4a685ae2a6160939d364b2902", size = 11939523, upload-time = "2026-05-21T14:34:18.077Z" }, - { url = "https://files.pythonhosted.org/packages/62/d5/bc97ff895ec35cf3925d4bd60f3b39d822f377a446906ec9bcc87405e59b/ruff-0.15.14-py3-none-win_arm64.whl", hash = "sha256:ff47b90a9ef6a40c9e2f3b479c1fb78531adf055b94c1eba0a7ba04b31951826", size = 11208607, upload-time = "2026-05-21T14:34:26.525Z" }, +version = "0.15.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/84/6f/a76f7d96e5c962f5b69cee865e49c15c1116897c01990faa8a57edb62e7f/ruff-0.15.15.tar.gz", hash = "sha256:b8dff018130b46d8e5bf0f926ef6b60cf871d6d5ae45fc9334e09632daa741d6", size = 4706985, upload-time = "2026-05-28T14:16:57.784Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/9d/3a45c05b8ab04b4705989de70a79008e27c8003296a0feaee9edc18dd7e9/ruff-0.15.15-py3-none-linux_armv6l.whl", hash = "sha256:cf93e5388f412e1b108b1f8b34a6e036b70fe8aff89393befad96fe48670311b", size = 10710652, upload-time = "2026-05-28T14:16:06.701Z" }, + { url = "https://files.pythonhosted.org/packages/05/66/da974431624bf3b49f6ee1f9543c02d929ff1cba78b0d5a79c38cf21f744/ruff-0.15.15-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ac5a646d1f6a7dadd5d50842dae2c1f9862ac887ef5d1b1375e02def791fde6e", size = 11096615, upload-time = "2026-05-28T14:16:23.313Z" }, + { url = "https://files.pythonhosted.org/packages/8c/09/7443452e5d290230a712103f2fdceeef7184f3ec99a2bd01c8be78aaceb5/ruff-0.15.15-py3-none-macosx_11_0_arm64.whl", hash = "sha256:77d955a431430c66f72dd94e379ad38a16daea3d25094872ac4edf9e797be530", size = 10436683, upload-time = "2026-05-28T14:16:40.974Z" }, + { url = "https://files.pythonhosted.org/packages/53/01/d330c26a57fa4f3943a14424904027428315b700fe4d14a84bb123a649e5/ruff-0.15.15-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7614ee79c69788cf6cedd568069ade9cecc22a1ad20494efe8d0c9ebb4b622d4", size = 10769064, upload-time = "2026-05-28T14:16:28.905Z" }, + { url = "https://files.pythonhosted.org/packages/1d/85/cc8770f8bdff541b1da8392d1634141fe4a0e3f4ee596605959b7906c27f/ruff-0.15.15-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3cdb1679e06a1f6b47bc384714ae96f6e2fb65ca441eb78c43d2ca554176ce1f", size = 10511987, upload-time = "2026-05-28T14:16:43.732Z" }, + { url = "https://files.pythonhosted.org/packages/7c/29/8c190c1472b63013583ba391f3342036e02010544c1270455ed8e519bdf3/ruff-0.15.15-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2728b93d7b23a603ea2c0ac6eb73d760bd38ec9de35f35fb41e18f7a3fee7622", size = 11275100, upload-time = "2026-05-28T14:16:55.244Z" }, + { url = "https://files.pythonhosted.org/packages/9f/6b/7e145ce2cc8e63d6834eca03d83a0e18d121def5c69f91b4cf4011ed4879/ruff-0.15.15-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be582fcc0db438902c7792b08d6ddf6c9b9e21addaa10092c2c741cfb09e5a45", size = 12176903, upload-time = "2026-05-28T14:16:14.368Z" }, + { url = "https://files.pythonhosted.org/packages/80/a3/d5974637f68e451f7fadf015cf3101d1cd7d8ba5027cffe0b9e3826ebe6b/ruff-0.15.15-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7aa77465b8ecaf1a27bea098d696f7fed5e1eccbd10b321b682d6de586ae5627", size = 11404550, upload-time = "2026-05-28T14:16:20.138Z" }, + { url = "https://files.pythonhosted.org/packages/fe/1c/e6e5e568f22be4fb05d6244234aba384c06b451252453b821e1a529263cf/ruff-0.15.15-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48decfa11d740de4889de623be1463308346312f2409a56e24aa280c86162dc4", size = 11382027, upload-time = "2026-05-28T14:16:46.615Z" }, + { url = "https://files.pythonhosted.org/packages/1d/01/170921b49fcd2e8858825593f91cf7146c3e40a5c3e6df763e4bb0484dde/ruff-0.15.15-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:a5015088452ca0081387063649ec67f06d3d1d6b8b936a1f836b5e9657ecd48c", size = 11366041, upload-time = "2026-05-28T14:16:26.247Z" }, + { url = "https://files.pythonhosted.org/packages/87/54/a7bad711d7de93254e15e06a4c375b89a03d18de45d3e5dcc86a4472fb1a/ruff-0.15.15-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:f5294aab6356c81600fcdea3a62bb1b924dfd5e91767c12318d3f68f86af57cd", size = 10741795, upload-time = "2026-05-28T14:16:17.11Z" }, + { url = "https://files.pythonhosted.org/packages/c9/31/38c075963668f8b41c6914ee0f6f318727fbe30ab9145cb29e6df464c5fa/ruff-0.15.15-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:db5bd4d802415cca656dc1616070b725952d6ae95eb5d4831e49fbd94a38f75f", size = 10511117, upload-time = "2026-05-28T14:16:31.767Z" }, + { url = "https://files.pythonhosted.org/packages/9d/96/6ff689e1f7e375d1d97075eca022f74c2bab59554a432fe4d2e6f091986a/ruff-0.15.15-py3-none-musllinux_1_2_i686.whl", hash = "sha256:587a6278ed42059191c1a466e490bd7930fb50bd2e255398bc29616c895a61cb", size = 10994867, upload-time = "2026-05-28T14:16:35.149Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c2/5dce0ab9f92a8d534fa62b9bf9caca3eddb8c1a81b616f5e195ada4f0d6e/ruff-0.15.15-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:df0c1c084f5f4be9812f61518a45c440d3c30d69ce4bf6c5270e66d38338f02a", size = 11482101, upload-time = "2026-05-28T14:16:49.598Z" }, + { url = "https://files.pythonhosted.org/packages/b1/c0/1003b60edd697c649faf61f1a34094b1abb38fb3d1181e3f895781250a08/ruff-0.15.15-py3-none-win32.whl", hash = "sha256:29428ea79694afbe756d45fd59b36f22b6b020dc0443cf7de0173046236964b9", size = 10716774, upload-time = "2026-05-28T14:16:52.337Z" }, + { url = "https://files.pythonhosted.org/packages/02/a8/1269eddd6945a06c23f055ef7848886e37cf9d6a8bebb386a3115f01470c/ruff-0.15.15-py3-none-win_amd64.whl", hash = "sha256:8df0323902e15e24bc4bf246da830573d3cf3352bd0b9a164eab335d111ff4a4", size = 11868463, upload-time = "2026-05-28T14:16:11.333Z" }, + { url = "https://files.pythonhosted.org/packages/4e/b2/920464c907b191e37469d477a1aa8bc048b8f36c4c1610dfa4ab87b39e18/ruff-0.15.15-py3-none-win_arm64.whl", hash = "sha256:3c8ceca6792f38196b8f589bc92eccd03eef286602da92e5dc05cc42ef6441b7", size = 11138498, upload-time = "2026-05-28T14:16:38.425Z" }, ] [[package]] @@ -3509,14 +3541,14 @@ wheels = [ [[package]] name = "s3transfer" -version = "0.16.0" +version = "0.17.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/05/04/74127fc843314818edfa81b5540e26dd537353b123a4edc563109d8f17dd/s3transfer-0.16.0.tar.gz", hash = "sha256:8e990f13268025792229cd52fa10cb7163744bf56e719e0b9cb925ab79abf920", size = 153827, upload-time = "2025-12-01T02:30:59.114Z" } +sdist = { url = "https://files.pythonhosted.org/packages/11/b3/bcdc2f58fa92592db511beda154c2c08d28f21f6c4637f06a42a24b10c21/s3transfer-0.17.1.tar.gz", hash = "sha256:042dd5e3b1b512355e35a23f0223e426b7042e80b97830ea2680ddce327fc45e", size = 159439, upload-time = "2026-05-26T19:45:01.714Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl", hash = "sha256:18e25d66fed509e3868dc1572b3f427ff947dd2c56f844a5bf09481ad3f3b2fe", size = 86830, upload-time = "2025-12-01T02:30:57.729Z" }, + { url = "https://files.pythonhosted.org/packages/85/dd/904873250a6554fbae40cddbf9198e3cc37a2f1319d5e1a5ce82fe269c17/s3transfer-0.17.1-py3-none-any.whl", hash = "sha256:5b9827d1044159bbb01b86ef8902760ea39281927f5de31de75e1d657177bf4c", size = 88264, upload-time = "2026-05-26T19:45:00.452Z" }, ] [[package]] @@ -3694,7 +3726,7 @@ wheels = [ [[package]] name = "sssom-pydantic" -version = "0.5.9" +version = "0.5.10" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "curies" }, @@ -3703,9 +3735,9 @@ dependencies = [ { name = "pyyaml" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/85/4a/86451b0a6410089b1734fa612f8989392aec5856453686d251ba99bbf465/sssom_pydantic-0.5.9.tar.gz", hash = "sha256:64870feaa5ef8f5b41fcac4028a5c51b0fd8b74ea7ee1c55ed23687119cc3b29", size = 61842, upload-time = "2026-05-26T20:04:21.393Z" } +sdist = { url = "https://files.pythonhosted.org/packages/95/4b/a4902af1cf9e55a4b05310594b025d163bfe0564d6479534beb1244038ef/sssom_pydantic-0.5.10.tar.gz", hash = "sha256:f011e5550eecfa006354dd62bdd732f477123ff470c19cac8b80dc4d46f7f095", size = 62512, upload-time = "2026-05-27T09:42:46.816Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/80/04/2a079709862c79484dec82a91feaab09dbd7442a4f2b529214927f79e0a9/sssom_pydantic-0.5.9-py3-none-any.whl", hash = "sha256:251cb1d7c09ff400f30487aa04d7dbdc8eaa5547ae7e84ae93a4569206b174e8", size = 75361, upload-time = "2026-05-26T20:04:19.33Z" }, + { url = "https://files.pythonhosted.org/packages/57/94/1cd079f84f29b619af179f2e3200d2d6e6f49ac9af451a6e0fad890e5b85/sssom_pydantic-0.5.10-py3-none-any.whl", hash = "sha256:b22f0473d6c5021aabda5e776a30f173b15bd7fccec67946fee2b9f52e7fadb4", size = 75873, upload-time = "2026-05-27T09:42:45.494Z" }, ] [[package]] @@ -3724,14 +3756,14 @@ wheels = [ [[package]] name = "starlette" -version = "1.1.0" +version = "1.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/95/66/4d20cdf39a8d6a51e663b7038e3b828ff211d3891a43a713fe7e4643f3a8/starlette-1.1.0.tar.gz", hash = "sha256:e83c7fe0ddecd8719c5b840080325aec0260acec86e9832899e377b91d65e90f", size = 2660060, upload-time = "2026-05-23T16:55:41.376Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c5/bf/616a066c2760f6c2b1ae3437cc28149734d069fbb46511712beae118a68c/starlette-1.2.0.tar.gz", hash = "sha256:3c5a6b23fff42492914e93890bb80cbfea72dbf37de268eec06185d62a4ca553", size = 2668923, upload-time = "2026-05-28T11:42:50.568Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/93/79/920b8e0a8b20f793e8d64855095cb8febabf6175b8550b6f7a547d813891/starlette-1.1.0-py3-none-any.whl", hash = "sha256:7f0dfd38e428aad5cb6f9f667f0ca1d2d8ca3f3385dccac8305f79ec98458382", size = 72899, upload-time = "2026-05-23T16:55:39.201Z" }, + { url = "https://files.pythonhosted.org/packages/9f/85/492183764d5d01d4514be3730fdb8e228a80605783099551c51627578b5d/starlette-1.2.0-py3-none-any.whl", hash = "sha256:36e0c76ac59157e75dc4b3bdeafba97fb04eaf1878045f15dbef666a6f092ed7", size = 73213, upload-time = "2026-05-28T11:42:48.801Z" }, ] [[package]] @@ -3827,19 +3859,19 @@ wheels = [ [[package]] name = "tornado" -version = "6.5.5" +version = "6.5.6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f8/f1/3173dfa4a18db4a9b03e5d55325559dab51ee653763bb8745a75af491286/tornado-6.5.5.tar.gz", hash = "sha256:192b8f3ea91bd7f1f50c06955416ed76c6b72f96779b962f07f911b91e8d30e9", size = 516006, upload-time = "2026-03-10T21:31:02.067Z" } +sdist = { url = "https://files.pythonhosted.org/packages/50/57/6d7303a77ae439d9189108f76c0c4fd89ee5e2cc8387bffb55232565c4ed/tornado-6.5.6.tar.gz", hash = "sha256:9a365179fe8ff6b8766f602c0f67c185d778193e9bdd828b19f0b6ed7764177d", size = 518139, upload-time = "2026-05-27T15:35:54.646Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/59/8c/77f5097695f4dd8255ecbd08b2a1ed8ba8b953d337804dd7080f199e12bf/tornado-6.5.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:487dc9cc380e29f58c7ab88f9e27cdeef04b2140862e5076a66fb6bb68bb1bfa", size = 445983, upload-time = "2026-03-10T21:30:44.28Z" }, - { url = "https://files.pythonhosted.org/packages/ab/5e/7625b76cd10f98f1516c36ce0346de62061156352353ef2da44e5c21523c/tornado-6.5.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:65a7f1d46d4bb41df1ac99f5fcb685fb25c7e61613742d5108b010975a9a6521", size = 444246, upload-time = "2026-03-10T21:30:46.571Z" }, - { url = "https://files.pythonhosted.org/packages/b2/04/7b5705d5b3c0fab088f434f9c83edac1573830ca49ccf29fb83bf7178eec/tornado-6.5.5-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e74c92e8e65086b338fd56333fb9a68b9f6f2fe7ad532645a290a464bcf46be5", size = 447229, upload-time = "2026-03-10T21:30:48.273Z" }, - { url = "https://files.pythonhosted.org/packages/34/01/74e034a30ef59afb4097ef8659515e96a39d910b712a89af76f5e4e1f93c/tornado-6.5.5-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:435319e9e340276428bbdb4e7fa732c2d399386d1de5686cb331ec8eee754f07", size = 448192, upload-time = "2026-03-10T21:30:51.22Z" }, - { url = "https://files.pythonhosted.org/packages/be/00/fe9e02c5a96429fce1a1d15a517f5d8444f9c412e0bb9eadfbe3b0fc55bf/tornado-6.5.5-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3f54aa540bdbfee7b9eb268ead60e7d199de5021facd276819c193c0fb28ea4e", size = 448039, upload-time = "2026-03-10T21:30:53.52Z" }, - { url = "https://files.pythonhosted.org/packages/82/9e/656ee4cec0398b1d18d0f1eb6372c41c6b889722641d84948351ae19556d/tornado-6.5.5-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:36abed1754faeb80fbd6e64db2758091e1320f6bba74a4cf8c09cd18ccce8aca", size = 447445, upload-time = "2026-03-10T21:30:55.541Z" }, - { url = "https://files.pythonhosted.org/packages/5a/76/4921c00511f88af86a33de770d64141170f1cfd9c00311aea689949e274e/tornado-6.5.5-cp39-abi3-win32.whl", hash = "sha256:dd3eafaaeec1c7f2f8fdcd5f964e8907ad788fe8a5a32c4426fbbdda621223b7", size = 448582, upload-time = "2026-03-10T21:30:57.142Z" }, - { url = "https://files.pythonhosted.org/packages/2c/23/f6c6112a04d28eed765e374435fb1a9198f73e1ec4b4024184f21faeb1ad/tornado-6.5.5-cp39-abi3-win_amd64.whl", hash = "sha256:6443a794ba961a9f619b1ae926a2e900ac20c34483eea67be4ed8f1e58d3ef7b", size = 448990, upload-time = "2026-03-10T21:30:58.857Z" }, - { url = "https://files.pythonhosted.org/packages/b7/c8/876602cbc96469911f0939f703453c1157b0c826ecb05bdd32e023397d4e/tornado-6.5.5-cp39-abi3-win_arm64.whl", hash = "sha256:2c9a876e094109333f888539ddb2de4361743e5d21eece20688e3e351e4990a6", size = 448016, upload-time = "2026-03-10T21:31:00.43Z" }, + { url = "https://files.pythonhosted.org/packages/1b/0d/b4f481e18c5a51864e6d12b9a05ecf72919696680b747c958c3fc1f4fbae/tornado-6.5.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:65fcfaafb079435c2c19dc9e07c0f1cf0fa9051759ed0a7d0a3ba7ea7f64919c", size = 447737, upload-time = "2026-05-27T15:35:38.122Z" }, + { url = "https://files.pythonhosted.org/packages/9e/9c/5430c39fcab1144d35860f457b15e9c08b4bc7ac86764354204e983d6183/tornado-6.5.6-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:38bc01b4acacded2de63ae78023548e41ebe6fbed3ec05a796d7ae3ad893887e", size = 445899, upload-time = "2026-05-27T15:35:40.519Z" }, + { url = "https://files.pythonhosted.org/packages/8b/79/fa7e14a2f939c807a8d30619b4eb604eab219601b78792516ebe22d40cf9/tornado-6.5.6-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b942e6a137fda31ff54bf8e6e2c8d1c37f1f50583f3ed53fb840b53b9601d104", size = 448964, upload-time = "2026-05-27T15:35:42.106Z" }, + { url = "https://files.pythonhosted.org/packages/a7/71/bd67d5f5199f937dafe03a49a37989f60f600ff6fef34c79412a829d97bd/tornado-6.5.6-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8666946e70171b8c3f1fc9b7876fac492e84822c4c7f3746f4e8f8bc9ac92a79", size = 449935, upload-time = "2026-05-27T15:35:43.906Z" }, + { url = "https://files.pythonhosted.org/packages/cc/a4/c24388c9cf5b3c3a513b56a158af9f23092c9a2810d789e294310797df21/tornado-6.5.6-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1c34cfab7ad6d104f052f55de06d39bbafc5885cfeb4da688803308dbcfa90b7", size = 449767, upload-time = "2026-05-27T15:35:45.793Z" }, + { url = "https://files.pythonhosted.org/packages/a5/eb/6a07ad550c3f7b37244bd0becdf293ec3d3e961783d8b720a97df50de1b2/tornado-6.5.6-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:385f35e4e22fb52551dfcda4cdc8c30c61c2c001aef5ddad99cdfe116952efd3", size = 449174, upload-time = "2026-05-27T15:35:47.485Z" }, + { url = "https://files.pythonhosted.org/packages/bb/84/3469e098dccdb6763130e06aacd786bb4363fca7b590a55c101ddf34ed30/tornado-6.5.6-cp39-abi3-win32.whl", hash = "sha256:db475f1b67b2809b10bb16264829087724ca8d24fe4ed47f7b8675cae453ef86", size = 450230, upload-time = "2026-05-27T15:35:49.322Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3c/273a04e0b9dd9016f1685cca0c1c8795a71ac88a34a8c889a0b443483226/tornado-6.5.6-cp39-abi3-win_amd64.whl", hash = "sha256:6739bf1e8eb09230f1280ddbd3236f0309db70f2c551a8dbc40f62babdf82f79", size = 450667, upload-time = "2026-05-27T15:35:51.194Z" }, + { url = "https://files.pythonhosted.org/packages/02/98/0cffe22a224f60c5fb1e3aa0b76f9da2e1ca78b0e9545e3d077c68ce60a7/tornado-6.5.6-cp39-abi3-win_arm64.whl", hash = "sha256:2543597b24a695d72338a9a77818362d72387c03ae173f1f169eadc5c91466ac", size = 449690, upload-time = "2026-05-27T15:35:52.902Z" }, ] [[package]] @@ -3856,11 +3888,11 @@ wheels = [ [[package]] name = "traitlets" -version = "5.14.3" +version = "5.15.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/eb/79/72064e6a701c2183016abbbfedaba506d81e30e232a68c9f0d6f6fcd1574/traitlets-5.14.3.tar.gz", hash = "sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7", size = 161621, upload-time = "2024-04-19T11:11:49.746Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/22/40f55b26baeab80c2d7b3f1db0682f8954e4617fee7d90ce634022ef05c6/traitlets-5.15.0.tar.gz", hash = "sha256:4fead733f81cf1c4c938e06f8ca4633896833c9d89eff878159457f4d4392971", size = 163197, upload-time = "2026-05-06T08:05:58.016Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f", size = 85359, upload-time = "2024-04-19T11:11:46.763Z" }, + { url = "https://files.pythonhosted.org/packages/da/98/a9937a969d018a23badfea0b381f66783649d48e0ea6c41923265c3cbeb3/traitlets-5.15.0-py3-none-any.whl", hash = "sha256:fb36a18867a6803deab09f3c5e0fa81bb7b26a5c9e82501c9933f759166eff40", size = 85877, upload-time = "2026-05-06T08:05:55.853Z" }, ] [[package]] @@ -3883,7 +3915,7 @@ wheels = [ [[package]] name = "typer" -version = "0.26.1" +version = "0.26.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-doc" }, @@ -3891,9 +3923,9 @@ dependencies = [ { name = "rich" }, { name = "shellingham" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/08/26/8e9a4f2c98caefcf4ac25788d48939516a9dd4265fcf9bdd578a2a1b55dd/typer-0.26.1.tar.gz", hash = "sha256:537d27ae686d82967f6383382a952cb32ba4768898541effccb69ca75bbd5d23", size = 198884, upload-time = "2026-05-26T17:49:07.912Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/15/f5fc7be23b7196bc065b282d9589a372392fb10860c80f9c1dd7eb008662/typer-0.26.3.tar.gz", hash = "sha256:3e2b9352f535e5303ef27806dadc2c8647687bdca5c902f03fec3fb88f46a46a", size = 198326, upload-time = "2026-05-28T20:30:50.984Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/be/27/8a22d4833fe8aa0836ce7fa59096ad50d7e93b83be6d5383f11f9a140d54/typer-0.26.1-py3-none-any.whl", hash = "sha256:933e4f0083521f3c57d6a5aedf3b073271b2f95a19761b171b494dd6fdb21ff6", size = 123097, upload-time = "2026-05-26T17:49:09.065Z" }, + { url = "https://files.pythonhosted.org/packages/cd/cc/c6c5dea061e2740355bfeef22ac6a41751bd2f3903e83921295569bdcec4/typer-0.26.3-py3-none-any.whl", hash = "sha256:e70549ec5a403ca8a0bf0802ddd9f3c6ff7a14ccbb859b01b697baa943636f33", size = 122338, upload-time = "2026-05-28T20:30:49.816Z" }, ] [[package]] @@ -4057,11 +4089,47 @@ wheels = [ [[package]] name = "wcwidth" -version = "0.6.0" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/ee/afaf0f85a9a18fe47a67f1e4422ed6cf1fe642f0ae0a2f81166231303c52/wcwidth-0.7.0.tar.gz", hash = "sha256:90e3a7ea092341c44b99562e75d09e4d5160fe7a3974c6fb842a101a95e7eed0", size = 182132, upload-time = "2026-05-02T16:04:12.653Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/52/e465037f5375f43533d1a80b6923955201596a99142ed524d77b571a1418/wcwidth-0.7.0-py3-none-any.whl", hash = "sha256:5d69154c429a82910e241c738cd0e2976fac8a2dd47a1a805f4afed1c0f136f2", size = 110825, upload-time = "2026-05-02T16:04:11.033Z" }, +] + +[[package]] +name = "websockets" +version = "16.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/35/a2/8e3becb46433538a38726c948d3399905a4c7cabd0df578ede5dc51f0ec2/wcwidth-0.6.0.tar.gz", hash = "sha256:cdc4e4262d6ef9a1a57e018384cbeb1208d8abbc64176027e2c2455c81313159", size = 159684, upload-time = "2026-02-06T19:19:40.919Z" } +sdist = { url = "https://files.pythonhosted.org/packages/04/24/4b2031d72e840ce4c1ccb255f693b15c334757fc50023e4db9537080b8c4/websockets-16.0.tar.gz", hash = "sha256:5f6261a5e56e8d5c42a4497b364ea24d94d9563e8fbd44e78ac40879c60179b5", size = 179346, upload-time = "2026-01-10T09:23:47.181Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl", hash = "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", size = 94189, upload-time = "2026-02-06T19:19:39.646Z" }, + { url = "https://files.pythonhosted.org/packages/cc/9c/baa8456050d1c1b08dd0ec7346026668cbc6f145ab4e314d707bb845bf0d/websockets-16.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:878b336ac47938b474c8f982ac2f7266a540adc3fa4ad74ae96fea9823a02cc9", size = 177364, upload-time = "2026-01-10T09:22:59.333Z" }, + { url = "https://files.pythonhosted.org/packages/7e/0c/8811fc53e9bcff68fe7de2bcbe75116a8d959ac699a3200f4847a8925210/websockets-16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:52a0fec0e6c8d9a784c2c78276a48a2bdf099e4ccc2a4cad53b27718dbfd0230", size = 175039, upload-time = "2026-01-10T09:23:01.171Z" }, + { url = "https://files.pythonhosted.org/packages/aa/82/39a5f910cb99ec0b59e482971238c845af9220d3ab9fa76dd9162cda9d62/websockets-16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e6578ed5b6981005df1860a56e3617f14a6c307e6a71b4fff8c48fdc50f3ed2c", size = 175323, upload-time = "2026-01-10T09:23:02.341Z" }, + { url = "https://files.pythonhosted.org/packages/bd/28/0a25ee5342eb5d5f297d992a77e56892ecb65e7854c7898fb7d35e9b33bd/websockets-16.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:95724e638f0f9c350bb1c2b0a7ad0e83d9cc0c9259f3ea94e40d7b02a2179ae5", size = 184975, upload-time = "2026-01-10T09:23:03.756Z" }, + { url = "https://files.pythonhosted.org/packages/f9/66/27ea52741752f5107c2e41fda05e8395a682a1e11c4e592a809a90c6a506/websockets-16.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c0204dc62a89dc9d50d682412c10b3542d748260d743500a85c13cd1ee4bde82", size = 186203, upload-time = "2026-01-10T09:23:05.01Z" }, + { url = "https://files.pythonhosted.org/packages/37/e5/8e32857371406a757816a2b471939d51c463509be73fa538216ea52b792a/websockets-16.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:52ac480f44d32970d66763115edea932f1c5b1312de36df06d6b219f6741eed8", size = 185653, upload-time = "2026-01-10T09:23:06.301Z" }, + { url = "https://files.pythonhosted.org/packages/9b/67/f926bac29882894669368dc73f4da900fcdf47955d0a0185d60103df5737/websockets-16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6e5a82b677f8f6f59e8dfc34ec06ca6b5b48bc4fcda346acd093694cc2c24d8f", size = 184920, upload-time = "2026-01-10T09:23:07.492Z" }, + { url = "https://files.pythonhosted.org/packages/3c/a1/3d6ccdcd125b0a42a311bcd15a7f705d688f73b2a22d8cf1c0875d35d34a/websockets-16.0-cp313-cp313-win32.whl", hash = "sha256:abf050a199613f64c886ea10f38b47770a65154dc37181bfaff70c160f45315a", size = 178255, upload-time = "2026-01-10T09:23:09.245Z" }, + { url = "https://files.pythonhosted.org/packages/6b/ae/90366304d7c2ce80f9b826096a9e9048b4bb760e44d3b873bb272cba696b/websockets-16.0-cp313-cp313-win_amd64.whl", hash = "sha256:3425ac5cf448801335d6fdc7ae1eb22072055417a96cc6b31b3861f455fbc156", size = 178689, upload-time = "2026-01-10T09:23:10.483Z" }, + { url = "https://files.pythonhosted.org/packages/f3/1d/e88022630271f5bd349ed82417136281931e558d628dd52c4d8621b4a0b2/websockets-16.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8cc451a50f2aee53042ac52d2d053d08bf89bcb31ae799cb4487587661c038a0", size = 177406, upload-time = "2026-01-10T09:23:12.178Z" }, + { url = "https://files.pythonhosted.org/packages/f2/78/e63be1bf0724eeb4616efb1ae1c9044f7c3953b7957799abb5915bffd38e/websockets-16.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:daa3b6ff70a9241cf6c7fc9e949d41232d9d7d26fd3522b1ad2b4d62487e9904", size = 175085, upload-time = "2026-01-10T09:23:13.511Z" }, + { url = "https://files.pythonhosted.org/packages/bb/f4/d3c9220d818ee955ae390cf319a7c7a467beceb24f05ee7aaaa2414345ba/websockets-16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fd3cb4adb94a2a6e2b7c0d8d05cb94e6f1c81a0cf9dc2694fb65c7e8d94c42e4", size = 175328, upload-time = "2026-01-10T09:23:14.727Z" }, + { url = "https://files.pythonhosted.org/packages/63/bc/d3e208028de777087e6fb2b122051a6ff7bbcca0d6df9d9c2bf1dd869ae9/websockets-16.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:781caf5e8eee67f663126490c2f96f40906594cb86b408a703630f95550a8c3e", size = 185044, upload-time = "2026-01-10T09:23:15.939Z" }, + { url = "https://files.pythonhosted.org/packages/ad/6e/9a0927ac24bd33a0a9af834d89e0abc7cfd8e13bed17a86407a66773cc0e/websockets-16.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caab51a72c51973ca21fa8a18bd8165e1a0183f1ac7066a182ff27107b71e1a4", size = 186279, upload-time = "2026-01-10T09:23:17.148Z" }, + { url = "https://files.pythonhosted.org/packages/b9/ca/bf1c68440d7a868180e11be653c85959502efd3a709323230314fda6e0b3/websockets-16.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19c4dc84098e523fd63711e563077d39e90ec6702aff4b5d9e344a60cb3c0cb1", size = 185711, upload-time = "2026-01-10T09:23:18.372Z" }, + { url = "https://files.pythonhosted.org/packages/c4/f8/fdc34643a989561f217bb477cbc47a3a07212cbda91c0e4389c43c296ebf/websockets-16.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a5e18a238a2b2249c9a9235466b90e96ae4795672598a58772dd806edc7ac6d3", size = 184982, upload-time = "2026-01-10T09:23:19.652Z" }, + { url = "https://files.pythonhosted.org/packages/dd/d1/574fa27e233764dbac9c52730d63fcf2823b16f0856b3329fc6268d6ae4f/websockets-16.0-cp314-cp314-win32.whl", hash = "sha256:a069d734c4a043182729edd3e9f247c3b2a4035415a9172fd0f1b71658a320a8", size = 177915, upload-time = "2026-01-10T09:23:21.458Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f1/ae6b937bf3126b5134ce1f482365fde31a357c784ac51852978768b5eff4/websockets-16.0-cp314-cp314-win_amd64.whl", hash = "sha256:c0ee0e63f23914732c6d7e0cce24915c48f3f1512ec1d079ed01fc629dab269d", size = 178381, upload-time = "2026-01-10T09:23:22.715Z" }, + { url = "https://files.pythonhosted.org/packages/06/9b/f791d1db48403e1f0a27577a6beb37afae94254a8c6f08be4a23e4930bc0/websockets-16.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:a35539cacc3febb22b8f4d4a99cc79b104226a756aa7400adc722e83b0d03244", size = 177737, upload-time = "2026-01-10T09:23:24.523Z" }, + { url = "https://files.pythonhosted.org/packages/bd/40/53ad02341fa33b3ce489023f635367a4ac98b73570102ad2cdd770dacc9a/websockets-16.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b784ca5de850f4ce93ec85d3269d24d4c82f22b7212023c974c401d4980ebc5e", size = 175268, upload-time = "2026-01-10T09:23:25.781Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/6158d4e459b984f949dcbbb0c5d270154c7618e11c01029b9bbd1bb4c4f9/websockets-16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:569d01a4e7fba956c5ae4fc988f0d4e187900f5497ce46339c996dbf24f17641", size = 175486, upload-time = "2026-01-10T09:23:27.033Z" }, + { url = "https://files.pythonhosted.org/packages/e5/2d/7583b30208b639c8090206f95073646c2c9ffd66f44df967981a64f849ad/websockets-16.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:50f23cdd8343b984957e4077839841146f67a3d31ab0d00e6b824e74c5b2f6e8", size = 185331, upload-time = "2026-01-10T09:23:28.259Z" }, + { url = "https://files.pythonhosted.org/packages/45/b0/cce3784eb519b7b5ad680d14b9673a31ab8dcb7aad8b64d81709d2430aa8/websockets-16.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:152284a83a00c59b759697b7f9e9cddf4e3c7861dd0d964b472b70f78f89e80e", size = 186501, upload-time = "2026-01-10T09:23:29.449Z" }, + { url = "https://files.pythonhosted.org/packages/19/60/b8ebe4c7e89fb5f6cdf080623c9d92789a53636950f7abacfc33fe2b3135/websockets-16.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bc59589ab64b0022385f429b94697348a6a234e8ce22544e3681b2e9331b5944", size = 186062, upload-time = "2026-01-10T09:23:31.368Z" }, + { url = "https://files.pythonhosted.org/packages/88/a8/a080593f89b0138b6cba1b28f8df5673b5506f72879322288b031337c0b8/websockets-16.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32da954ffa2814258030e5a57bc73a3635463238e797c7375dc8091327434206", size = 185356, upload-time = "2026-01-10T09:23:32.627Z" }, + { url = "https://files.pythonhosted.org/packages/c2/b6/b9afed2afadddaf5ebb2afa801abf4b0868f42f8539bfe4b071b5266c9fe/websockets-16.0-cp314-cp314t-win32.whl", hash = "sha256:5a4b4cc550cb665dd8a47f868c8d04c8230f857363ad3c9caf7a0c3bf8c61ca6", size = 178085, upload-time = "2026-01-10T09:23:33.816Z" }, + { url = "https://files.pythonhosted.org/packages/9f/3e/28135a24e384493fa804216b79a6a6759a38cc4ff59118787b9fb693df93/websockets-16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b14dc141ed6d2dde437cddb216004bcac6a1df0935d79656387bd41632ba0bbd", size = 178531, upload-time = "2026-01-10T09:23:35.016Z" }, + { url = "https://files.pythonhosted.org/packages/6f/28/258ebab549c2bf3e64d2b0217b973467394a9cea8c42f70418ca2c5d0d2e/websockets-16.0-py3-none-any.whl", hash = "sha256:1637db62fad1dc833276dded54215f2c7fa46912301a24bd94d45d46a011ceec", size = 171598, upload-time = "2026-01-10T09:23:45.395Z" }, ] [[package]] @@ -4087,55 +4155,55 @@ wheels = [ [[package]] name = "wrapt" -version = "2.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2e/64/925f213fdcbb9baeb1530449ac71a4d57fc361c053d06bf78d0c5c7cd80c/wrapt-2.1.2.tar.gz", hash = "sha256:3996a67eecc2c68fd47b4e3c564405a5777367adfd9b8abb58387b63ee83b21e", size = 81678, upload-time = "2026-03-06T02:53:25.134Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4c/7a/d936840735c828b38d26a854e85d5338894cda544cb7a85a9d5b8b9c4df7/wrapt-2.1.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:787fd6f4d67befa6fe2abdffcbd3de2d82dfc6fb8a6d850407c53332709d030b", size = 61259, upload-time = "2026-03-06T02:53:41.922Z" }, - { url = "https://files.pythonhosted.org/packages/5e/88/9a9b9a90ac8ca11c2fdb6a286cb3a1fc7dd774c00ed70929a6434f6bc634/wrapt-2.1.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4bdf26e03e6d0da3f0e9422fd36bcebf7bc0eeb55fdf9c727a09abc6b9fe472e", size = 61851, upload-time = "2026-03-06T02:52:48.672Z" }, - { url = "https://files.pythonhosted.org/packages/03/a9/5b7d6a16fd6533fed2756900fc8fc923f678179aea62ada6d65c92718c00/wrapt-2.1.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bbac24d879aa22998e87f6b3f481a5216311e7d53c7db87f189a7a0266dafffb", size = 121446, upload-time = "2026-03-06T02:54:14.013Z" }, - { url = "https://files.pythonhosted.org/packages/45/bb/34c443690c847835cfe9f892be78c533d4f32366ad2888972c094a897e39/wrapt-2.1.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:16997dfb9d67addc2e3f41b62a104341e80cac52f91110dece393923c0ebd5ca", size = 123056, upload-time = "2026-03-06T02:54:10.829Z" }, - { url = "https://files.pythonhosted.org/packages/93/b9/ff205f391cb708f67f41ea148545f2b53ff543a7ac293b30d178af4d2271/wrapt-2.1.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:162e4e2ba7542da9027821cb6e7c5e068d64f9a10b5f15512ea28e954893a267", size = 117359, upload-time = "2026-03-06T02:53:03.623Z" }, - { url = "https://files.pythonhosted.org/packages/1f/3d/1ea04d7747825119c3c9a5e0874a40b33594ada92e5649347c457d982805/wrapt-2.1.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f29c827a8d9936ac320746747a016c4bc66ef639f5cd0d32df24f5eacbf9c69f", size = 121479, upload-time = "2026-03-06T02:53:45.844Z" }, - { url = "https://files.pythonhosted.org/packages/78/cc/ee3a011920c7a023b25e8df26f306b2484a531ab84ca5c96260a73de76c0/wrapt-2.1.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:a9dd9813825f7ecb018c17fd147a01845eb330254dff86d3b5816f20f4d6aaf8", size = 116271, upload-time = "2026-03-06T02:54:46.356Z" }, - { url = "https://files.pythonhosted.org/packages/98/fd/e5ff7ded41b76d802cf1191288473e850d24ba2e39a6ec540f21ae3b57cb/wrapt-2.1.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6f8dbdd3719e534860d6a78526aafc220e0241f981367018c2875178cf83a413", size = 120573, upload-time = "2026-03-06T02:52:50.163Z" }, - { url = "https://files.pythonhosted.org/packages/47/c5/242cae3b5b080cd09bacef0591691ba1879739050cc7c801ff35c8886b66/wrapt-2.1.2-cp313-cp313-win32.whl", hash = "sha256:5c35b5d82b16a3bc6e0a04349b606a0582bc29f573786aebe98e0c159bc48db6", size = 58205, upload-time = "2026-03-06T02:53:47.494Z" }, - { url = "https://files.pythonhosted.org/packages/12/69/c358c61e7a50f290958809b3c61ebe8b3838ea3e070d7aac9814f95a0528/wrapt-2.1.2-cp313-cp313-win_amd64.whl", hash = "sha256:f8bc1c264d8d1cf5b3560a87bbdd31131573eb25f9f9447bb6252b8d4c44a3a1", size = 60452, upload-time = "2026-03-06T02:53:30.038Z" }, - { url = "https://files.pythonhosted.org/packages/8e/66/c8a6fcfe321295fd8c0ab1bd685b5a01462a9b3aa2f597254462fc2bc975/wrapt-2.1.2-cp313-cp313-win_arm64.whl", hash = "sha256:3beb22f674550d5634642c645aba4c72a2c66fb185ae1aebe1e955fae5a13baf", size = 58842, upload-time = "2026-03-06T02:52:52.114Z" }, - { url = "https://files.pythonhosted.org/packages/da/55/9c7052c349106e0b3f17ae8db4b23a691a963c334de7f9dbd60f8f74a831/wrapt-2.1.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0fc04bc8664a8bc4c8e00b37b5355cffca2535209fba1abb09ae2b7c76ddf82b", size = 63075, upload-time = "2026-03-06T02:53:19.108Z" }, - { url = "https://files.pythonhosted.org/packages/09/a8/ce7b4006f7218248dd71b7b2b732d0710845a0e49213b18faef64811ffef/wrapt-2.1.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a9b9d50c9af998875a1482a038eb05755dfd6fe303a313f6a940bb53a83c3f18", size = 63719, upload-time = "2026-03-06T02:54:33.452Z" }, - { url = "https://files.pythonhosted.org/packages/e4/e5/2ca472e80b9e2b7a17f106bb8f9df1db11e62101652ce210f66935c6af67/wrapt-2.1.2-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2d3ff4f0024dd224290c0eabf0240f1bfc1f26363431505fb1b0283d3b08f11d", size = 152643, upload-time = "2026-03-06T02:52:42.721Z" }, - { url = "https://files.pythonhosted.org/packages/36/42/30f0f2cefca9d9cbf6835f544d825064570203c3e70aa873d8ae12e23791/wrapt-2.1.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3278c471f4468ad544a691b31bb856374fbdefb7fee1a152153e64019379f015", size = 158805, upload-time = "2026-03-06T02:54:25.441Z" }, - { url = "https://files.pythonhosted.org/packages/bb/67/d08672f801f604889dcf58f1a0b424fe3808860ede9e03affc1876b295af/wrapt-2.1.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a8914c754d3134a3032601c6984db1c576e6abaf3fc68094bb8ab1379d75ff92", size = 145990, upload-time = "2026-03-06T02:53:57.456Z" }, - { url = "https://files.pythonhosted.org/packages/68/a7/fd371b02e73babec1de6ade596e8cd9691051058cfdadbfd62a5898f3295/wrapt-2.1.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ff95d4264e55839be37bafe1536db2ab2de19da6b65f9244f01f332b5286cfbf", size = 155670, upload-time = "2026-03-06T02:54:55.309Z" }, - { url = "https://files.pythonhosted.org/packages/86/2d/9fe0095dfdb621009f40117dcebf41d7396c2c22dca6eac779f4c007b86c/wrapt-2.1.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:76405518ca4e1b76fbb1b9f686cff93aebae03920cc55ceeec48ff9f719c5f67", size = 144357, upload-time = "2026-03-06T02:54:24.092Z" }, - { url = "https://files.pythonhosted.org/packages/0e/b6/ec7b4a254abbe4cde9fa15c5d2cca4518f6b07d0f1b77d4ee9655e30280e/wrapt-2.1.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c0be8b5a74c5824e9359b53e7e58bef71a729bacc82e16587db1c4ebc91f7c5a", size = 150269, upload-time = "2026-03-06T02:53:31.268Z" }, - { url = "https://files.pythonhosted.org/packages/6e/6b/2fabe8ebf148f4ee3c782aae86a795cc68ffe7d432ef550f234025ce0cfa/wrapt-2.1.2-cp313-cp313t-win32.whl", hash = "sha256:f01277d9a5fc1862f26f7626da9cf443bebc0abd2f303f41c5e995b15887dabd", size = 59894, upload-time = "2026-03-06T02:54:15.391Z" }, - { url = "https://files.pythonhosted.org/packages/ca/fb/9ba66fc2dedc936de5f8073c0217b5d4484e966d87723415cc8262c5d9c2/wrapt-2.1.2-cp313-cp313t-win_amd64.whl", hash = "sha256:84ce8f1c2104d2f6daa912b1b5b039f331febfeee74f8042ad4e04992bd95c8f", size = 63197, upload-time = "2026-03-06T02:54:41.943Z" }, - { url = "https://files.pythonhosted.org/packages/c0/1c/012d7423c95d0e337117723eb8ecf73c622ce15a97847e84cf3f8f26cd7e/wrapt-2.1.2-cp313-cp313t-win_arm64.whl", hash = "sha256:a93cd767e37faeddbe07d8fc4212d5cba660af59bdb0f6372c93faaa13e6e679", size = 60363, upload-time = "2026-03-06T02:54:48.093Z" }, - { url = "https://files.pythonhosted.org/packages/39/25/e7ea0b417db02bb796182a5316398a75792cd9a22528783d868755e1f669/wrapt-2.1.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:1370e516598854e5b4366e09ce81e08bfe94d42b0fd569b88ec46cc56d9164a9", size = 61418, upload-time = "2026-03-06T02:53:55.706Z" }, - { url = "https://files.pythonhosted.org/packages/ec/0f/fa539e2f6a770249907757eaeb9a5ff4deb41c026f8466c1c6d799088a9b/wrapt-2.1.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:6de1a3851c27e0bd6a04ca993ea6f80fc53e6c742ee1601f486c08e9f9b900a9", size = 61914, upload-time = "2026-03-06T02:52:53.37Z" }, - { url = "https://files.pythonhosted.org/packages/53/37/02af1867f5b1441aaeda9c82deed061b7cd1372572ddcd717f6df90b5e93/wrapt-2.1.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:de9f1a2bbc5ac7f6012ec24525bdd444765a2ff64b5985ac6e0692144838542e", size = 120417, upload-time = "2026-03-06T02:54:30.74Z" }, - { url = "https://files.pythonhosted.org/packages/c3/b7/0138a6238c8ba7476c77cf786a807f871672b37f37a422970342308276e7/wrapt-2.1.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:970d57ed83fa040d8b20c52fe74a6ae7e3775ae8cff5efd6a81e06b19078484c", size = 122797, upload-time = "2026-03-06T02:54:51.539Z" }, - { url = "https://files.pythonhosted.org/packages/e1/ad/819ae558036d6a15b7ed290d5b14e209ca795dd4da9c58e50c067d5927b0/wrapt-2.1.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3969c56e4563c375861c8df14fa55146e81ac11c8db49ea6fb7f2ba58bc1ff9a", size = 117350, upload-time = "2026-03-06T02:54:37.651Z" }, - { url = "https://files.pythonhosted.org/packages/8b/2d/afc18dc57a4600a6e594f77a9ae09db54f55ba455440a54886694a84c71b/wrapt-2.1.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:57d7c0c980abdc5f1d98b11a2aa3bb159790add80258c717fa49a99921456d90", size = 121223, upload-time = "2026-03-06T02:54:35.221Z" }, - { url = "https://files.pythonhosted.org/packages/b9/5b/5ec189b22205697bc56eb3b62aed87a1e0423e9c8285d0781c7a83170d15/wrapt-2.1.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:776867878e83130c7a04237010463372e877c1c994d449ca6aaafeab6aab2586", size = 116287, upload-time = "2026-03-06T02:54:19.654Z" }, - { url = "https://files.pythonhosted.org/packages/f7/2d/f84939a7c9b5e6cdd8a8d0f6a26cabf36a0f7e468b967720e8b0cd2bdf69/wrapt-2.1.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:fab036efe5464ec3291411fabb80a7a39e2dd80bae9bcbeeca5087fdfa891e19", size = 119593, upload-time = "2026-03-06T02:54:16.697Z" }, - { url = "https://files.pythonhosted.org/packages/0b/fe/ccd22a1263159c4ac811ab9374c061bcb4a702773f6e06e38de5f81a1bdc/wrapt-2.1.2-cp314-cp314-win32.whl", hash = "sha256:e6ed62c82ddf58d001096ae84ce7f833db97ae2263bff31c9b336ba8cfe3f508", size = 58631, upload-time = "2026-03-06T02:53:06.498Z" }, - { url = "https://files.pythonhosted.org/packages/65/0a/6bd83be7bff2e7efaac7b4ac9748da9d75a34634bbbbc8ad077d527146df/wrapt-2.1.2-cp314-cp314-win_amd64.whl", hash = "sha256:467e7c76315390331c67073073d00662015bb730c566820c9ca9b54e4d67fd04", size = 60875, upload-time = "2026-03-06T02:53:50.252Z" }, - { url = "https://files.pythonhosted.org/packages/6c/c0/0b3056397fe02ff80e5a5d72d627c11eb885d1ca78e71b1a5c1e8c7d45de/wrapt-2.1.2-cp314-cp314-win_arm64.whl", hash = "sha256:da1f00a557c66225d53b095a97eace0fc5349e3bfda28fa34ffae238978ee575", size = 59164, upload-time = "2026-03-06T02:53:59.128Z" }, - { url = "https://files.pythonhosted.org/packages/71/ed/5d89c798741993b2371396eb9d4634f009ff1ad8a6c78d366fe2883ea7a6/wrapt-2.1.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:62503ffbc2d3a69891cf29beeaccdb4d5e0a126e2b6a851688d4777e01428dbb", size = 63163, upload-time = "2026-03-06T02:52:54.873Z" }, - { url = "https://files.pythonhosted.org/packages/c6/8c/05d277d182bf36b0a13d6bd393ed1dec3468a25b59d01fba2dd70fe4d6ae/wrapt-2.1.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c7e6cd120ef837d5b6f860a6ea3745f8763805c418bb2f12eeb1fa6e25f22d22", size = 63723, upload-time = "2026-03-06T02:52:56.374Z" }, - { url = "https://files.pythonhosted.org/packages/f4/27/6c51ec1eff4413c57e72d6106bb8dec6f0c7cdba6503d78f0fa98767bcc9/wrapt-2.1.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3769a77df8e756d65fbc050333f423c01ae012b4f6731aaf70cf2bef61b34596", size = 152652, upload-time = "2026-03-06T02:53:23.79Z" }, - { url = "https://files.pythonhosted.org/packages/db/4c/d7dd662d6963fc7335bfe29d512b02b71cdfa23eeca7ab3ac74a67505deb/wrapt-2.1.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a76d61a2e851996150ba0f80582dd92a870643fa481f3b3846f229de88caf044", size = 158807, upload-time = "2026-03-06T02:53:35.742Z" }, - { url = "https://files.pythonhosted.org/packages/b4/4d/1e5eea1a78d539d346765727422976676615814029522c76b87a95f6bcdd/wrapt-2.1.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6f97edc9842cf215312b75fe737ee7c8adda75a89979f8e11558dfff6343cc4b", size = 146061, upload-time = "2026-03-06T02:52:57.574Z" }, - { url = "https://files.pythonhosted.org/packages/89/bc/62cabea7695cd12a288023251eeefdcb8465056ddaab6227cb78a2de005b/wrapt-2.1.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4006c351de6d5007aa33a551f600404ba44228a89e833d2fadc5caa5de8edfbf", size = 155667, upload-time = "2026-03-06T02:53:39.422Z" }, - { url = "https://files.pythonhosted.org/packages/e9/99/6f2888cd68588f24df3a76572c69c2de28287acb9e1972bf0c83ce97dbc1/wrapt-2.1.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:a9372fc3639a878c8e7d87e1556fa209091b0a66e912c611e3f833e2c4202be2", size = 144392, upload-time = "2026-03-06T02:54:22.41Z" }, - { url = "https://files.pythonhosted.org/packages/40/51/1dfc783a6c57971614c48e361a82ca3b6da9055879952587bc99fe1a7171/wrapt-2.1.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3144b027ff30cbd2fca07c0a87e67011adb717eb5f5bd8496325c17e454257a3", size = 150296, upload-time = "2026-03-06T02:54:07.848Z" }, - { url = "https://files.pythonhosted.org/packages/6c/38/cbb8b933a0201076c1f64fc42883b0023002bdc14a4964219154e6ff3350/wrapt-2.1.2-cp314-cp314t-win32.whl", hash = "sha256:3b8d15e52e195813efe5db8cec156eebe339aaf84222f4f4f051a6c01f237ed7", size = 60539, upload-time = "2026-03-06T02:54:00.594Z" }, - { url = "https://files.pythonhosted.org/packages/82/dd/e5176e4b241c9f528402cebb238a36785a628179d7d8b71091154b3e4c9e/wrapt-2.1.2-cp314-cp314t-win_amd64.whl", hash = "sha256:08ffa54146a7559f5b8df4b289b46d963a8e74ed16ba3687f99896101a3990c5", size = 63969, upload-time = "2026-03-06T02:54:39Z" }, - { url = "https://files.pythonhosted.org/packages/5c/99/79f17046cf67e4a95b9987ea129632ba8bcec0bc81f3fb3d19bdb0bd60cd/wrapt-2.1.2-cp314-cp314t-win_arm64.whl", hash = "sha256:72aaa9d0d8e4ed0e2e98019cea47a21f823c9dd4b43c7b77bba6679ffcca6a00", size = 60554, upload-time = "2026-03-06T02:53:14.132Z" }, - { url = "https://files.pythonhosted.org/packages/1a/c7/8528ac2dfa2c1e6708f647df7ae144ead13f0a31146f43c7264b4942bf12/wrapt-2.1.2-py3-none-any.whl", hash = "sha256:b8fd6fa2b2c4e7621808f8c62e8317f4aae56e59721ad933bac5239d913cf0e8", size = 43993, upload-time = "2026-03-06T02:53:12.905Z" }, +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/9f/06263fcd8ad6c405f05a3905fd7a84dd3176eb5ad46e44bccc0cd16348bb/wrapt-2.2.1.tar.gz", hash = "sha256:6744f504375775d7609c82c8d3d94af1c9a6f05586984536905908ba905277b9", size = 127620, upload-time = "2026-05-22T14:49:43.056Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/d1/a1b08f8f4fac8cbb156fa51cf64ee2c7f7f74f9875ba3cf70b3c58368694/wrapt-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d2beb1c7cab10603aecdc42f8edd6ff013f9a32e4543474e38e6b77ce9975aeb", size = 80831, upload-time = "2026-05-22T14:48:15.598Z" }, + { url = "https://files.pythonhosted.org/packages/54/ce/57890814991446a845e09b3445ce8b694f27eb0577004f2c2a36a9772ed4/wrapt-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e0cb7e4dd71f4c32e5e84843cd3c4cd65dda034314004bbe1d7f99af2426ab80", size = 81375, upload-time = "2026-05-22T14:48:17.071Z" }, + { url = "https://files.pythonhosted.org/packages/38/65/08d7a6c76ac4493bdb668205ee9c1de1bd5daca61717c3e9aa49b4c01499/wrapt-2.2.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:95821352042722cd9f1108874579a47989d0a7e12a37d87d2fc4af20fd99ab8a", size = 167417, upload-time = "2026-05-22T14:48:18.303Z" }, + { url = "https://files.pythonhosted.org/packages/62/ce/f1ccbee7a1bfe5cdc6b3da6bab4b45713d628b9294da32a39f563d648140/wrapt-2.2.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:abd621552ede77c4c69be7fac44ba911225b0c812b6ba604e5964cf98085b474", size = 166948, upload-time = "2026-05-22T14:48:19.768Z" }, + { url = "https://files.pythonhosted.org/packages/86/2a/f85d48d1cd4869aee6704028d257d740a47c1c467b457ce396b4b5b55d07/wrapt-2.2.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e3677c7146ce694874941ba82b57092cc4875445aadf29d72807351023105143", size = 158148, upload-time = "2026-05-22T14:48:21.96Z" }, + { url = "https://files.pythonhosted.org/packages/fe/5c/93939ad11d4a12358ab1aab219a2ef5efa5612e0db6b9fc65af8af1a891b/wrapt-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9a5934eaea872e17936b5f45501eba5ab0bce9a74122e172b663d7c28c459c4a", size = 165905, upload-time = "2026-05-22T14:48:23.373Z" }, + { url = "https://files.pythonhosted.org/packages/e0/22/b8c2aa89862ff58605934d7abf4b70e6a5a1c33df96656f49035ccdf1c8a/wrapt-2.2.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f5b9daf6b629fce418e0cc3dd0436eac045188fa35deadb7a7f3941d5b8203f9", size = 156712, upload-time = "2026-05-22T14:48:24.767Z" }, + { url = "https://files.pythonhosted.org/packages/5d/78/bf00a7b02239c12bb02ddcc3c0b971bfcc36e578c5a44f1ccfef5b458545/wrapt-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f53ac9f3ef573326d009ed809beff4efcac6451931c2b8132586da4b9e53ff31", size = 166560, upload-time = "2026-05-22T14:48:26.83Z" }, + { url = "https://files.pythonhosted.org/packages/fe/93/6390ca9c5b787683cef588d04f57c8d41b9a2323b5597a65f18638c90ef2/wrapt-2.2.1-cp313-cp313-win32.whl", hash = "sha256:1ffa9cfd4bdb581539951b14ae661ff20ed0c3599b3e911a131ee0ec5ac11337", size = 77817, upload-time = "2026-05-22T14:48:28.221Z" }, + { url = "https://files.pythonhosted.org/packages/97/73/ce10f0e71c0cfaa1a65faadb8efd4852028b3bb9ba28932b8889df769d38/wrapt-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:368eac1e20fd0bb03dd3cc42bf9887154c3861b60989389ccb5fac032617d215", size = 80736, upload-time = "2026-05-22T14:48:30.139Z" }, + { url = "https://files.pythonhosted.org/packages/c7/4c/89f4a6818fafbbd840330e4fa3873073e1bfc166133a64cac7f8fde7a5e3/wrapt-2.2.1-cp313-cp313-win_arm64.whl", hash = "sha256:c754dafdf5aaf0b401b644a90a30046929a0dd1a536e0ff0ec959a59155d9c7f", size = 79099, upload-time = "2026-05-22T14:48:31.405Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f2/9a8741c46f8c208ac0a45b25ba170bcb4fb72a2781d5fb97dbd7b6be73cb/wrapt-2.2.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ed928d0fda15fc0adc8d13305c8b3c0f2fba5b0669950c9e6d019d9162a3b3e8", size = 82802, upload-time = "2026-05-22T14:48:33.307Z" }, + { url = "https://files.pythonhosted.org/packages/9c/0d/e9c855716a3705eef1416456bdf062b60620726fdc59428ff670fc3c60dc/wrapt-2.2.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fafb4e739e43544d12cb4abd1605fd4683b6ca6a9ad682b7fd8f4d21973eafa8", size = 83329, upload-time = "2026-05-22T14:48:34.593Z" }, + { url = "https://files.pythonhosted.org/packages/3b/d6/a88f1c13112b7831adac75cea65d8310e0d696d570c8961844c90a57b865/wrapt-2.2.1-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:74d6a0c31472fe5d814917266b9f46495d7c61ed890af08b468acea92fb89a8d", size = 202937, upload-time = "2026-05-22T14:48:35.859Z" }, + { url = "https://files.pythonhosted.org/packages/42/65/e29d54aef06a4d898a5b8a25589a0b3769bde454f922fad8f6f89fbfb650/wrapt-2.2.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab5be648d5a0b86b7438864f8df3c705a65cef35a2fd3e5561e3e203167e0f27", size = 209997, upload-time = "2026-05-22T14:48:38.153Z" }, + { url = "https://files.pythonhosted.org/packages/2a/91/e4454263516cf0e12640912fbca9a83654e424f0a6ddb79f5cd7ce14bf33/wrapt-2.2.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9d8f204c8e3a8bf9ece17e0a83d137fd807440977f8a5e762d59306795011440", size = 194856, upload-time = "2026-05-22T14:48:39.69Z" }, + { url = "https://files.pythonhosted.org/packages/de/d0/fe0ee202286afdf4a7f77dd29f195703145764d572aec209c5086e57d924/wrapt-2.2.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d047f6498c973874ba08ac3f97c69a2c4b2211c8de6f4c205f75cb1c9522596e", size = 205654, upload-time = "2026-05-22T14:48:43.456Z" }, + { url = "https://files.pythonhosted.org/packages/23/b6/87d860dfc6460c246af70b1fd5c8b76df77571b42a493459423ded94fd7d/wrapt-2.2.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:7a4fdb9326aab4a5a477a1640e5ad786a8495901009d7e7b038371edd23a9d2b", size = 192206, upload-time = "2026-05-22T14:48:44.858Z" }, + { url = "https://files.pythonhosted.org/packages/df/46/3eea8cde077d985f239a38c0257087b8064fd9ee9b1a99e282d2c86da4ef/wrapt-2.2.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c8cc5094b08abeae52da9c73c8a32003623be691a5193df2f4e3eac3d557c394", size = 198428, upload-time = "2026-05-22T14:48:46.319Z" }, + { url = "https://files.pythonhosted.org/packages/18/dc/b927ee9c7fc67adc3a5658f246a0d275425eb840ba36e7b702e70f18bde8/wrapt-2.2.1-cp313-cp313t-win32.whl", hash = "sha256:9907a4402ab6db12b7077a0ea5d7a4d028ecb22c8eee2b53527080d347cd1562", size = 79448, upload-time = "2026-05-22T14:48:47.901Z" }, + { url = "https://files.pythonhosted.org/packages/ec/b3/fd30b473fe498c70e6b9a5f328b8d3fbaf1b8c3c481465f59724bba8eb70/wrapt-2.2.1-cp313-cp313t-win_amd64.whl", hash = "sha256:5590d63f5243251641cf543009b4c9314a79d0598fdb8a8e4cfc918494536c53", size = 83021, upload-time = "2026-05-22T14:48:49.201Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f3/96c39153a8737a6e9aa85adef254ac4195bea3f2d24efc60472ccc3c9e2e/wrapt-2.2.1-cp313-cp313t-win_arm64.whl", hash = "sha256:c318a64b53d97b841d7b5e637517e50a27be64bc695128422953d4b21710954e", size = 80295, upload-time = "2026-05-22T14:48:50.479Z" }, + { url = "https://files.pythonhosted.org/packages/0a/a3/11d7f34ebbf3231bc907a3e6d5ee051b14d034c1bc7b65a97d5cc00516df/wrapt-2.2.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:6f56a647e4eaf5f0ca40330fb070f566bdf9f7b0db89a1af20d71c28dcd7a0ab", size = 80879, upload-time = "2026-05-22T14:48:51.802Z" }, + { url = "https://files.pythonhosted.org/packages/13/3c/b74cfd984cef560b900fb1a727af20352d89e1f06bf2e1114dd3f00f5f5a/wrapt-2.2.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:64b7deeda4b70408e382328d8bbe52a256fe9bc63ae3db86d804608367e5422c", size = 81462, upload-time = "2026-05-22T14:48:53.18Z" }, + { url = "https://files.pythonhosted.org/packages/15/a3/7c8f704b8dc07dfe0a5d01c2edbfd88317aa8e5e3fa7c743eb7a085ae767/wrapt-2.2.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b9cf53ba90717db2e292401de290776c498d4bbfb0d4a559ca2895db8b9dcb5c", size = 167251, upload-time = "2026-05-22T14:48:54.562Z" }, + { url = "https://files.pythonhosted.org/packages/80/85/a34d1888d97247da6c2ff6118c3a721c73ed8cc4dd198c00208bb73b6f80/wrapt-2.2.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cf3638274ab9d9b724c9baa0b4c04e132cd6faefb78b4dd3dd1a02a4bdaad41e", size = 166316, upload-time = "2026-05-22T14:48:56.065Z" }, + { url = "https://files.pythonhosted.org/packages/e9/d7/72ffaeb01eebc704afe3fb99e840480f4bda45f0fa66e3381b6a39251c8f/wrapt-2.2.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aed9658797d0b45d6c49adcfc6b41f66e6f2d0c6de3ec79e16cf4b1855df240f", size = 157952, upload-time = "2026-05-22T14:48:57.924Z" }, + { url = "https://files.pythonhosted.org/packages/24/5b/36f5d6b024e4edfdd90b140742d11ebcf7836daf5c9daf326c55c24db412/wrapt-2.2.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1d676ee388bc42a04d56dd7deb5605244dac2e35cc2fadbb43c9fa25bbd93508", size = 166130, upload-time = "2026-05-22T14:48:59.384Z" }, + { url = "https://files.pythonhosted.org/packages/81/06/9296d9e97bfdef5483dfcc859d57b095b257144b2bc5300ab521e06f4bc7/wrapt-2.2.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e395f7bc31851ef9b612050368cb446e9bc14cd7454b025018980349caf25ae5", size = 156604, upload-time = "2026-05-22T14:49:00.921Z" }, + { url = "https://files.pythonhosted.org/packages/53/37/16953929ed6776175720e58fc966e779926d8d71e2c7b2273230590ca71f/wrapt-2.2.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f1845c2a8cc1180ccccfa45785dd06f562730d19ef75be180334254012b6283", size = 166007, upload-time = "2026-05-22T14:49:02.332Z" }, + { url = "https://files.pythonhosted.org/packages/b9/73/20ee58c0612dae7c31131a7095345812ed2c7b389019e175f68cde34e5b4/wrapt-2.2.1-cp314-cp314-win32.whl", hash = "sha256:436addbc4bb4fc0a88c702577f51195d7d73683a7f3e0e5b253d8404d7847243", size = 78327, upload-time = "2026-05-22T14:49:03.722Z" }, + { url = "https://files.pythonhosted.org/packages/22/b3/ef7c3295d02e0448a71c639a36a057f46d524d057c9486291a7a3039e65c/wrapt-2.2.1-cp314-cp314-win_amd64.whl", hash = "sha256:50972a1d974ea07725a7f6b1cec5f8759008afd030a0024843ebe7d52de47f2b", size = 81144, upload-time = "2026-05-22T14:49:05.093Z" }, + { url = "https://files.pythonhosted.org/packages/ac/dc/7bdf336953f99f4ceb0a584bb8870e42c8f26f93ea10c87834dad62f1668/wrapt-2.2.1-cp314-cp314-win_arm64.whl", hash = "sha256:1c9934ea5d92957e3cd0adbc0845539dccfd62710ebe16195a8c66c53954db36", size = 79569, upload-time = "2026-05-22T14:49:06.413Z" }, + { url = "https://files.pythonhosted.org/packages/6a/6d/6dfae80150ff1919c356d1dd528f049bcdfaae29b4d284bc957e022caef4/wrapt-2.2.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:17de18fc12cea55b8a9587314cb830573e37fb33b247a7515696350863714188", size = 82892, upload-time = "2026-05-22T14:49:07.925Z" }, + { url = "https://files.pythonhosted.org/packages/82/7b/4e34766a7d7804ffce9e71befe47e9b3225dc350c49c94493c4ab39fd3a5/wrapt-2.2.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a9dec1aca52dddde7df94818310fa2fe79739c8f385b2014c4cb1035f5508199", size = 83333, upload-time = "2026-05-22T14:49:09.257Z" }, + { url = "https://files.pythonhosted.org/packages/9d/57/0b34db3e8de44ccfece62d7b337abd1631dd810f5adc5f3db571727836b5/wrapt-2.2.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:69f2e9244542cb34dd59c7f073445b9e54ad9f3fce8d93606c368a1b499fc413", size = 202899, upload-time = "2026-05-22T14:49:10.572Z" }, + { url = "https://files.pythonhosted.org/packages/e5/45/ac0c459f154b99d92789a6cba7ca727185b83513b986f8ec7fe2aacddcbf/wrapt-2.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2d83966dc7f4f45e8b97b5933685ac2e6e67fc0e19246ea314bceb9a8970c956", size = 209986, upload-time = "2026-05-22T14:49:12.229Z" }, + { url = "https://files.pythonhosted.org/packages/b7/e4/77e37ff33ad018fa81ade52c25fa327b80b56f81d734279a63614fcb4cbc/wrapt-2.2.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:78b0aa6bfb7be8deed0ab23e7aa028cc5210c29bc2d32a04d52b50e517a7307e", size = 194893, upload-time = "2026-05-22T14:49:14.139Z" }, + { url = "https://files.pythonhosted.org/packages/dd/9d/7ea651d1ab032fc5fa222fbec91d0f8a1397f6ae04ebb93fa7219aa921d7/wrapt-2.2.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:05d5cb74d1b232ec8cfa130a8f900708699ff2491d97b8f85a4cdc5996294b85", size = 205636, upload-time = "2026-05-22T14:49:15.714Z" }, + { url = "https://files.pythonhosted.org/packages/09/af/8e88031a701275b9085c54e64bc88c0b1cd55c77eadd400691c371cd76c4/wrapt-2.2.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f6518b94edb9150452e9aba08027d4cc293433753ec1fbefb4629a21cbc74181", size = 192267, upload-time = "2026-05-22T14:49:17.283Z" }, + { url = "https://files.pythonhosted.org/packages/bf/a8/e657ca876b06710194f243d81c4b0896ade646e244bdbec2d87c8c56a8bd/wrapt-2.2.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ed55af48b3eb28f43228ca2306788892bcb629eb2b5c4876e2a3659872c2f17a", size = 198378, upload-time = "2026-05-22T14:49:18.785Z" }, + { url = "https://files.pythonhosted.org/packages/c8/59/822efe4ea722a3961331bfa35b7d90937790d2c20f0616de1997ccc3aebd/wrapt-2.2.1-cp314-cp314t-win32.whl", hash = "sha256:2e08688ab16525897da6589d56d0aebaf417bbe91c2d8e3b96203b1efa596e85", size = 80226, upload-time = "2026-05-22T14:49:20.264Z" }, + { url = "https://files.pythonhosted.org/packages/ab/31/2a7dc5f6abb2fca0b6e1610e120419f603650aceb4f1d3ac4cae0354e162/wrapt-2.2.1-cp314-cp314t-win_amd64.whl", hash = "sha256:fd0135d34387f5fd087d9be368ea77ea89cf2451dc1cd1c622d35021bcb3ab50", size = 83835, upload-time = "2026-05-22T14:49:21.634Z" }, + { url = "https://files.pythonhosted.org/packages/9f/c0/782b86e28d1ceebeb74cccea12d2cd3d2ba0bd68e3dec20b1bc5873f6127/wrapt-2.2.1-cp314-cp314t-win_arm64.whl", hash = "sha256:f70db64e8266d7c45d3b735f2e08eeb434b5e03da9a479ae42b2e2e486a21a00", size = 80722, upload-time = "2026-05-22T14:49:23.59Z" }, + { url = "https://files.pythonhosted.org/packages/53/46/29ac9daf11a86c22a8c38cd9236c62928ccae83f7ceb06bd3b0467cf9d05/wrapt-2.2.1-py3-none-any.whl", hash = "sha256:3aafea2975caef8ca49400640dde02cc7426e798f24870ed01f490bc3cffd32f", size = 61000, upload-time = "2026-05-22T14:49:41.593Z" }, ] [[package]] @@ -4343,11 +4411,11 @@ wheels = [ [[package]] name = "zipp" -version = "3.23.1" +version = "4.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/30/21/093488dfc7cc8964ded15ab726fad40f25fd3d788fd741cc1c5a17d78ee8/zipp-3.23.1.tar.gz", hash = "sha256:32120e378d32cd9714ad503c1d024619063ec28aad2248dc6672ad13edfa5110", size = 25965, upload-time = "2026-04-13T23:21:46.6Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/d8/eab98a517c14134c0b2eb4e2387bc5f457334293ec5d2dd3857ec2966802/zipp-4.1.0.tar.gz", hash = "sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602", size = 26214, upload-time = "2026-05-18T20:08:57.967Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/08/8a/0861bec20485572fbddf3dfba2910e38fe249796cb73ecdeb74e07eeb8d3/zipp-3.23.1-py3-none-any.whl", hash = "sha256:0b3596c50a5c700c9cb40ba8d86d9f2cc4807e9bedb06bcdf7fac85633e444dc", size = 10378, upload-time = "2026-04-13T23:21:45.386Z" }, + { url = "https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl", hash = "sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f", size = 10238, upload-time = "2026-05-18T20:08:57.045Z" }, ] [[package]] From 3ab5c12cdc587467a3a4198c0925c0e13ebd1b8a Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Fri, 29 May 2026 07:24:07 -0700 Subject: [PATCH 091/128] minor comments and tidying --- tests/utils/test_s3.py | 47 +++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index 0955eb59..d6a26b53 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -34,9 +34,8 @@ upload_file, ) -AWS_REGION = "us-east-1" -AWS_ENV_VARS = ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_ENDPOINT_URL_S3", "AWS_ENDPOINT_URL"] - +HTTP_200 = 200 +HTTP_204 = 204 SAMPLE_FILES = [ "dir_one/file1.txt", @@ -63,13 +62,15 @@ def mock_s3_client() -> Generator[Any, Any]: Resets the cached client before and after to prevent state leaking between tests. """ with mock_aws(): - client = boto3.client("s3", region_name=AWS_REGION) + client = boto3.client("s3") for bucket in FILES_IN_BUCKETS: client.create_bucket(Bucket=bucket) + # delete any existing client reset_s3_client() assert s3_utils._s3_client is None # noqa: SLF001 + # patch in the client that we have just created with patch.object(s3_utils, "get_s3_client", return_value=client): yield client @@ -130,18 +131,25 @@ def populate_mock_s3(client: Any, file_list_by_bucket: dict[str, list[str]]) -> client.head_object(Bucket=bucket, Key=file) -# Client creation / reset -@mock_aws -@pytest.mark.s3 -@pytest.mark.parametrize("endpoint_url", ["http://localhost", None]) -def test_get_s3_client_success_via_args(endpoint_url: str | None, monkeypatch: pytest.MonkeyPatch) -> None: - """Verify that get_s3_client creates a client with the correct credentials and endpoint URL using args for the creds.""" +def prep_client_init(monkeypatch: pytest.MonkeyPatch) -> None: + """Set up environment variables to allow get_s3_client to initialize without error.""" reset_s3_client() assert s3_utils._s3_client is None # noqa: SLF001 # set up env vars to ensure that the argument takes precedence monkeypatch.setenv("AWS_ENDPOINT_URL", "http://env-endpoint.com") + monkeypatch.delenv("AWS_ENDPOINT_URL_S3", raising=False) monkeypatch.setenv("AWS_ACCESS_KEY_ID", "aws_access_key_id_env_var") monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "aws_secret_access_key_env_var") + + +# Client creation / reset +@mock_aws +@pytest.mark.s3 +@pytest.mark.parametrize("endpoint_url", ["http://localhost", None]) +def test_get_s3_client_success_via_args(endpoint_url: str | None, monkeypatch: pytest.MonkeyPatch) -> None: + """Verify that get_s3_client creates a client with the correct credentials and endpoint URL using args for the creds.""" + prep_client_init(monkeypatch) + args = { "aws_access_key_id": "aws_access_key_id_argument", "aws_secret_access_key": "aws_secret_access_key_argument", @@ -165,12 +173,8 @@ def test_get_s3_client_success_via_args(endpoint_url: str | None, monkeypatch: p @pytest.mark.parametrize("endpoint_url", ["http://localhost", None]) def test_get_s3_client_success_via_env(endpoint_url: str | None, monkeypatch: pytest.MonkeyPatch) -> None: """Verify that get_s3_client creates a client with the correct credentials and endpoint URL using env vars for the creds.""" - reset_s3_client() - assert s3_utils._s3_client is None # noqa: SLF001 - # set up the endpoint URL as an env var to ensure that the argument takes precedence - monkeypatch.setenv("AWS_ENDPOINT_URL", "http://env-endpoint.com") - monkeypatch.setenv("AWS_ACCESS_KEY_ID", "aws_access_key_id_env_var") - monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "aws_secret_access_key_env_var") + prep_client_init(monkeypatch) + args = { "endpoint_url": endpoint_url, } @@ -247,6 +251,7 @@ def test_get_s3_client_incomplete_creds_via_env( monkeypatch.setenv(key.upper(), value) else: monkeypatch.delenv(key.upper(), raising=False) + # ensure that the AWS config file cannot accidentally be used to provide creds monkeypatch.setenv("AWS_CONFIG_FILE", "/dev/null") with pytest.raises( @@ -629,7 +634,7 @@ def test_stream_to_s3_uploads_large_file(mock_s3_client: Any) -> None: @pytest.mark.skip("TODO: add test(s)") -def test_accepts_custom_requests_implementation() -> None: +def test_stream_to_s3_accepts_custom_requests_implementation() -> None: """A subclassed or alternate requests module works as a drop-in.""" # TODO: add test here? @@ -789,7 +794,7 @@ def strip_checksum_algorithm(method: Callable): """ @functools.wraps(method) - def wrapper(*args, **kwargs): + def wrapper(*args: Any, **kwargs: Any) -> Any: """Remove the ChecksumAlgorithm argument from the call.""" kwargs.pop("ChecksumAlgorithm", None) return method(*args, **kwargs) @@ -823,7 +828,7 @@ def test_copy_object(mocked_s3_client_no_checksum: Any, destination: str) -> Non obj = mocked_s3_client_no_checksum.get_object(Bucket=destination, Key="dst/path/to/file.txt") assert obj["Body"].read() == b"copy me" - assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert response["ResponseMetadata"]["HTTPStatusCode"] == HTTP_200 @pytest.mark.s3 @@ -983,12 +988,12 @@ def test_delete_object_removes_object(mock_s3_client: Any, bucket: str, protocol resp = delete_object(s3_path) assert object_exists(s3_path) is False - assert resp.get("ResponseMetadata", {}).get("HTTPStatusCode") == 204 + assert resp.get("ResponseMetadata", {}).get("HTTPStatusCode") == HTTP_204 # retry the deletion resp = delete_object(s3_path) assert object_exists(s3_path) is False - assert resp.get("ResponseMetadata", {}).get("HTTPStatusCode") == 204 + assert resp.get("ResponseMetadata", {}).get("HTTPStatusCode") == HTTP_204 # delete_object - bucket does not exist From 4d29312f649815d96de34261d7ccdcbf6faf25fd Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Fri, 29 May 2026 08:48:11 -0700 Subject: [PATCH 092/128] break up large function --- src/cdm_data_loaders/ncbi_ftp/promote.py | 242 +++++++++++++++-------- 1 file changed, 160 insertions(+), 82 deletions(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index e7cfb6ea..841dbee3 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -40,6 +40,14 @@ _MAX_DRY_RUN_LOGS = 10 +# Precompile regexes for performance + +# Extract the accession from a path (e.g., "raw_data/GCF/000/001/405/GCF_000001405.39/file" -> "GCF_000001405.39") +_ACCESSION_REGEX = re.compile(r"(GC[AF]_\d{9}\.\d+)") + +# Extract the database, and 3-digit groups from an accession +# (e.g., "GCF_000001405.39" -> ("GCF", "000", "001", "405")) +_ACCESSION_PARTS_REGEX = re.compile(r"(GC[AF])_(\d{3})(\d{3})(\d{3})\.\d+") # Promote from S3 staging prefix @@ -137,7 +145,144 @@ def promote_from_s3( # noqa: PLR0913 # Promote data files (per-file loop) -def _promote_data_files( # noqa: PLR0913, PLR0915 +def _group_files_by_assembly( + data_files: list[str], normalized_staging_prefix: str +) -> defaultdict[tuple[str, str], list[str]]: + """Group files by assembly; skip download_report.json and non-raw_data paths. + + :param data_files: list of S3 keys for staged data files + :param normalized_staging_prefix: normalized staging prefix in S3 + :return: dict mapping (assembly_dir, accession) to list of staged keys for that assembly + """ + assembly_files: defaultdict[tuple[str, str], list[str]] = defaultdict(list) + for staged_key in data_files: + if staged_key.endswith("download_report.json"): + continue + rel_path = staged_key[len(normalized_staging_prefix) :] + if not rel_path.startswith("raw_data/"): + continue + acc_match = _ACCESSION_REGEX.search(staged_key) + adir_match = re.search(r"raw_data/GC[AF]/\d+/\d+/\d+/([^/]+)/", staged_key) + if acc_match and adir_match: + assembly_files[(adir_match.group(1), acc_match.group(1))].append(staged_key) + return assembly_files + + +def _promote_file( # noqa: PLR0913 + staged_key: str, + normalized_staging_prefix: str, + lakehouse_key_prefix: str, + staging_bucket: str, + lakehouse_bucket: str, + sidecars: set[str], +) -> tuple[DescriptorResource, str]: + """Download one staged file, re-upload to Lakehouse with MD5 metadata. + + :param staged_key: S3 key of the staged file + :param normalized_staging_prefix: normalized staging prefix in S3 + :param lakehouse_key_prefix: S3 key prefix for final Lakehouse locations + :param staging_bucket: S3 bucket containing the staged file + :param lakehouse_bucket: S3 bucket for the final Lakehouse destination + :param sidecars: set of S3 keys for sidecar files (to check for MD5 metadata) + :return: ``(resource_dict, staged_key)`` on success; raises on failure. + """ + s3 = get_s3_client() + rel_path = staged_key[len(normalized_staging_prefix) :] + final_key = lakehouse_key_prefix + rel_path + final_key_path = PurePosixPath(final_key) + + with tempfile.NamedTemporaryFile(delete=False) as tmp: + tmp_path = tmp.name + try: + s3.download_file(Bucket=staging_bucket, Key=staged_key, Filename=tmp_path) + + metadata: dict[str, str] = {} + md5_key = staged_key + ".md5" + if md5_key in sidecars: + md5_obj = s3.get_object(Bucket=staging_bucket, Key=md5_key) + metadata["md5"] = md5_obj["Body"].read().decode().strip() + + upload_succeeded = upload_file( + tmp_path, + f"{lakehouse_bucket}/{final_key_path.parent}", + user_metadata=metadata, + object_name=final_key_path.name, + show_progress=False, + ) + if not upload_succeeded: + msg = f"upload_file returned False for {staged_key}" + raise RuntimeError(msg) + + fname = final_key_path.name + ext = fname.rsplit(".", 1)[-1] if "." in fname else "" + resource: DescriptorResource = { + "name": fname.lower(), + "path": final_key, + "format": ext, + "bytes": Path(tmp_path).stat().st_size, + "hash": metadata.get("md5"), + } + return resource, staged_key + finally: + Path(tmp_path).unlink() + + +def _write_descriptor_for_assembly( + assembly_dir: str, + accession: str, + resources: list[DescriptorResource], + lakehouse_bucket: str, + lakehouse_key_prefix: str, +) -> bool: + """Create and upload a frictionless descriptor for an assembly. + + :param assembly_dir: full assembly directory name + :param accession: assembly accession (e.g. "GCF_000001405.39") + :param resources: list of DescriptorResource dicts for the assembly's files + :param lakehouse_bucket: S3 bucket for the final Lakehouse destination + :param lakehouse_key_prefix: S3 key prefix for final Lakehouse locations + :return: True if the descriptor was successfully written, False otherwise + """ + try: + descriptor_key = build_descriptor_key(assembly_dir, lakehouse_key_prefix) + if object_exists(f"{lakehouse_bucket}/{descriptor_key}"): + logger.debug("Descriptor already exists, skipping: %s", descriptor_key) + else: + descriptor = create_descriptor(assembly_dir, accession, resources) + descriptor_key = upload_descriptor( + descriptor, assembly_dir, lakehouse_bucket, lakehouse_key_prefix, dry_run=False + ) + logger.debug("Uploaded descriptor: %s", descriptor_key) + return True + except Exception: + logger.exception("Failed to write descriptor for %s", assembly_dir) + return False + + +def _batch_delete( + promoted_keys: list[str], + sidecars: set[str], + staging_bucket: str, +) -> None: + """Batch-delete all staged data files and their sidecars in one API call. + + :param promoted_keys: list of staged keys that were successfully promoted + :param sidecars: set of all sidecar keys in staging (to check for existence of sidecars for promoted files) + :param staging_bucket: S3 bucket containing the staged files + """ + keys_to_delete = list(promoted_keys) + keys_to_delete.extend( + key + sidecar_ext + for key in promoted_keys + for sidecar_ext in (".md5", ".crc64nvme") + if key + sidecar_ext in sidecars + ) + del_errors = delete_objects(staging_bucket, keys_to_delete) + for err in del_errors: + logger.warning("Failed to delete staged file %s: %s", err.get("Key"), err.get("Message")) + + +def _promote_data_files( # noqa: PLR0913 data_files: list[str], sidecars: set[str], normalized_staging_prefix: str, @@ -157,68 +302,21 @@ def _promote_data_files( # noqa: PLR0913, PLR0915 :return: (promoted_count, failed_count, descriptors_written, promoted_accessions) """ - s3 = get_s3_client() promoted = 0 failed = 0 descriptors_written = 0 promoted_accessions: set[str] = set() - - # Group files by assembly; skip download_report.json and non-raw_data paths - assembly_files: defaultdict[tuple[str, str], list[str]] = defaultdict(list) - for staged_key in data_files: - if staged_key.endswith("download_report.json"): - continue - rel_path = staged_key[len(normalized_staging_prefix) :] - if not rel_path.startswith("raw_data/"): - continue - acc_match = re.search(r"(GC[AF]_\d{9}\.\d+)", staged_key) - adir_match = re.search(r"raw_data/GC[AF]/\d+/\d+/\d+/([^/]+)/", staged_key) - if acc_match and adir_match: - assembly_files[(adir_match.group(1), acc_match.group(1))].append(staged_key) + assembly_files = _group_files_by_assembly(data_files, normalized_staging_prefix) def _promote_one(staged_key: str) -> tuple[DescriptorResource, str]: - """Download one staged file, re-upload to Lakehouse with MD5 metadata. - - :return: ``(resource_dict, staged_key)`` on success; raises on failure. - """ - rel_path = staged_key[len(normalized_staging_prefix) :] - final_key = lakehouse_key_prefix + rel_path - final_key_path = PurePosixPath(final_key) - - with tempfile.NamedTemporaryFile(delete=False) as tmp: - tmp_path = tmp.name - try: - s3.download_file(Bucket=staging_bucket, Key=staged_key, Filename=tmp_path) - - metadata: dict[str, str] = {} - md5_key = staged_key + ".md5" - if md5_key in sidecars: - md5_obj = s3.get_object(Bucket=staging_bucket, Key=md5_key) - metadata["md5"] = md5_obj["Body"].read().decode().strip() - - upload_succeeded = upload_file( - tmp_path, - f"{lakehouse_bucket}/{final_key_path.parent}", - user_metadata=metadata, - object_name=final_key_path.name, - show_progress=False, - ) - if not upload_succeeded: - msg = f"upload_file returned False for {staged_key}" - raise RuntimeError(msg) - - fname = final_key_path.name - ext = fname.rsplit(".", 1)[-1] if "." in fname else "" - resource: DescriptorResource = { - "name": fname.lower(), - "path": final_key, - "format": ext, - "bytes": Path(tmp_path).stat().st_size, - "hash": metadata.get("md5"), - } - return resource, staged_key - finally: - Path(tmp_path).unlink() + return _promote_file( + staged_key, + normalized_staging_prefix, + lakehouse_key_prefix, + staging_bucket, + lakehouse_bucket, + sidecars, + ) total_files = sum(len(v) for v in assembly_files.values()) _dry_run_log_count = 0 @@ -262,31 +360,11 @@ def _promote_one(staged_key: str) -> tuple[DescriptorResource, str]: # Write descriptor and delete staged files immediately after a fully successful assembly if assembly_failed == 0 and promoted_keys: - try: - descriptor_key = build_descriptor_key(adir, lakehouse_key_prefix) - if object_exists(f"{lakehouse_bucket}/{descriptor_key}"): - logger.debug("Descriptor already exists, skipping: %s", descriptor_key) - else: - descriptor = create_descriptor(adir, acc, resources) - descriptor_key = upload_descriptor( - descriptor, adir, lakehouse_bucket, lakehouse_key_prefix, dry_run=False - ) - logger.debug("Uploaded descriptor: %s", descriptor_key) - descriptors_written += 1 - except Exception: - logger.exception("Failed to write descriptor for %s", adir) - - # Batch-delete all staged data files and their sidecars in one API call - keys_to_delete = list(promoted_keys) - keys_to_delete.extend( - key + sidecar_ext - for key in promoted_keys - for sidecar_ext in (".md5", ".crc64nvme") - if key + sidecar_ext in sidecars - ) - del_errors = delete_objects(staging_bucket, keys_to_delete) - for err in del_errors: - logger.warning("Failed to delete staged file %s: %s", err.get("Key"), err.get("Message")) + if _write_descriptor_for_assembly(adir, acc, resources, lakehouse_bucket, lakehouse_key_prefix): + descriptors_written += 1 + + # delete staged files in batch with their sidecars (if any) + _batch_delete(promoted_keys, sidecars, staging_bucket) return promoted, failed, descriptors_written, promoted_accessions From 9a40b661d8279deea57a3cd92627fb82c5f74573 Mon Sep 17 00:00:00 2001 From: i alarmed alien Date: Fri, 29 May 2026 09:18:07 -0700 Subject: [PATCH 093/128] Apply suggestions from code review Co-authored-by: Matt Dawson --- src/cdm_data_loaders/utils/s3.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index 3212ce7a..6ade8c7a 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -42,7 +42,7 @@ def get_s3_client(args: dict[str, str | None] | None = None) -> botocore.client. - endpoint_url: the endpoint URL for the S3 client (e.g., "https://s3.amazonaws.com" or "https://my-s3-server.com") If arguments are not provided, the client will be created using boto3's default - configuration method, which looks environment variables (AWS_ACCESS_KEY_ID, + configuration method, which looks for environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, and AWS_ENDPOINT_URL_S3 or AWS_ENDPOINT_URL) or an ``./aws`` config directory. See the boto3 documentation for more details. @@ -65,9 +65,7 @@ def get_s3_client(args: dict[str, str | None] | None = None) -> botocore.client. kwargs = {k: v for k, v in args.items() if k in valid_kwargs and v is not None} # make sure that if aws_access_key_id or aws_secret_access_key is provided, the other is also provided. - if (kwargs.get("aws_access_key_id") and not kwargs.get("aws_secret_access_key")) or ( - not kwargs.get("aws_access_key_id") and kwargs.get("aws_secret_access_key") - ): + if bool(kwargs.get("aws_access_key_id")) ^ bool(kwargs.get("aws_secret_access_key")): msg = "Cannot initialise s3 client: aws_access_key_id and aws_secret_access_key must be provided together, either via args or environment variables or a config file" raise ValueError(msg) From 0862f1bf4c137e5d84e1b36db4bfd131b2e599c2 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Fri, 29 May 2026 14:48:55 -0700 Subject: [PATCH 094/128] break up another large function; add tests --- src/cdm_data_loaders/ncbi_ftp/promote.py | 189 +++++++++++++------- tests/ncbi_ftp/test_promote.py | 216 +++++++++++++++++++++++ 2 files changed, 340 insertions(+), 65 deletions(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index 841dbee3..71e43714 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -42,13 +42,16 @@ # Precompile regexes for performance -# Extract the accession from a path (e.g., "raw_data/GCF/000/001/405/GCF_000001405.39/file" -> "GCF_000001405.39") +# Extract the accession from a path (e.g., "raw_data/GCF/000/001/405/GCF_000001405.39_Some_description/file" -> "GCF_000001405.39") _ACCESSION_REGEX = re.compile(r"(GC[AF]_\d{9}\.\d+)") # Extract the database, and 3-digit groups from an accession # (e.g., "GCF_000001405.39" -> ("GCF", "000", "001", "405")) _ACCESSION_PARTS_REGEX = re.compile(r"(GC[AF])_(\d{3})(\d{3})(\d{3})\.\d+") +# Extract the assembly_dir from a path (e.g., "raw_data/GCF/000/001/405/GCF_000001405.39_Some_description/file" -> "GCF_000001405.39_Some_description") +_ASSEMBLY_DIR_REGEX = re.compile(r"raw_data/GC[AF]/\d+/\d+/\d+/([^/]+)/") + # Promote from S3 staging prefix @@ -372,6 +375,109 @@ def _promote_one(staged_key: str) -> tuple[DescriptorResource, str]: # Archive assemblies +def _get_accession_path_prefix(accession: str, lakehouse_key_prefix: str) -> str | None: + """Get the S3 key prefix for all files related to an accession. + + :param accession: assembly accession (e.g. "GCF_000001405.39_Some_description") + :param lakehouse_key_prefix: S3 key prefix for the Lakehouse dataset root + :return: S3 key prefix under which all files for the accession are stored, or None if the accession format is invalid + """ + m = _ACCESSION_PARTS_REGEX.match(accession) + if not m: + logger.warning("Invalid accession format: %s", accession) + return None + db = m.group(1) + p1, p2, p3 = m.group(2), m.group(3), m.group(4) + return f"{lakehouse_key_prefix}raw_data/{db}/{p1}/{p2}/{p3}/{accession}" + + +def _get_source_dest_pairs_for_accession( + accession: str, + lakehouse_bucket: str, + lakehouse_key_prefix: str, + release_tag: str, + archive_reason: str, +) -> list[tuple[str, str]]: + """Get list of (source_key, archive_key) pairs for all objects related to an accession. + + :param accession: assembly accession (e.g. "GCF_000001405.39_Some_description") + :param lakehouse_bucket: S3 bucket for the Lakehouse (source and archive destination) + :param lakehouse_key_prefix: S3 key prefix for the Lakehouse dataset root + :param release_tag: release tag for the archive + :param archive_reason: reason for archiving + :return: list of (source_key, archive_key) pairs for all objects related to the accession + """ + source_prefix = _get_accession_path_prefix(accession, lakehouse_key_prefix) + if not source_prefix: + return [] + matching_objs: list[dict[str, Any]] = list_matching_objects(f"{lakehouse_bucket}/{source_prefix}") + return [ + ( + obj["Key"], + f"{lakehouse_key_prefix}archive/{release_tag}/{archive_reason}/{obj['Key'][len(lakehouse_key_prefix) :]}", + ) + for obj in matching_objs + ] + + +def _dry_run_output(key_pairs: list[tuple[str, str]], log_count: int) -> int: + """Log source and archive key pairs for a dry run, with a limit on how many are logged at INFO level. + + :param key_pairs: list of (source_key, archive_key) pairs + :param log_count: current count of logged entries + :return: updated count of logged entries + """ + _dry_run_log_count = log_count + for source_key, archive_key in key_pairs: + if _dry_run_log_count < _MAX_DRY_RUN_LOGS: + logger.info("[dry-run] would archive: %s -> %s", source_key, archive_key) + else: + logger.debug("[dry-run] would archive: %s -> %s", source_key, archive_key) + _dry_run_log_count += 1 + return _dry_run_log_count + + +def _archive_objects(key_pairs: list[tuple[str, str]], lakehouse_bucket: str, *, delete_source: bool) -> int: + """Copy objects from source keys to archive keys, optionally deleting the source objects. + + :param key_pairs: list of (source_key, archive_key) pairs + :param lakehouse_bucket: S3 bucket for the Lakehouse (source and archive destination) + :param delete_source: if True, delete the source object after copying + :return: number of objects successfully archived + """ + archived = 0 + if not key_pairs: + return archived + keys_to_delete: list[str] = [] + n_workers = min(32, len(key_pairs)) + with ThreadPoolExecutor(max_workers=n_workers) as executor: + futures = { + executor.submit( + copy_object, + f"{lakehouse_bucket}/{src}", + f"{lakehouse_bucket}/{arch}", + ): src + for src, arch in key_pairs + } + for future in as_completed(futures): + src = futures[future] + try: + future.result() + archived += 1 + if delete_source: + keys_to_delete.append(src) + logger.debug(" Archived: %s", src) + except Exception: + logger.exception("Failed to archive %s", src) + + if delete_source and keys_to_delete: + del_errors = delete_objects(lakehouse_bucket, keys_to_delete) + for err in del_errors: + logger.warning("Failed to delete %s: %s", err.get("Key"), err.get("Message")) + + return archived + + def _archive_assemblies( # noqa: PLR0913 manifest_local_path: str, lakehouse_bucket: str, @@ -398,7 +504,6 @@ def _archive_assemblies( # noqa: PLR0913 :param dry_run: if True, log without making changes :return: number of objects archived """ - s3 = get_s3_client() release_tag = ncbi_release or "unknown" archived = 0 @@ -407,79 +512,33 @@ def _archive_assemblies( # noqa: PLR0913 _dry_run_log_count = 0 for accession in tqdm.tqdm(accessions, unit="accession", desc="Archiving"): - m = re.match(r"(GC[AF])_(\d{3})(\d{3})(\d{3})\.\d+", accession) - if not m: - logger.warning("Cannot parse accession for archival: %s", accession) + # get list of (source_key, archive_key) pairs for all objects related to this accession + key_pairs: list[tuple[str, str]] = _get_source_dest_pairs_for_accession( + accession, + lakehouse_bucket, + lakehouse_key_prefix, + release_tag, + archive_reason, + ) + if not key_pairs: + logger.debug("No objects found for %s, skipping archive", accession) continue - db = m.group(1) - p1, p2, p3 = m.group(2), m.group(3), m.group(4) - source_prefix = f"{lakehouse_key_prefix}raw_data/{db}/{p1}/{p2}/{p3}/" - - paginator = s3.get_paginator("list_objects_v2") - matching_keys: list[str] = [] - for page in paginator.paginate(Bucket=lakehouse_bucket, Prefix=source_prefix): - matching_keys.extend(obj["Key"] for obj in page.get("Contents", []) if accession in obj["Key"]) - - if not matching_keys: - logger.debug("No objects found for %s, skipping archive", accession) + # Archive all files for this accession + if dry_run: + _dry_run_log_count = _dry_run_output(key_pairs, _dry_run_log_count) + archived += len(key_pairs) continue + archived += _archive_objects(key_pairs, lakehouse_bucket, delete_source=delete_source) # Infer assembly_dir from key paths for descriptor archival assembly_dir: str | None = None - for key in matching_keys: - adir_match = re.search(r"raw_data/GC[AF]/\d+/\d+/\d+/([^/]+)/", key) + for src, _ in key_pairs: + adir_match = _ASSEMBLY_DIR_REGEX.search(src) if adir_match: assembly_dir = adir_match.group(1) break - key_pairs = [ - ( - source_key, - f"{lakehouse_key_prefix}archive/{release_tag}/{archive_reason}/{source_key[len(lakehouse_key_prefix) :]}", - ) - for source_key in matching_keys - ] - - if dry_run: - for source_key, archive_key in key_pairs: - if _dry_run_log_count < _MAX_DRY_RUN_LOGS: - logger.info("[dry-run] would archive: %s -> %s", source_key, archive_key) - else: - logger.debug("[dry-run] would archive: %s -> %s", source_key, archive_key) - _dry_run_log_count += 1 - archived += len(key_pairs) - continue - - # Copy all files for this accession concurrently - keys_to_delete: list[str] = [] - n_workers = min(32, len(key_pairs)) - with ThreadPoolExecutor(max_workers=n_workers) as executor: - futures = { - executor.submit( - copy_object, - f"{lakehouse_bucket}/{src}", - f"{lakehouse_bucket}/{arch}", - ): src - for src, arch in key_pairs - } - for future in as_completed(futures): - src = futures[future] - try: - future.result() - archived += 1 - if delete_source: - keys_to_delete.append(src) - logger.debug(" Archived: %s", src) - except Exception: - logger.exception("Failed to archive %s", src) - - # Batch-delete source keys in a single API call - if keys_to_delete: - del_errors = delete_objects(lakehouse_bucket, keys_to_delete) - for err in del_errors: - logger.warning("Failed to delete %s: %s", err.get("Key"), err.get("Message")) - # Archive the frictionless descriptor alongside raw data if assembly_dir: try: diff --git a/tests/ncbi_ftp/test_promote.py b/tests/ncbi_ftp/test_promote.py index 95740f3e..34bc82e7 100644 --- a/tests/ncbi_ftp/test_promote.py +++ b/tests/ncbi_ftp/test_promote.py @@ -1,7 +1,10 @@ """Tests for ncbi_ftp.promote module — S3 promote, archive, manifest trimming.""" import hashlib +import logging from pathlib import Path +from typing import Any +from unittest.mock import patch import botocore.client import pytest @@ -9,6 +12,10 @@ from cdm_data_loaders.ncbi_ftp.promote import ( DEFAULT_LAKEHOUSE_KEY_PREFIX, _archive_assemblies, + _archive_objects, + _dry_run_output, + _get_accession_path_prefix, + _get_source_dest_pairs_for_accession, _trim_manifest, promote_from_s3, ) @@ -140,6 +147,215 @@ def test_trim_manifest( assert acc not in remaining +# Helpers for _archive_assemblies tests + + +def _mock_list_matching_objects(path: str) -> list[dict[str, Any]]: + bucket = "some-bucket" + prefix = "some/prefix/" + if path == f"{bucket}/{prefix}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6": + return [ + {"Key": f"{prefix}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz"}, + {"Key": f"{prefix}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_protein.faa.gz"}, + ] + if path == f"{bucket}/{prefix}raw_data/GCF/000/005/845/GCF_000005845.2_ASM584v2": + return [ + {"Key": f"{prefix}raw_data/GCF/000/005/845/GCF_000005845.2_ASM584v2/GCF_000005845.2_genomic.fna.gz"}, + ] + return [] + + +@pytest.mark.parametrize( + ("accession", "prefix", "expected"), + [ + pytest.param( + "GCF_012345678.90_Some_description", + "some/prefix/", + "some/prefix/raw_data/GCF/012/345/678/GCF_012345678.90_Some_description", + id="standard", + ), + pytest.param( + "GCF_000001215.4_Release_6", + "another/prefix/", + "another/prefix/raw_data/GCF/000/001/215/GCF_000001215.4_Release_6", + id="standard-2", + ), + pytest.param("INVALID_ACCESSION", "prefix/", None, id="invalid-format"), + ], +) +def test_get_accession_path_prefix( + accession: str, + prefix: str, + expected: str | None, +) -> None: + """get_accession_path_prefix returns correct path for valid accessions, None for invalid.""" + result = _get_accession_path_prefix(accession, prefix) + assert result == expected + + +@pytest.mark.parametrize( + ("accession", "bucket", "prefix", "release_tag", "archive_reason", "expected"), + [ + pytest.param( + "GCF_000001215.4_Release_6", + "some-bucket", + "some/prefix/", + "2024-01", + "test-reason", + [ + ( + "some/prefix/raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz", + "some/prefix/archive/2024-01/test-reason/raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz", + ), + ( + "some/prefix/raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_protein.faa.gz", + "some/prefix/archive/2024-01/test-reason/raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_protein.faa.gz", + ), + ], + id="standard", + ), + pytest.param( + "GCF_000005845.2_ASM584v2", + "some-bucket", + "some/prefix/", + "2024-01", + "test-reason", + [ + ( + "some/prefix/raw_data/GCF/000/005/845/GCF_000005845.2_ASM584v2/GCF_000005845.2_genomic.fna.gz", + "some/prefix/archive/2024-01/test-reason/raw_data/GCF/000/005/845/GCF_000005845.2_ASM584v2/GCF_000005845.2_genomic.fna.gz", + ), + ], + id="standard-2", + ), + pytest.param( + "GCF_000001405.39_GRCh38.p14", + "some-bucket", + "some/prefix/", + "2024-01", + "test-reason", + [], + id="accession-not-found", + ), + pytest.param( + "INVALID_ACCESSION", + "some-bucket", + "some/prefix/", + "2024-01", + "test-reason", + [], + id="invalid-format", + ), + ], +) +def test_get_source_dest_pairs_for_accession( # noqa: PLR0913 + accession: str, + bucket: str, + prefix: str, + release_tag: str, + archive_reason: str, + expected: list[tuple[str, str]], +) -> None: + """get_source_dest_pairs_for_accession returns correct source-dest pairs for valid accessions, empty list for invalid.""" + with patch("cdm_data_loaders.ncbi_ftp.promote.list_matching_objects", side_effect=_mock_list_matching_objects): + result = _get_source_dest_pairs_for_accession( + accession, + bucket, + prefix, + release_tag, + archive_reason, + ) + assert result == expected + + +@pytest.mark.parametrize( + ("key_pairs", "log_count", "expected", "info_log_strings"), + [ + pytest.param( + [("source1", "dest1"), ("source2", "dest2")], + 0, + 2, + ["[dry-run] would archive: source1 -> dest1", "[dry-run] would archive: source2 -> dest2"], + id="standard", + ), + pytest.param( + [("source1", "dest1"), ("source2", "dest2"), ("source3", "dest3")], + 9, + 12, + ["[dry-run] would archive: source1 -> dest1"], + id="exceeds-log-cutoff", + ), + pytest.param( + [], + 0, + 0, + [], + id="empty", + ), + ], +) +def test_dry_run_output( + key_pairs: list[tuple[str, str]], + log_count: int, + expected: int, + info_log_strings: list[str], + caplog: pytest.LogCaptureFixture, +) -> None: + """In dry_run mode, _archive_assemblies logs source-dest pairs but does not copy.""" + with caplog.at_level(logging.INFO): + result = _dry_run_output(key_pairs, log_count) + assert result == expected + for log_string in info_log_strings: + assert log_string in caplog.text + + +@pytest.mark.parametrize( + ("key_pairs", "bucket", "delete_source", "expected", "existing_objects"), + [ + pytest.param( + [("source1", "dest1"), ("source2", "dest2")], + TEST_BUCKET, + False, + 2, + ["source1", "source2", "dest1", "dest2"], + id="keep-source", + ), + pytest.param( + [("source1", "dest1"), ("source2", "dest2")], + TEST_BUCKET, + True, + 2, + ["dest1", "dest2"], + id="delete-source", + ), + pytest.param( + [], + TEST_BUCKET, + True, + 0, + [], + id="empty", + ), + ], +) +@pytest.mark.s3 +def test_archive_objects( # noqa: PLR0913 + mock_s3_client_no_checksum: botocore.client.BaseClient, + key_pairs: list[tuple[str, str]], + bucket: str, + delete_source: bool, + expected: int, + existing_objects: list[str], +) -> None: + """_archive_assemblies copies source to dest for each pair, and deletes source if delete_source=True.""" + for source, _ in key_pairs: + mock_s3_client_no_checksum.put_object(Bucket=bucket, Key=source, Body=b"data") + assert _archive_objects(key_pairs, bucket, delete_source=delete_source) == expected + assert mock_s3_client_no_checksum.list_objects_v2(Bucket=bucket).get("KeyCount", 0) == len(existing_objects) + for obj in mock_s3_client_no_checksum.list_objects_v2(Bucket=bucket).get("Contents", []): + assert obj["Key"] in existing_objects, f"Unexpected object in bucket: {obj['Key']}" + + @pytest.mark.s3 def test_archive_assemblies_removed(mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path) -> None: """Removed accessions are archived and originals deleted.""" From 1b5f2b20b561a2a45f3524fa8c00e896f4129a9c Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Fri, 29 May 2026 15:18:43 -0700 Subject: [PATCH 095/128] updates for latest merge of develop branch --- docker-compose.yml | 8 ++++---- scripts/s3_local.py | 12 ++++++------ tests/integration/conftest.py | 16 +--------------- tests/ncbi_ftp/test_manifest.py | 3 ++- tests/utils/test_s3.py | 28 ---------------------------- 5 files changed, 13 insertions(+), 54 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 04cffa62..39a0b3db 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,9 +22,9 @@ services: minio: condition: service_healthy environment: - MINIO_ENDPOINT_URL: http://minio:9000 - MINIO_ACCESS_KEY: minioadmin - MINIO_SECRET_KEY: minioadmin + AWS_ENDPOINT_URL: http://minio:9000 + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin entrypoint: - /bin/sh - -c @@ -32,7 +32,7 @@ services: attempts=0 until python3 -c " import urllib.request, os - urllib.request.urlopen(os.environ['MINIO_ENDPOINT_URL'] + '/minio/health/live', timeout=1) + urllib.request.urlopen(os.environ['AWS_ENDPOINT_URL'] + '/minio/health/live', timeout=1) " 2>/dev/null; do attempts=$$((attempts + 1)) if [ "$$attempts" -ge 30 ]; then diff --git a/scripts/s3_local.py b/scripts/s3_local.py index b8a27612..2f3926de 100755 --- a/scripts/s3_local.py +++ b/scripts/s3_local.py @@ -11,9 +11,9 @@ Environment variables (with defaults for the walkthrough): - MINIO_ENDPOINT_URL http://localhost:9000 - MINIO_ACCESS_KEY minioadmin - MINIO_SECRET_KEY minioadmin + AWS_ENDPOINT_URL http://localhost:9000 + AWS_ACCESS_KEY_ID minioadmin + AWS_SECRET_ACCESS_KEY minioadmin """ import json @@ -28,9 +28,9 @@ def _client() -> BaseClient: return boto3.client( "s3", - endpoint_url=os.environ.get("MINIO_ENDPOINT_URL", "http://localhost:9000"), - aws_access_key_id=os.environ.get("MINIO_ACCESS_KEY", "minioadmin"), - aws_secret_access_key=os.environ.get("MINIO_SECRET_KEY", "minioadmin"), + endpoint_url=os.environ.get("AWS_ENDPOINT_URL", "http://localhost:9000"), + aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID", "minioadmin"), + aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY", "minioadmin"), ) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 3f44b6c3..f3017b31 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -24,12 +24,6 @@ from cdm_data_loaders.ncbi_ftp.assembly import build_accession_path from cdm_data_loaders.utils.s3 import reset_s3_client -# MinIO connection defaults - -MINIO_ENDPOINT_URL = os.environ["MINIO_ENDPOINT_URL"] -MINIO_ACCESS_KEY = os.environ["MINIO_ACCESS_KEY"] -MINIO_SECRET_KEY = os.environ["MINIO_SECRET_KEY"] - # Maximum length of a bucket name per S3/DNS spec _MAX_BUCKET_LEN = 63 @@ -44,9 +38,6 @@ def _minio_reachable() -> bool: try: client = boto3.client( "s3", - endpoint_url=MINIO_ENDPOINT_URL, - aws_access_key_id=MINIO_ACCESS_KEY, - aws_secret_access_key=MINIO_SECRET_KEY, config=botocore.config.Config( connect_timeout=1, read_timeout=1, @@ -82,12 +73,7 @@ def minio_s3_client() -> botocore.client.BaseClient: Patches ``get_s3_client`` on every module that uses it so internal calls are transparently routed to MinIO. """ - client = boto3.client( - "s3", - endpoint_url=MINIO_ENDPOINT_URL, - aws_access_key_id=MINIO_ACCESS_KEY, - aws_secret_access_key=MINIO_SECRET_KEY, - ) + client = boto3.client("s3") reset_s3_client() with ( diff --git a/tests/ncbi_ftp/test_manifest.py b/tests/ncbi_ftp/test_manifest.py index f8b321d6..ffdd69c7 100644 --- a/tests/ncbi_ftp/test_manifest.py +++ b/tests/ncbi_ftp/test_manifest.py @@ -392,8 +392,9 @@ def test_verify_transfer_candidates_empty_input(mock_connect: MagicMock) -> None assert verify_transfer_candidates([], {}, _BUCKET, "prefix/") == [] +@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") -def test_verify_transfer_candidates_unknown_accession_kept(mock_connect: MagicMock) -> None: +def test_verify_transfer_candidates_unknown_accession_kept(mock_connect: MagicMock, mock_s3: MagicMock) -> None: """Accessions not in assemblies dict are kept (conservative).""" mock_connect.return_value = MagicMock() assert verify_transfer_candidates(["GCF_999999999.1"], {}, _BUCKET, "prefix/") == ["GCF_999999999.1"] diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index e2d2b0a2..a9b4ec8e 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -788,24 +788,6 @@ def test_upload_dir_raises_on_empty_destination(sample_dir: Path) -> None: upload_dir(sample_dir, "") -# FIXME: once moto supports CRC64NVME, this can be removed -def strip_checksum_algorithm(method: Callable): - """Wrap a boto3 S3 method to remove the ChecksumAlgorithm argument before calling moto. - - Moto does not implement CRC64NVME checksums, so any call that includes - ChecksumAlgorithm='CRC64NVME' would fail. This wrapper silently drops the - argument so the rest of the call proceeds normally against the moto backend. - """ - - @functools.wraps(method) - def wrapper(*args: Any, **kwargs: Any) -> Any: - """Remove the ChecksumAlgorithm argument from the call.""" - kwargs.pop("ChecksumAlgorithm", None) - return method(*args, **kwargs) - - return wrapper - - @pytest.fixture def mocked_s3_client_no_checksum(mock_s3_client: Any) -> Any: """Yield the mocked S3 client with copy_object patched to strip ChecksumAlgorithm. @@ -865,16 +847,6 @@ def test_copy_file_source_object_nonexistent() -> None: copy_object(s3_path, f"{CDM_LAKE_BUCKET}/a/different/path/to/file") -@pytest.mark.s3 -@pytest.mark.usefixtures("mock_s3_client") -def test_copy_file_source_object_nonexistent() -> None: - """Ensure that the code throws an error if the source object does not exist.""" - s3_path = f"{CDM_LAKE_BUCKET}/some/path/to/file" - assert object_exists(s3_path) is False - with pytest.raises(Exception, match="The specified key does not exist"): - copy_object(s3_path, f"{CDM_LAKE_BUCKET}/a/different/path/to/file") - - # copy_directory tests def put_objects(mock_s3_client: Any, bucket: str, keys: list[str], body: bytes = b"data") -> None: """Helper to seed objects into a bucket.""" From 7c4b7f173a1e19b467e2fd9de343ccb9c35c25d0 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Fri, 29 May 2026 15:32:59 -0700 Subject: [PATCH 096/128] Potential fix for pull request finding 'Unused import' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- tests/integration/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index f3017b31..35dfc0ac 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -7,7 +7,6 @@ """ import hashlib -import os import re from pathlib import Path from typing import Any From d9b4a06d493b26ed06e7e9f44bb94861e7072975 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 1 Jun 2026 15:15:15 -0700 Subject: [PATCH 097/128] revert to previous head_object response --- pyproject.toml | 9 +++--- src/cdm_data_loaders/ncbi_ftp/manifest.py | 17 +++++++---- src/cdm_data_loaders/utils/s3.py | 30 ++++--------------- tests/integration/test_manifest_e2e.py | 25 ++++++++++++++-- tests/ncbi_ftp/conftest.py | 10 ++++++- tests/ncbi_ftp/test_manifest.py | 35 ++++++++++++++++------- tests/ncbi_ftp/test_metadata.py | 9 +++++- tests/pipelines/test_ncbi_ftp_download.py | 31 +++++++++++++------- tests/utils/test_s3.py | 30 ++++++++++++------- 9 files changed, 125 insertions(+), 71 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6b0ce015..add214e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -208,10 +208,11 @@ S3_ENDPOINT_URL = "http://localhost:9000" # aws config settings AWS_CONFIG_FILE = "/dev/null" # ensure tests don't pick up creds from a real config file AWS_DEFAULT_REGION = "us-east-1" -AWS_ENDPOINT_URL = { unset = true } -AWS_ENDPOINT_URL_S3 = { unset = true} -AWS_ACCESS_KEY_ID = { unset = true} -AWS_SECRET_ACCESS_KEY = { unset = true} +# default to use containerized S3 test store +AWS_ENDPOINT_URL = "http://localhost:9000" +AWS_ENDPOINT_URL_S3 = { unset = true } +AWS_ACCESS_KEY_ID = "minioadmin" +AWS_SECRET_ACCESS_KEY = "minioadmin" [tool.uv.sources] cdm-schema = { git = "https://github.com/kbase/cdm-schema.git" } diff --git a/src/cdm_data_loaders/ncbi_ftp/manifest.py b/src/cdm_data_loaders/ncbi_ftp/manifest.py index 0df571ea..cb7cdae0 100644 --- a/src/cdm_data_loaders/ncbi_ftp/manifest.py +++ b/src/cdm_data_loaders/ncbi_ftp/manifest.py @@ -18,6 +18,8 @@ from pathlib import Path from typing import Any +from botocore.exceptions import ClientError + from cdm_data_loaders.ncbi_ftp.assembly import ( FILE_FILTERS, FTP_HOST, @@ -405,13 +407,16 @@ def _does_accession_need_update( """ for fname, expected_md5 in target_checksums.items(): s3_path = f"{bucket}/{s3_prefix}{fname}" - obj_info = head_object(s3_path) - - if obj_info is None: - logger.debug("File missing from store: %s", s3_path) - return True + s3_md5 = "" + try: + obj_info = head_object(s3_path) + s3_md5 = obj_info.get("Metadata",{}).get("md5", "") + except ClientError as e: + if e.response["Error"]["Code"] == "404": # type: ignore[union-attr] + logger.debug("File missing from store: %s", s3_path) + return True + raise - s3_md5 = obj_info["metadata"].get("md5", "") if s3_md5 != expected_md5: logger.debug("MD5 mismatch for %s: S3=%s FTP=%s", s3_path, s3_md5, expected_md5) return True diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index 8c71ea7f..027af27c 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -158,32 +158,17 @@ def list_matching_objects(s3_path: str) -> list[dict[str, Any]]: return contents -def head_object(s3_path: str) -> dict[str, Any] | None: - """Return metadata for an S3 object, or None if it does not exist. - - The returned dict contains: - - ``size``: content length in bytes - - ``metadata``: user metadata dict - - ``checksum_crc64nvme``: CRC64NVME checksum string (if available) +def head_object(s3_path: str) -> dict[str, Any]: + """Check whether an object exists on s3. :param s3_path: path to the object on s3, INCLUDING the bucket name :type s3_path: str - :return: dict with object info, or None if the object does not exist - :rtype: dict[str, Any] | None + :return: response from the head_object request + :rtype: dict[str, Any] """ s3 = get_s3_client() (bucket, key) = split_s3_path(s3_path) - try: - resp = s3.head_object(Bucket=bucket, Key=key, ChecksumMode="ENABLED") - except Exception as e: # noqa: BLE001 - if e.response["Error"]["Code"] == "404": # type: ignore[union-attr] - return None - raise - return { - "size": resp["ContentLength"], - "metadata": resp.get("Metadata", {}), - "checksum_crc64nvme": resp.get("ChecksumCRC64NVME"), - } + return s3.head_object(Bucket=bucket, Key=key, ChecksumMode="ENABLED") def object_exists(s3_path: str) -> bool: @@ -194,11 +179,8 @@ def object_exists(s3_path: str) -> bool: :return: True if the object exists, False otherwise :rtype: bool """ - s3 = get_s3_client() - - (bucket, key) = split_s3_path(s3_path) try: - s3.head_object(Bucket=bucket, Key=key) + head_object(s3_path) except Exception as e: # noqa: BLE001 error_string = str(e) if not error_string.startswith("An error occurred (404) when calling the HeadObject operation: Not Found"): diff --git a/tests/integration/test_manifest_e2e.py b/tests/integration/test_manifest_e2e.py index d85cccfe..e9eb8f24 100644 --- a/tests/integration/test_manifest_e2e.py +++ b/tests/integration/test_manifest_e2e.py @@ -35,6 +35,8 @@ # assemblies, keeping FTP traffic minimal. STABLE_PREFIX = "900" +# Max number of files to transfer for partial upload +MAX_PARTIAL_FILES=2 # Helpers @@ -172,8 +174,12 @@ def test_prunes_existing_matching_md5( pytest.skip(f"No latest assemblies in prefix {STABLE_PREFIX}") # Pick one assembly to pre-seed in MinIO with correct checksums - acc = next(iter(sorted(latest))) + # and one to partially pre-seed to ensure it doesn't get pruned + i_latest = iter(sorted(latest)) + acc = next(i_latest) + acc_part = next(i_latest) rec = latest[acc] + rec_part = latest[acc_part] ftp_dir = rec.ftp_path.replace("https://ftp.ncbi.nlm.nih.gov", "") # Fetch the real md5checksums.txt from FTP @@ -186,9 +192,20 @@ def test_prunes_existing_matching_md5( checksums = parse_md5_checksums_file(md5_text) # Seed MinIO with dummy files that have the right MD5 metadata - rel = build_accession_path(rec.assembly_dir) s3 = minio_s3_client path_prefix = DEFAULT_LAKEHOUSE_KEY_PREFIX + rel = build_accession_path(rec.assembly_dir) + for fname, md5 in checksums.items(): + if any(fname.endswith(suffix) for suffix in FILE_FILTERS): + key = f"{path_prefix}{rel}{fname}" + s3.put_object( + Bucket=test_bucket, + Key=key, + Body=b"placeholder", + Metadata={"md5": md5}, + ) + rel = build_accession_path(rec_part.assembly_dir) + file_count = 0 for fname, md5 in checksums.items(): if any(fname.endswith(suffix) for suffix in FILE_FILTERS): key = f"{path_prefix}{rel}{fname}" @@ -198,6 +215,9 @@ def test_prunes_existing_matching_md5( Body=b"placeholder", Metadata={"md5": md5}, ) + file_count+=1 + if file_count>MAX_PARTIAL_FILES: + break # verify_transfer_candidates should prune the seeded assembly candidates = sorted(latest.keys()) @@ -209,6 +229,7 @@ def test_prunes_existing_matching_md5( ) assert acc not in result, f"Expected {acc} to be pruned (MD5 matches)" + assert acc_part in result, f"Expected {acc_part} to not be pruned (missing files)" # Other candidates without seeded data should remain remaining_candidates = [c for c in candidates if c != acc] for c in remaining_candidates: diff --git a/tests/ncbi_ftp/conftest.py b/tests/ncbi_ftp/conftest.py index e9d8cdaf..97f5e035 100644 --- a/tests/ncbi_ftp/conftest.py +++ b/tests/ncbi_ftp/conftest.py @@ -46,8 +46,16 @@ @pytest.fixture -def mock_s3_client() -> Generator[botocore.client.BaseClient]: +def mock_s3_client(monkeypatch: pytest.MonkeyPatch) -> Generator[botocore.client.BaseClient]: """Yield a mocked S3 client with the CDM Lake bucket created.""" + # Remove any real endpoint/credential env vars so moto intercepts all HTTP calls. + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + monkeypatch.delenv("AWS_ENDPOINT_URL", raising=False) + monkeypatch.delenv("AWS_ENDPOINT_URL_S3", raising=False) + boto3.DEFAULT_SESSION = None + with mock_aws(): client = boto3.client("s3", region_name=AWS_REGION) client.create_bucket(Bucket=TEST_BUCKET) diff --git a/tests/ncbi_ftp/test_manifest.py b/tests/ncbi_ftp/test_manifest.py index ffdd69c7..2aff5232 100644 --- a/tests/ncbi_ftp/test_manifest.py +++ b/tests/ncbi_ftp/test_manifest.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch import pytest +from botocore.exceptions import ClientError from cdm_data_loaders.ncbi_ftp.manifest import ( AssemblyRecord, @@ -317,11 +318,11 @@ def test_verify_transfer_candidates_prunes_when_all_match( def head_side_effect(s3_path: str) -> dict | None: if "_genomic.fna.gz" in s3_path: - return {"size": 100, "metadata": {"md5": "d41d8cd98f00b204e9800998ecf8427e"}, "checksum_crc64nvme": None} + return {"ContentLength": 100, "Metadata": {"md5": "d41d8cd98f00b204e9800998ecf8427e"}, "ChecksumCRC64NVME": None} if "_protein.faa.gz" in s3_path: - return {"size": 100, "metadata": {"md5": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4"}, "checksum_crc64nvme": None} + return {"ContentLength": 100, "Metadata": {"md5": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4"}, "ChecksumCRC64NVME": None} if "_assembly_report.txt" in s3_path: - return {"size": 100, "metadata": {"md5": "ffffffffffffffffffffffffffffffff"}, "checksum_crc64nvme": None} + return {"ContentLength": 100, "Metadata": {"md5": "ffffffffffffffffffffffffffffffff"}, "ChecksumCRC64NVME": None} return None mock_head.side_effect = head_side_effect @@ -340,12 +341,12 @@ def test_verify_transfer_candidates_keeps_when_md5_differs( ) -> None: """Assembly is kept when at least one file has a different MD5.""" mock_connect.return_value = MagicMock() - mock_head.return_value = {"size": 100, "metadata": {"md5": "WRONG"}, "checksum_crc64nvme": None} + mock_head.return_value = {"ContentLength": 100, "Metadata": {"md5": "WRONG"}, "ChecksumCRC64NVME": None} assert verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) == ["GCF_000001215.4"] @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) -@patch("cdm_data_loaders.ncbi_ftp.manifest.head_object", return_value=None) +@patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") def test_verify_transfer_candidates_keeps_when_s3_object_missing( @@ -356,6 +357,12 @@ def test_verify_transfer_candidates_keeps_when_s3_object_missing( ) -> None: """Assembly is kept when at least one file doesn't exist in S3.""" mock_connect.return_value = MagicMock() + + def head_side_effect(s3_path: str) -> dict[str, Any]: + if "GCF_000001215.4" in s3_path: + raise ClientError({ "Error": { "Code": "404", "Message": "Not found" } }, "HeadObject") + return {} + mock_head.side_effect = head_side_effect assert verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) == ["GCF_000001215.4"] @@ -371,7 +378,7 @@ def test_verify_transfer_candidates_keeps_when_no_md5_metadata( ) -> None: """Assembly is kept when S3 object exists but has no md5 metadata.""" mock_connect.return_value = MagicMock() - mock_head.return_value = {"size": 100, "metadata": {}, "checksum_crc64nvme": None} + mock_head.return_value = {"ContentLength": 100, "Metadata": {}, "ChecksumCRC64NVME": None} assert verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) == ["GCF_000001215.4"] @@ -401,7 +408,7 @@ def test_verify_transfer_candidates_unknown_accession_kept(mock_connect: MagicMo @patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) -@patch("cdm_data_loaders.ncbi_ftp.manifest.head_object", return_value=None) +@patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") def test_verify_transfer_candidates_short_circuits_on_first_mismatch( @@ -412,6 +419,12 @@ def test_verify_transfer_candidates_short_circuits_on_first_mismatch( ) -> None: """Verification stops checking after the first missing/mismatched file.""" mock_connect.return_value = MagicMock() + + def head_side_effects(s3_path: str) -> dict[str, Any]: + if "GCF_000001215.4" in s3_path: + raise ClientError({ "Error": { "Code": "404", "Message": "Not found" }}, "HeadObject") + return {} + mock_head.side_effect = head_side_effects verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) assert mock_head.call_count == 1 @@ -432,12 +445,12 @@ def test_verify_transfer_candidates_mixed( def head_side_effect(s3_path: str) -> dict | None: if "GCF_000001215.4_Release_6_plus_ISO1_MT/" in s3_path: if "_genomic.fna.gz" in s3_path: - return {"size": 1, "metadata": {"md5": "d41d8cd98f00b204e9800998ecf8427e"}, "checksum_crc64nvme": None} + return {"ContentLength": 1, "Metadata": {"md5": "d41d8cd98f00b204e9800998ecf8427e"}, "ChecksumCRC64NVME": None} if "_protein.faa.gz" in s3_path: - return {"size": 1, "metadata": {"md5": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4"}, "checksum_crc64nvme": None} + return {"ContentLength": 1, "Metadata": {"md5": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4"}, "ChecksumCRC64NVME": None} if "_assembly_report.txt" in s3_path: - return {"size": 1, "metadata": {"md5": "ffffffffffffffffffffffffffffffff"}, "checksum_crc64nvme": None} - return None + return {"ContentLength": 1, "Metadata": {"md5": "ffffffffffffffffffffffffffffffff"}, "ChecksumCRC64NVME": None} + raise ClientError({ "Error": { "Code": "404", "Message": "Not found" }}, "HeadObject") mock_head.side_effect = head_side_effect result = verify_transfer_candidates(["GCF_000001215.4", "GCF_000001405.40"], _assemblies(), _BUCKET, _KEY_PREFIX) diff --git a/tests/ncbi_ftp/test_metadata.py b/tests/ncbi_ftp/test_metadata.py index d9b86c4f..cbb58f13 100644 --- a/tests/ncbi_ftp/test_metadata.py +++ b/tests/ncbi_ftp/test_metadata.py @@ -190,8 +190,15 @@ def test_validate_descriptor_empty_raises() -> None: @pytest.fixture -def mock_s3() -> Generator[botocore.client.BaseClient]: +def mock_s3(monkeypatch: pytest.MonkeyPatch) -> Generator[botocore.client.BaseClient]: """Mocked S3 client with the CDM Lake bucket pre-created.""" + # Remove any real endpoint/credential env vars so moto intercepts all HTTP calls. + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + monkeypatch.delenv("AWS_ENDPOINT_URL", raising=False) + monkeypatch.delenv("AWS_ENDPOINT_URL_S3", raising=False) + boto3.DEFAULT_SESSION = None with mock_aws(): client = boto3.client("s3", region_name=AWS_REGION) client.create_bucket(Bucket=TEST_BUCKET) diff --git a/tests/pipelines/test_ncbi_ftp_download.py b/tests/pipelines/test_ncbi_ftp_download.py index 8e8602ee..d51aa0ef 100644 --- a/tests/pipelines/test_ncbi_ftp_download.py +++ b/tests/pipelines/test_ncbi_ftp_download.py @@ -255,8 +255,15 @@ def test_handles_download_failure(self, tmp_path: Path) -> None: _STAGING_PREFIX = "staging/run1/" -def _make_moto_s3(): +def _make_moto_s3(monkeypatch: pytest.MonkeyPatch): """Return a moto-backed S3 client with the test bucket created.""" + # Remove any real endpoint/credential env vars so moto intercepts all HTTP calls. + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + monkeypatch.delenv("AWS_ENDPOINT_URL", raising=False) + monkeypatch.delenv("AWS_ENDPOINT_URL_S3", raising=False) + boto3.DEFAULT_SESSION = None client = boto3.client("s3", region_name="us-east-1") client.create_bucket(Bucket=_TEST_BUCKET) return client @@ -277,10 +284,11 @@ def test_download_and_stage_manifest_source( tmp_path: Path, manifest_s3_key: str | None, use_local: bool, + monkeypatch: pytest.MonkeyPatch, ) -> None: """Assembly paths from the manifest are processed regardless of source (S3 or local).""" reset_s3_client() - s3 = _make_moto_s3() + s3 = _make_moto_s3(monkeypatch) manifest_local: Path | None = None if manifest_s3_key is not None: @@ -340,6 +348,7 @@ def test_download_and_stage_exactly_one_source_required( s3_key: str | None, local_path: str | None, should_raise: bool, + monkeypatch: pytest.MonkeyPatch, ) -> None: """ValueError is raised when both or neither manifest sources are given.""" reset_s3_client() @@ -353,7 +362,7 @@ def test_download_and_stage_exactly_one_source_required( manifest_local_path=local_path, ) else: - s3 = _make_moto_s3() + s3 = _make_moto_s3(monkeypatch) # For s3_only: seed the object; for local_only: create the file if s3_key is not None: s3.put_object(Bucket=_TEST_BUCKET, Key=s3_key, Body=_MANIFEST_CONTENT.encode()) @@ -390,10 +399,10 @@ def test_download_and_stage_exactly_one_source_required( @mock_aws -def test_download_and_stage_uploads_to_staging(tmp_path: Path) -> None: +def test_download_and_stage_uploads_to_staging(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Files produced by download_assembly_to_local and download_report.json are all staged to S3.""" reset_s3_client() - s3 = _make_moto_s3() + s3 = _make_moto_s3(monkeypatch) manifest_local = tmp_path / "manifest.txt" # Single assembly so the fake download writes exactly the files we expect @@ -446,10 +455,10 @@ def _fake_download(path, output_dir, **kwargs): # noqa: ARG001 @mock_aws -def test_download_and_stage_dry_run_skips_upload(tmp_path: Path) -> None: +def test_download_and_stage_dry_run_skips_upload(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """dry_run=True leaves S3 empty and returns staged_objects=0.""" reset_s3_client() - s3 = _make_moto_s3() + s3 = _make_moto_s3(monkeypatch) manifest_local = tmp_path / "manifest.txt" manifest_local.write_text(_MANIFEST_CONTENT) @@ -499,10 +508,10 @@ def _fake_download(path, output_dir, **kwargs): # noqa: ARG001 ], ) @mock_aws -def test_download_and_stage_limit_forwarded(tmp_path: Path, limit: int) -> None: +def test_download_and_stage_limit_forwarded(tmp_path: Path, limit: int, monkeypatch: pytest.MonkeyPatch) -> None: """The limit parameter truncates the number of assemblies processed.""" reset_s3_client() - s3 = _make_moto_s3() + s3 = _make_moto_s3(monkeypatch) manifest_local = tmp_path / "manifest.txt" manifest_local.write_text(_MANIFEST_CONTENT) @@ -538,10 +547,10 @@ def test_download_and_stage_limit_forwarded(tmp_path: Path, limit: int) -> None: @mock_aws -def test_download_and_stage_report_shape(tmp_path: Path) -> None: +def test_download_and_stage_report_shape(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Return value contains all expected keys including staged_objects, staging_key_prefix, dry_run.""" reset_s3_client() - s3 = _make_moto_s3() + s3 = _make_moto_s3(monkeypatch) manifest_local = tmp_path / "manifest.txt" manifest_local.write_text(_MANIFEST_CONTENT) diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index a9b4ec8e..d9191b2f 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -60,14 +60,23 @@ @pytest.fixture -def mock_s3_client() -> Generator[Any, Any]: +def mock_s3_client(monkeypatch: pytest.MonkeyPatch) -> Generator[Any, Any]: """Yield a mocked S3 client with both valid buckets created. The function get_s3_client() is patched to ensure that all module functions use this client. Resets the cached client before and after to prevent state leaking between tests. """ + # Remove any real endpoint/credential env vars so moto intercepts all HTTP calls. + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "testing") + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + monkeypatch.delenv("AWS_ENDPOINT_URL", raising=False) + monkeypatch.delenv("AWS_ENDPOINT_URL_S3", raising=False) + boto3.DEFAULT_SESSION = None + with mock_aws(): + reset_s3_client() client = boto3.client("s3") for bucket in FILES_IN_BUCKETS: client.create_bucket(Bucket=bucket) @@ -451,12 +460,13 @@ def test_head_object_and_object_exists_true_and_false(mock_s3_client: Any, proto for f in file_list: output = head_object(f"{protocol}{bucket}/{f}") assert output is not None - assert isinstance(output["size"], int) + assert isinstance(output["ContentLength"], int) assert object_exists(f"{protocol}{bucket}/{f}") is True nonexistent_file = f"{protocol}{bucket}/a-file-i-just-made-up.txt" assert object_exists(nonexistent_file) is False - assert head_object(nonexistent_file) is None + with pytest.raises(ClientError, match="404"): + _ = head_object(nonexistent_file) @pytest.mark.parametrize("s3_path", ["absent", "dir_one", "dir_one/", "dir_one/file1.tnt"]) @@ -1049,18 +1059,16 @@ def test_head_object_returns_info(mock_s3_client: Any) -> None: mock_s3_client.put_object(Bucket=CDM_LAKE_BUCKET, Key="info/file.txt", Body=b"hello", Metadata={"md5": "abc123"}) result = head_object(f"{CDM_LAKE_BUCKET}/info/file.txt") assert result is not None - assert result["size"] == SIZE_HELLO - assert result["metadata"]["md5"] == "abc123" - # moto may not populate CRC64NVME, but the key should be present - assert "checksum_crc64nvme" in result + assert result["ContentLength"] == SIZE_HELLO + assert result["Metadata"]["md5"] == "abc123" @pytest.mark.s3 @pytest.mark.usefixtures("mock_s3_client") -def test_head_object_returns_none_for_missing() -> None: +def test_head_object_raises_for_missing() -> None: """Verify that head_object returns None for a non-existent object.""" - result = head_object(f"{CDM_LAKE_BUCKET}/does/not/exist.txt") - assert result is None + with pytest.raises(ClientError, match="404"): + _ = head_object(f"{CDM_LAKE_BUCKET}/does/not/exist.txt") @pytest.mark.parametrize("protocol", ["", "s3://", "s3a://"]) @@ -1070,7 +1078,7 @@ def test_head_object_with_protocols(mock_s3_client: Any, protocol: str) -> None: mock_s3_client.put_object(Bucket=CDM_LAKE_BUCKET, Key="proto/file.txt", Body=b"data") result = head_object(f"{protocol}{CDM_LAKE_BUCKET}/proto/file.txt") assert result is not None - assert result["size"] == SIZE_DATA + assert result["ContentLength"] == SIZE_DATA # copy_object From 4dea37b3dd0e36b6f3ba687bc87e4a2c7f2556e5 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 1 Jun 2026 15:17:43 -0700 Subject: [PATCH 098/128] formatting --- src/cdm_data_loaders/ncbi_ftp/manifest.py | 4 +-- tests/integration/test_manifest_e2e.py | 6 ++-- tests/ncbi_ftp/test_manifest.py | 44 ++++++++++++++++++----- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/manifest.py b/src/cdm_data_loaders/ncbi_ftp/manifest.py index cb7cdae0..9407b886 100644 --- a/src/cdm_data_loaders/ncbi_ftp/manifest.py +++ b/src/cdm_data_loaders/ncbi_ftp/manifest.py @@ -410,9 +410,9 @@ def _does_accession_need_update( s3_md5 = "" try: obj_info = head_object(s3_path) - s3_md5 = obj_info.get("Metadata",{}).get("md5", "") + s3_md5 = obj_info.get("Metadata", {}).get("md5", "") except ClientError as e: - if e.response["Error"]["Code"] == "404": # type: ignore[union-attr] + if e.response["Error"]["Code"] == "404": # type: ignore[union-attr] logger.debug("File missing from store: %s", s3_path) return True raise diff --git a/tests/integration/test_manifest_e2e.py b/tests/integration/test_manifest_e2e.py index e9eb8f24..6d3ef5dc 100644 --- a/tests/integration/test_manifest_e2e.py +++ b/tests/integration/test_manifest_e2e.py @@ -36,7 +36,7 @@ STABLE_PREFIX = "900" # Max number of files to transfer for partial upload -MAX_PARTIAL_FILES=2 +MAX_PARTIAL_FILES = 2 # Helpers @@ -215,8 +215,8 @@ def test_prunes_existing_matching_md5( Body=b"placeholder", Metadata={"md5": md5}, ) - file_count+=1 - if file_count>MAX_PARTIAL_FILES: + file_count += 1 + if file_count > MAX_PARTIAL_FILES: break # verify_transfer_candidates should prune the seeded assembly diff --git a/tests/ncbi_ftp/test_manifest.py b/tests/ncbi_ftp/test_manifest.py index 2aff5232..6f709b4f 100644 --- a/tests/ncbi_ftp/test_manifest.py +++ b/tests/ncbi_ftp/test_manifest.py @@ -318,11 +318,23 @@ def test_verify_transfer_candidates_prunes_when_all_match( def head_side_effect(s3_path: str) -> dict | None: if "_genomic.fna.gz" in s3_path: - return {"ContentLength": 100, "Metadata": {"md5": "d41d8cd98f00b204e9800998ecf8427e"}, "ChecksumCRC64NVME": None} + return { + "ContentLength": 100, + "Metadata": {"md5": "d41d8cd98f00b204e9800998ecf8427e"}, + "ChecksumCRC64NVME": None, + } if "_protein.faa.gz" in s3_path: - return {"ContentLength": 100, "Metadata": {"md5": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4"}, "ChecksumCRC64NVME": None} + return { + "ContentLength": 100, + "Metadata": {"md5": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4"}, + "ChecksumCRC64NVME": None, + } if "_assembly_report.txt" in s3_path: - return {"ContentLength": 100, "Metadata": {"md5": "ffffffffffffffffffffffffffffffff"}, "ChecksumCRC64NVME": None} + return { + "ContentLength": 100, + "Metadata": {"md5": "ffffffffffffffffffffffffffffffff"}, + "ChecksumCRC64NVME": None, + } return None mock_head.side_effect = head_side_effect @@ -360,8 +372,9 @@ def test_verify_transfer_candidates_keeps_when_s3_object_missing( def head_side_effect(s3_path: str) -> dict[str, Any]: if "GCF_000001215.4" in s3_path: - raise ClientError({ "Error": { "Code": "404", "Message": "Not found" } }, "HeadObject") + raise ClientError({"Error": {"Code": "404", "Message": "Not found"}}, "HeadObject") return {} + mock_head.side_effect = head_side_effect assert verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) == ["GCF_000001215.4"] @@ -422,8 +435,9 @@ def test_verify_transfer_candidates_short_circuits_on_first_mismatch( def head_side_effects(s3_path: str) -> dict[str, Any]: if "GCF_000001215.4" in s3_path: - raise ClientError({ "Error": { "Code": "404", "Message": "Not found" }}, "HeadObject") + raise ClientError({"Error": {"Code": "404", "Message": "Not found"}}, "HeadObject") return {} + mock_head.side_effect = head_side_effects verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) assert mock_head.call_count == 1 @@ -445,12 +459,24 @@ def test_verify_transfer_candidates_mixed( def head_side_effect(s3_path: str) -> dict | None: if "GCF_000001215.4_Release_6_plus_ISO1_MT/" in s3_path: if "_genomic.fna.gz" in s3_path: - return {"ContentLength": 1, "Metadata": {"md5": "d41d8cd98f00b204e9800998ecf8427e"}, "ChecksumCRC64NVME": None} + return { + "ContentLength": 1, + "Metadata": {"md5": "d41d8cd98f00b204e9800998ecf8427e"}, + "ChecksumCRC64NVME": None, + } if "_protein.faa.gz" in s3_path: - return {"ContentLength": 1, "Metadata": {"md5": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4"}, "ChecksumCRC64NVME": None} + return { + "ContentLength": 1, + "Metadata": {"md5": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4"}, + "ChecksumCRC64NVME": None, + } if "_assembly_report.txt" in s3_path: - return {"ContentLength": 1, "Metadata": {"md5": "ffffffffffffffffffffffffffffffff"}, "ChecksumCRC64NVME": None} - raise ClientError({ "Error": { "Code": "404", "Message": "Not found" }}, "HeadObject") + return { + "ContentLength": 1, + "Metadata": {"md5": "ffffffffffffffffffffffffffffffff"}, + "ChecksumCRC64NVME": None, + } + raise ClientError({"Error": {"Code": "404", "Message": "Not found"}}, "HeadObject") mock_head.side_effect = head_side_effect result = verify_transfer_candidates(["GCF_000001215.4", "GCF_000001405.40"], _assemblies(), _BUCKET, _KEY_PREFIX) From 1da08e489468169194d0a2e97be557da06d70451 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 1 Jun 2026 15:58:50 -0700 Subject: [PATCH 099/128] address review comments --- src/cdm_data_loaders/ncbi_ftp/manifest.py | 12 ++---- src/cdm_data_loaders/utils/s3.py | 6 ++- tests/ncbi_ftp/test_manifest.py | 50 ++++++++--------------- 3 files changed, 26 insertions(+), 42 deletions(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/manifest.py b/src/cdm_data_loaders/ncbi_ftp/manifest.py index 9407b886..4e355b82 100644 --- a/src/cdm_data_loaders/ncbi_ftp/manifest.py +++ b/src/cdm_data_loaders/ncbi_ftp/manifest.py @@ -225,13 +225,9 @@ def compute_diff( known = set(previous_assemblies) if previous_assemblies is not None else (previous_accessions or set()) for acc, rec in current.items(): - if rec.status == "replaced": + if rec.status in ("replaced", "suppressed"): if acc in known: - diff.replaced.append(acc) - continue - if rec.status == "suppressed": - if acc in known: - diff.suppressed.append(acc) + getattr(diff, rec.status).append(acc) continue if rec.status != "latest": continue @@ -479,8 +475,8 @@ def _progress(done: int, total: int, acc: str) -> None: s3_prefix = f"{key_prefix}{s3_rel}" # Quick check: does *anything* exist under this prefix? - resp = s3.list_objects_v2(Bucket=bucket, Prefix=s3_prefix, MaxKeys=1) - if resp.get("KeyCount", 0) == 0: + resp = list_matching_objects(f"{bucket}/{s3_prefix}", max_keys=1) + if not resp: # Nothing in the store — definitely needs downloading confirmed.append(acc) skipped_missing += 1 diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index 027af27c..4101b98e 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -132,7 +132,7 @@ def split_s3_path(s3_path: str) -> tuple[str | None, str]: return (path_parts[0], path_parts[1]) -def list_matching_objects(s3_path: str) -> list[dict[str, Any]]: +def list_matching_objects(s3_path: str, *, max_keys: int = 1000) -> list[dict[str, Any]]: """List the remote paths that start with ``s3_path``. Note: since s3 paths are basically cosmetic, this function returns all paths that start with @@ -143,13 +143,15 @@ def list_matching_objects(s3_path: str) -> list[dict[str, Any]]: :param s3_path: directory to be listed, INCLUDING the bucket name :type s3_path: str + :param max_keys: maximum number of keys to return. boto3 defaults to 1000 records max. + :type max_keys: int :return: list of object metadata dicts in the directory :rtype: list[dict[str, Any]] """ s3 = get_s3_client() (bucket, key) = split_s3_path(s3_path) paginator = s3.get_paginator("list_objects_v2") - page_iterator = paginator.paginate(Bucket=bucket, Prefix=key) + page_iterator = paginator.paginate(Bucket=bucket, Prefix=key, MaxKeys=max_keys) contents = [] for page in page_iterator: diff --git a/tests/ncbi_ftp/test_manifest.py b/tests/ncbi_ftp/test_manifest.py index 6f709b4f..b558f162 100644 --- a/tests/ncbi_ftp/test_manifest.py +++ b/tests/ncbi_ftp/test_manifest.py @@ -281,20 +281,6 @@ def test_ftp_dir_from_url(url: str, expected: str, kwargs: dict) -> None: ) -def _mock_s3_with_objects() -> MagicMock: - """Return a mock S3 client whose list_objects_v2 always reports objects exist.""" - client = MagicMock() - client.list_objects_v2.return_value = {"KeyCount": 1} - return client - - -def _mock_s3_empty() -> MagicMock: - """Return a mock S3 client whose list_objects_v2 reports no objects.""" - client = MagicMock() - client.list_objects_v2.return_value = {"KeyCount": 0} - return client - - _BUCKET = "cdm-lake" _KEY_PREFIX = "tenant-general-warehouse/kbase/datasets/ncbi/" @@ -303,7 +289,7 @@ def _assemblies() -> dict: return parse_assembly_summary(SAMPLE_SUMMARY) -@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) +@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects", return_value=["one key"]) @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") @@ -311,7 +297,7 @@ def test_verify_transfer_candidates_prunes_when_all_match( mock_connect: MagicMock, mock_retrieve: MagicMock, mock_head: MagicMock, - mock_s3: MagicMock, + mock_list: MagicMock, ) -> None: """Assemblies where every file matches S3 are pruned from the list.""" mock_connect.return_value = MagicMock() @@ -341,7 +327,7 @@ def head_side_effect(s3_path: str) -> dict | None: assert verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) == [] -@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) +@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects", return_value=["one key"]) @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") @@ -349,7 +335,7 @@ def test_verify_transfer_candidates_keeps_when_md5_differs( mock_connect: MagicMock, mock_retrieve: MagicMock, mock_head: MagicMock, - mock_s3: MagicMock, + mock_list: MagicMock, ) -> None: """Assembly is kept when at least one file has a different MD5.""" mock_connect.return_value = MagicMock() @@ -357,7 +343,7 @@ def test_verify_transfer_candidates_keeps_when_md5_differs( assert verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) == ["GCF_000001215.4"] -@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) +@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects", return_value=["one key"]) @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") @@ -365,7 +351,7 @@ def test_verify_transfer_candidates_keeps_when_s3_object_missing( mock_connect: MagicMock, mock_retrieve: MagicMock, mock_head: MagicMock, - mock_s3: MagicMock, + mock_list: MagicMock, ) -> None: """Assembly is kept when at least one file doesn't exist in S3.""" mock_connect.return_value = MagicMock() @@ -379,7 +365,7 @@ def head_side_effect(s3_path: str) -> dict[str, Any]: assert verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) == ["GCF_000001215.4"] -@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) +@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects", return_value=["one key"]) @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") @@ -387,7 +373,7 @@ def test_verify_transfer_candidates_keeps_when_no_md5_metadata( mock_connect: MagicMock, mock_retrieve: MagicMock, mock_head: MagicMock, - mock_s3: MagicMock, + mock_list: MagicMock, ) -> None: """Assembly is kept when S3 object exists but has no md5 metadata.""" mock_connect.return_value = MagicMock() @@ -395,11 +381,11 @@ def test_verify_transfer_candidates_keeps_when_no_md5_metadata( assert verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) == ["GCF_000001215.4"] -@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) +@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects", return_value=["one key"]) @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", side_effect=Exception("FTP error")) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") def test_verify_transfer_candidates_keeps_when_ftp_fails( - mock_connect: MagicMock, mock_retrieve: MagicMock, mock_s3: MagicMock + mock_connect: MagicMock, mock_retrieve: MagicMock, mock_list: MagicMock ) -> None: """Assembly is kept (conservative) when md5checksums.txt cannot be fetched.""" mock_connect.return_value = MagicMock() @@ -412,15 +398,15 @@ def test_verify_transfer_candidates_empty_input(mock_connect: MagicMock) -> None assert verify_transfer_candidates([], {}, _BUCKET, "prefix/") == [] -@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) +@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects", return_value=["one key"]) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") -def test_verify_transfer_candidates_unknown_accession_kept(mock_connect: MagicMock, mock_s3: MagicMock) -> None: +def test_verify_transfer_candidates_unknown_accession_kept(mock_connect: MagicMock, mock_list: MagicMock) -> None: """Accessions not in assemblies dict are kept (conservative).""" mock_connect.return_value = MagicMock() assert verify_transfer_candidates(["GCF_999999999.1"], {}, _BUCKET, "prefix/") == ["GCF_999999999.1"] -@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) +@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects", return_value=["one key"]) @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") @@ -428,7 +414,7 @@ def test_verify_transfer_candidates_short_circuits_on_first_mismatch( mock_connect: MagicMock, mock_retrieve: MagicMock, mock_head: MagicMock, - mock_s3: MagicMock, + mock_list: MagicMock, ) -> None: """Verification stops checking after the first missing/mismatched file.""" mock_connect.return_value = MagicMock() @@ -443,7 +429,7 @@ def head_side_effects(s3_path: str) -> dict[str, Any]: assert mock_head.call_count == 1 -@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_with_objects()) +@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects", return_value=["one key"]) @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") @patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") @@ -451,7 +437,7 @@ def test_verify_transfer_candidates_mixed( mock_connect: MagicMock, mock_retrieve: MagicMock, mock_head: MagicMock, - mock_s3: MagicMock, + mock_list: MagicMock, ) -> None: """A mix of matching and non-matching assemblies: matched pruned, unmatched kept.""" mock_connect.return_value = MagicMock() @@ -483,11 +469,11 @@ def head_side_effect(s3_path: str) -> dict | None: assert result == ["GCF_000001405.40"] -@patch("cdm_data_loaders.ncbi_ftp.manifest.get_s3_client", return_value=_mock_s3_empty()) +@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects", return_value=[]) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") def test_verify_transfer_candidates_skips_ftp_when_folder_missing( mock_connect: MagicMock, - mock_s3: MagicMock, + mock_list: MagicMock, ) -> None: """Accessions with no objects in S3 are confirmed without FTP round-trip.""" result = verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) From ffe2c6cf24c278465ff874ad96bf3913bded4b88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Jun 2026 16:14:32 +0000 Subject: [PATCH 100/128] Update moto[s3] requirement from >=5.1.22 to >=5.2.1 Updates the requirements on [moto[s3]](https://github.com/getmoto/moto) to permit the latest version. - [Release notes](https://github.com/getmoto/moto/releases) - [Changelog](https://github.com/getmoto/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/getmoto/moto/compare/5.1.22...5.2.1) --- updated-dependencies: - dependency-name: moto[s3] dependency-version: 5.2.1 dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- uv.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index da58937d..f65c5bea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ uniref = "cdm_data_loaders.pipelines.uniref:cli" [dependency-groups] dev = [ "berdl-notebook-utils>=0.0.1", - "moto[s3]>=5.1.22", + "moto[s3]>=5.2.1", "pytest>=9.0.2", "pytest-asyncio>=1.3.0", "pytest-cov>=7.1.0", diff --git a/uv.lock b/uv.lock index 08956bc1..58a4cd90 100644 --- a/uv.lock +++ b/uv.lock @@ -485,7 +485,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "berdl-notebook-utils", git = "https://github.com/BERDataLakehouse/spark_notebook.git?subdirectory=notebook_utils" }, - { name = "moto", extras = ["s3"], specifier = ">=5.1.22" }, + { name = "moto", extras = ["s3"], specifier = ">=5.2.1" }, { name = "pytest", specifier = ">=9.0.2" }, { name = "pytest-asyncio", specifier = ">=1.3.0" }, { name = "pytest-cov", specifier = ">=7.1.0" }, @@ -2599,7 +2599,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess" }, + { name = "ptyprocess", marker = "(platform_python_implementation != 'PyPy' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ From d360ddf5057cde522d0f4bf510ba2a27ff3ba6c2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Jun 2026 16:14:46 +0000 Subject: [PATCH 101/128] Update uv-build requirement from <0.20.0,>=0.11.8 to >=0.11.18,<0.20.0 Updates the requirements on [uv-build](https://github.com/astral-sh/uv) to permit the latest version. - [Release notes](https://github.com/astral-sh/uv/releases) - [Changelog](https://github.com/astral-sh/uv/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/uv/compare/0.11.8...0.11.18) --- updated-dependencies: - dependency-name: uv-build dependency-version: 0.11.18 dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index da58937d..4b9e130d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -180,7 +180,7 @@ max-complexity = 15 convention = "google" [build-system] -requires = ["uv_build>=0.11.8,<0.20.0"] +requires = ["uv_build>=0.11.18,<0.20.0"] build-backend = "uv_build" [tool.pytest] From 586a7dd18a863c1aa0c7dadeeac92e316f182963 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Jun 2026 16:15:37 +0000 Subject: [PATCH 102/128] Bump bioregistry from 0.13.57 to 0.13.58 Bumps [bioregistry](https://github.com/biopragmatics/bioregistry) from 0.13.57 to 0.13.58. - [Release notes](https://github.com/biopragmatics/bioregistry/releases) - [Commits](https://github.com/biopragmatics/bioregistry/compare/v0.13.57...v0.13.58) --- updated-dependencies: - dependency-name: bioregistry dependency-version: 0.13.58 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- uv.lock | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index da58937d..79ff426f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ authors = [ ] dependencies = [ - "bioregistry>=0.13.57", + "bioregistry>=0.13.58", "boto3[crt]>=1.42.55", "click>=8.4.1", "defusedxml>=0.7.1", diff --git a/uv.lock b/uv.lock index 08956bc1..0b294194 100644 --- a/uv.lock +++ b/uv.lock @@ -361,21 +361,24 @@ dependencies = [ [[package]] name = "bioregistry" -version = "0.13.57" +version = "0.13.58" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "curies" }, + { name = "idna" }, { name = "more-click" }, { name = "pydantic", extra = ["email"] }, { name = "pystow" }, + { name = "python-multipart" }, { name = "requests" }, { name = "sssom-pydantic" }, { name = "tqdm" }, + { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/eb/b4/1843ebbe998ec88b19b2128e4e1ea51976d12949066e5b9260ca1dbe5ea5/bioregistry-0.13.57.tar.gz", hash = "sha256:03d504adb861b858fbfcfe3a6e83057fa10a6f5992e46958f971b53a3fe812ea", size = 6048550, upload-time = "2026-05-23T04:33:55.516Z" } +sdist = { url = "https://files.pythonhosted.org/packages/44/5e/22e9ffc92d81d2325b2134723e971671ea09e1467b07e76d557c45e6c6e6/bioregistry-0.13.58.tar.gz", hash = "sha256:06e6f1188988502cf117d4476107e03f0b1460cc31fdaf7f470493ccf5f82361", size = 6050994, upload-time = "2026-05-30T04:43:14.693Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b8/b5/20c3ccde3f3432851f8470d4f95271a018ac73e05e563cfbce63d0527f40/bioregistry-0.13.57-py3-none-any.whl", hash = "sha256:59a0341b666fd960e7508792e5cffb128de3677202c1dbdf17c12051fbe145ef", size = 6139134, upload-time = "2026-05-23T04:33:52.303Z" }, + { url = "https://files.pythonhosted.org/packages/4b/9b/b9ce93afd4abb2c8a55c1f4204f4043617992f26248ef89ef4d8fe811648/bioregistry-0.13.58-py3-none-any.whl", hash = "sha256:2272f23aa2bc31b384a52613b890de7310f89d3b0a3bec83490395babc6655aa", size = 6141702, upload-time = "2026-05-30T04:43:11.605Z" }, ] [[package]] @@ -467,7 +470,7 @@ xml = [ [package.metadata] requires-dist = [ - { name = "bioregistry", specifier = ">=0.13.57" }, + { name = "bioregistry", specifier = ">=0.13.58" }, { name = "boto3", extras = ["crt"], specifier = ">=1.42.55" }, { name = "click", specifier = ">=8.4.1" }, { name = "defusedxml", specifier = ">=0.7.1" }, @@ -2599,7 +2602,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess" }, + { name = "ptyprocess", marker = "(platform_python_implementation != 'PyPy' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ From 2990f7dd185a7fa119a2894b0a30ad7079b5a64b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Jun 2026 16:15:49 +0000 Subject: [PATCH 103/128] Update dlt[deltalake,duckdb,filesystem,parquet] requirement Updates the requirements on [dlt[deltalake,duckdb,filesystem,parquet]](https://github.com/dlt-hub/dlt) to permit the latest version. - [Release notes](https://github.com/dlt-hub/dlt/releases) - [Commits](https://github.com/dlt-hub/dlt/compare/1.27.0...1.27.2) --- updated-dependencies: - dependency-name: dlt[deltalake,duckdb,filesystem,parquet] dependency-version: 1.27.2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- uv.lock | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index da58937d..59fecaf3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ "click>=8.4.1", "defusedxml>=0.7.1", "delta-spark>=4.1.0", - "dlt[deltalake,duckdb,filesystem,parquet]>=1.27.0", + "dlt[deltalake,duckdb,filesystem,parquet]>=1.27.2", "frictionless[aws]>=5.19.0", "frozendict>=2.4.7", "ipykernel>=7.2.0", diff --git a/uv.lock b/uv.lock index 08956bc1..3981cb34 100644 --- a/uv.lock +++ b/uv.lock @@ -472,7 +472,7 @@ requires-dist = [ { name = "click", specifier = ">=8.4.1" }, { name = "defusedxml", specifier = ">=0.7.1" }, { name = "delta-spark", specifier = ">=4.1.0" }, - { name = "dlt", extras = ["deltalake", "duckdb", "filesystem", "parquet"], specifier = ">=1.27.0" }, + { name = "dlt", extras = ["deltalake", "duckdb", "filesystem", "parquet"], specifier = ">=1.27.2" }, { name = "frictionless", extras = ["aws"], specifier = ">=5.19.0" }, { name = "frozendict", specifier = ">=2.4.7" }, { name = "ipykernel", specifier = ">=7.2.0" }, @@ -919,7 +919,7 @@ wheels = [ [[package]] name = "dlt" -version = "1.27.1" +version = "1.27.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -948,9 +948,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "tzdata" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6f/df/79ef0330010c40ad94db093846df5c3a67b07eae8b991bd91e3ad1ea9c50/dlt-1.27.1.tar.gz", hash = "sha256:b4cee83b07b417a27450eea5b9d16f1e6042374635673cee39ad858d406a33e1", size = 1070230, upload-time = "2026-05-27T20:17:04.823Z" } +sdist = { url = "https://files.pythonhosted.org/packages/da/40/5aab6d1ff64226df39a3e241e31452907e32793e18f87ec99c956ce0ddd0/dlt-1.27.2.tar.gz", hash = "sha256:e9cf70883df0d04a1d40e1dea95d34dee29de978a9500d1d434eae881b5d0279", size = 1070289, upload-time = "2026-05-29T16:33:47.725Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d8/3b/1424f5cf91947af89ae9ff1bb14a8cd2c08d1c0de2161c4c916f264a6bac/dlt-1.27.1-py3-none-any.whl", hash = "sha256:b3af5996f742b05537e755664e6343faf58fc1349f4ebf3d4fc06632dfff4905", size = 1347625, upload-time = "2026-05-27T20:17:08.144Z" }, + { url = "https://files.pythonhosted.org/packages/e8/69/d30f79f9fc1f08268c71c4a17f150429f4d00833f18f51059250c02ee03b/dlt-1.27.2-py3-none-any.whl", hash = "sha256:1c54d394864ca2cf1b042a28dafe393f24afbd60653edffc20dc6b77066cab2a", size = 1347718, upload-time = "2026-05-29T16:33:43.835Z" }, ] [package.optional-dependencies] @@ -2599,7 +2599,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess" }, + { name = "ptyprocess", marker = "(platform_python_implementation != 'PyPy' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ From c14ca4d7563cdaa3600f6a926939a3cffd044de0 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Tue, 2 Jun 2026 10:49:11 -0700 Subject: [PATCH 104/128] consolidate regex --- src/cdm_data_loaders/ncbi_ftp/assembly.py | 19 +++--------------- src/cdm_data_loaders/ncbi_ftp/constants.py | 16 +++++++++++++++ src/cdm_data_loaders/ncbi_ftp/manifest.py | 19 +++++------------- src/cdm_data_loaders/ncbi_ftp/promote.py | 23 ++++++---------------- 4 files changed, 30 insertions(+), 47 deletions(-) create mode 100644 src/cdm_data_loaders/ncbi_ftp/constants.py diff --git a/src/cdm_data_loaders/ncbi_ftp/assembly.py b/src/cdm_data_loaders/ncbi_ftp/assembly.py index 827ebb31..bfd387a4 100644 --- a/src/cdm_data_loaders/ncbi_ftp/assembly.py +++ b/src/cdm_data_loaders/ncbi_ftp/assembly.py @@ -6,20 +6,18 @@ """ import contextlib -import re import time from ftplib import FTP from pathlib import Path from typing import Any +from cdm_data_loaders.ncbi_ftp.constants import ACCESSION_PARTS_REGEX, ASSEMBLY_PATH_REGEX, FTP_HOST from cdm_data_loaders.utils.cdm_logger import get_cdm_logger from cdm_data_loaders.utils.checksums import compute_md5 from cdm_data_loaders.utils.ftp_client import connect_ftp, ftp_noop_keepalive, ftp_retrieve_text logger = get_cdm_logger() -FTP_HOST = "ftp.ncbi.nlm.nih.gov" - FILE_FILTERS = [ "_gene_ontology.gaf.gz", "_genomic.fna.gz", @@ -33,17 +31,6 @@ "_normalized_gene_expression_counts.txt.gz", ] -# Pre-compile regex patterns for performance - -# Extracts database (GCF or GCA) and 3-digit prefixes from an assembly directory name -# (e.g. "GCF_000001215.4" → ("GCF", "000", "001", "215")) -ASSEMBLY_DIR_REGEX = re.compile(r"(GC[AF])_(\d{3})(\d{3})(\d{3})\.\d+.*") - -# Extracts database, full assembly directory, and accession from an FTP path -# (e.g. "/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/" -# → ("GCF", "GCF_000001215.4_Release_6_plus_ISO1_MT", "GCF_000001215.4")) -ASSEMBLY_PATH_REGEX = re.compile(r"/(GC[AF])/\d{3}/\d{3}/\d{3}/((GC[AF]_\d{9}\.\d+)_[^/]+)/?$") - def parse_md5_checksums_file(text: str) -> dict[str, str]: """Parse an NCBI ``md5checksums.txt`` file into a filename-to-hash mapping. @@ -74,7 +61,7 @@ def build_accession_path(assembly_dir: str) -> str: :return: relative path string :raises ValueError: if the assembly directory name cannot be parsed """ - m = ASSEMBLY_DIR_REGEX.match(assembly_dir) + m = ACCESSION_PARTS_REGEX.match(assembly_dir) if not m: msg = f"Cannot parse accession: {assembly_dir}" raise ValueError(msg) @@ -90,7 +77,7 @@ def parse_assembly_path(assembly_path: str) -> tuple[str, str, str]: :raises ValueError: if the path cannot be parsed """ if m := ASSEMBLY_PATH_REGEX.search(assembly_path.rstrip("/")): - return m.group(1), m.group(2), m.group(3) + return m.group(3), m.group(1), m.group(2) msg = f"Cannot parse assembly path: {assembly_path}" raise ValueError(msg) diff --git a/src/cdm_data_loaders/ncbi_ftp/constants.py b/src/cdm_data_loaders/ncbi_ftp/constants.py new file mode 100644 index 00000000..a4c829b1 --- /dev/null +++ b/src/cdm_data_loaders/ncbi_ftp/constants.py @@ -0,0 +1,16 @@ +"""Constants, globals, and utility functions for NCBI FTP modules.""" + +import re + +FTP_HOST = "ftp.ncbi.nlm.nih.gov" + +# Pre-compiled regex patterns + +# Extracts the database (GCF or GCA) and 3-digit parts from an accession ID +# (e.g., "GCF_000001215.4" -> ("GCF, "000", "001", "215")) +ACCESSION_PARTS_REGEX = re.compile(r"(GC[AF])_(\d{3})(\d{3})(\d{3})\.\d+.*") + +# Extracts the full assembly directory, accession, and database (GCA or GCF) from an FTP path +# (e.g. "/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/" +# → ("GCF_000001215.4_Release_6_plus_ISO1_MT", "GCF_000001215.4", "GCF")) +ASSEMBLY_PATH_REGEX = re.compile(r"(((GC[AF])_\d{9}\.\d+)_[^/]+)/?.*$") diff --git a/src/cdm_data_loaders/ncbi_ftp/manifest.py b/src/cdm_data_loaders/ncbi_ftp/manifest.py index 4e355b82..f391818c 100644 --- a/src/cdm_data_loaders/ncbi_ftp/manifest.py +++ b/src/cdm_data_loaders/ncbi_ftp/manifest.py @@ -26,9 +26,10 @@ build_accession_path, parse_md5_checksums_file, ) +from cdm_data_loaders.ncbi_ftp.constants import ACCESSION_PARTS_REGEX, ASSEMBLY_PATH_REGEX from cdm_data_loaders.utils.cdm_logger import get_cdm_logger from cdm_data_loaders.utils.ftp_client import FTP, connect_ftp, ftp_noop_keepalive, ftp_retrieve_text -from cdm_data_loaders.utils.s3 import get_s3_client, head_object, list_matching_objects +from cdm_data_loaders.utils.s3 import head_object, list_matching_objects logger = get_cdm_logger() @@ -42,15 +43,6 @@ "genbank": "/genomes/ASSEMBLY_REPORTS/assembly_summary_genbank.txt", } -# Pre-compile regex patterns for performance - -# Extracts the full directory name and accession id from an S3 key -# (e.g. "GCF_000001215.4_Release_6_plus_ISO1_MT" and "GCF_000001215.4" -# from "some/prefix/GCF_000001215.4_Release_6_plus_ISO1_MT/") -ACCESSION_REGEX = re.compile(r"((GC[AF]_\d{9}\.\d+)[^/]*)/") - -# Extracts the 3-digit prefix from an accession (e.g. GCF_000001215.4 → "000") -ACCESSION_PREFIX_REGEX = re.compile(r"GC[AF]_(\d{3})\d{6}\.\d+") # Data structures @@ -172,8 +164,8 @@ def get_latest_assembly_paths(assemblies: dict[str, AssemblyRecord], ftp_host: s def accession_prefix(accession: str) -> str | None: """Extract the 3-digit prefix from an accession (e.g. ``GCF_000005845.2`` → ``"000"``).""" - m = re.match(r"GC[AF]_(\d{3})\d{6}\.\d+", accession) - return m.group(1) if m else None + m = ACCESSION_PARTS_REGEX.match(accession) + return m.group(2) if m else None def filter_by_prefix_range( @@ -271,7 +263,7 @@ def _extract_accession_dir_and_id_from_s3_key(key: str) -> tuple[str | None, str e.g. "some/prefix/GCF_000001215.4_Release_6_plus_ISO1_MT/file.gz" → ("GCF_000001215.4_Release_6_plus_ISO1_MT", "GCF_000001215.4") """ - m = ACCESSION_REGEX.search(key) + m = ASSEMBLY_PATH_REGEX.search(key) return (m.group(1), m.group(2)) if m else (None, None) @@ -451,7 +443,6 @@ def verify_transfer_candidates( # noqa: PLR0913 if not accessions: return [] - s3 = get_s3_client() ftp: Any = None # lazily connected only when needed confirmed: list[str] = [] pruned = 0 diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index 71e43714..a703b93f 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -17,6 +17,7 @@ import botocore.exceptions import tqdm +from cdm_data_loaders.ncbi_ftp.constants import ACCESSION_PARTS_REGEX, ASSEMBLY_PATH_REGEX from cdm_data_loaders.ncbi_ftp.metadata import ( DescriptorResource, archive_descriptor, @@ -40,17 +41,6 @@ _MAX_DRY_RUN_LOGS = 10 -# Precompile regexes for performance - -# Extract the accession from a path (e.g., "raw_data/GCF/000/001/405/GCF_000001405.39_Some_description/file" -> "GCF_000001405.39") -_ACCESSION_REGEX = re.compile(r"(GC[AF]_\d{9}\.\d+)") - -# Extract the database, and 3-digit groups from an accession -# (e.g., "GCF_000001405.39" -> ("GCF", "000", "001", "405")) -_ACCESSION_PARTS_REGEX = re.compile(r"(GC[AF])_(\d{3})(\d{3})(\d{3})\.\d+") - -# Extract the assembly_dir from a path (e.g., "raw_data/GCF/000/001/405/GCF_000001405.39_Some_description/file" -> "GCF_000001405.39_Some_description") -_ASSEMBLY_DIR_REGEX = re.compile(r"raw_data/GC[AF]/\d+/\d+/\d+/([^/]+)/") # Promote from S3 staging prefix @@ -164,10 +154,9 @@ def _group_files_by_assembly( rel_path = staged_key[len(normalized_staging_prefix) :] if not rel_path.startswith("raw_data/"): continue - acc_match = _ACCESSION_REGEX.search(staged_key) - adir_match = re.search(r"raw_data/GC[AF]/\d+/\d+/\d+/([^/]+)/", staged_key) - if acc_match and adir_match: - assembly_files[(adir_match.group(1), acc_match.group(1))].append(staged_key) + m = ASSEMBLY_PATH_REGEX.search(staged_key) + if m: + assembly_files[(m.group(1), m.group(2))].append(staged_key) return assembly_files @@ -382,7 +371,7 @@ def _get_accession_path_prefix(accession: str, lakehouse_key_prefix: str) -> str :param lakehouse_key_prefix: S3 key prefix for the Lakehouse dataset root :return: S3 key prefix under which all files for the accession are stored, or None if the accession format is invalid """ - m = _ACCESSION_PARTS_REGEX.match(accession) + m = ACCESSION_PARTS_REGEX.match(accession) if not m: logger.warning("Invalid accession format: %s", accession) return None @@ -534,7 +523,7 @@ def _archive_assemblies( # noqa: PLR0913 # Infer assembly_dir from key paths for descriptor archival assembly_dir: str | None = None for src, _ in key_pairs: - adir_match = _ASSEMBLY_DIR_REGEX.search(src) + adir_match = ASSEMBLY_PATH_REGEX.search(src) if adir_match: assembly_dir = adir_match.group(1) break From 607a9e3ec5e6bf099b9af6a23ed6a1f832a24176 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Tue, 2 Jun 2026 11:14:41 -0700 Subject: [PATCH 105/128] address review comments --- src/cdm_data_loaders/ncbi_ftp/assembly.py | 19 ++++++++----------- src/cdm_data_loaders/ncbi_ftp/manifest.py | 10 +++++----- src/cdm_data_loaders/ncbi_ftp/promote.py | 1 - 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/assembly.py b/src/cdm_data_loaders/ncbi_ftp/assembly.py index bfd387a4..25ce7fd4 100644 --- a/src/cdm_data_loaders/ncbi_ftp/assembly.py +++ b/src/cdm_data_loaders/ncbi_ftp/assembly.py @@ -13,7 +13,7 @@ from cdm_data_loaders.ncbi_ftp.constants import ACCESSION_PARTS_REGEX, ASSEMBLY_PATH_REGEX, FTP_HOST from cdm_data_loaders.utils.cdm_logger import get_cdm_logger -from cdm_data_loaders.utils.checksums import compute_md5 +from cdm_data_loaders.utils.checksums import compute_md5, verify_md5 from cdm_data_loaders.utils.ftp_client import connect_ftp, ftp_noop_keepalive, ftp_retrieve_text logger = get_cdm_logger() @@ -61,12 +61,11 @@ def build_accession_path(assembly_dir: str) -> str: :return: relative path string :raises ValueError: if the assembly directory name cannot be parsed """ - m = ACCESSION_PARTS_REGEX.match(assembly_dir) - if not m: - msg = f"Cannot parse accession: {assembly_dir}" - raise ValueError(msg) - db, p1, p2, p3 = m.groups() - return f"raw_data/{db}/{p1}/{p2}/{p3}/{assembly_dir}/" + if m := ACCESSION_PARTS_REGEX.match(assembly_dir): + db, p1, p2, p3 = m.groups() + return f"raw_data/{db}/{p1}/{p2}/{p3}/{assembly_dir}/" + msg = f"Cannot parse accession: {assembly_dir}" + raise ValueError(msg) def parse_assembly_path(assembly_path: str) -> tuple[str, str, str]: @@ -102,15 +101,13 @@ def _download_and_verify( # noqa: PLR0913 # Returns False if file checksums mismatch, True otherwise def validate_file(local_file: Path) -> bool: if expected_md5: - actual_md5 = compute_md5(str(local_file)) - if actual_md5 == expected_md5: + if verify_md5(local_file, expected_md5): (dest_dir / f"{filename}.md5").write_text(expected_md5) else: logger.warning( - " MD5 mismatch for %s: expected %s, got %s", + " MD5 mismatch for %s: expected %s", filename, expected_md5, - actual_md5, ) return False logger.debug(" MD5 verified: %s", filename) diff --git a/src/cdm_data_loaders/ncbi_ftp/manifest.py b/src/cdm_data_loaders/ncbi_ftp/manifest.py index f391818c..e411db9e 100644 --- a/src/cdm_data_loaders/ncbi_ftp/manifest.py +++ b/src/cdm_data_loaders/ncbi_ftp/manifest.py @@ -187,11 +187,11 @@ def filter_by_prefix_range( filtered: dict[str, AssemblyRecord] = {} for acc, rec in assemblies.items(): pfx = accession_prefix(acc) - if pfx is None: - continue - if prefix_from is not None and pfx < prefix_from: - continue - if prefix_to is not None and pfx > prefix_to: + if ( + pfx is None + or (prefix_from is not None and pfx < prefix_from) + or (prefix_to is not None and pfx > prefix_to) + ): continue filtered[acc] = rec return filtered diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index a703b93f..fe6fd24d 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -6,7 +6,6 @@ manifest so that a re-run of Phase 2 only downloads remaining entries. """ -import re import tempfile from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed From fa8157a658f17158eb67138d6caa6ea201917f95 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Tue, 2 Jun 2026 11:15:17 -0700 Subject: [PATCH 106/128] remove unused import --- src/cdm_data_loaders/ncbi_ftp/manifest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/manifest.py b/src/cdm_data_loaders/ncbi_ftp/manifest.py index e411db9e..59908fbe 100644 --- a/src/cdm_data_loaders/ncbi_ftp/manifest.py +++ b/src/cdm_data_loaders/ncbi_ftp/manifest.py @@ -10,7 +10,6 @@ import contextlib import csv import json -import re import time from collections.abc import Callable, Iterable from dataclasses import dataclass, field From 02a3ce229e99dc7b549278884e8e38430fc2dfc7 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Tue, 2 Jun 2026 11:22:49 -0700 Subject: [PATCH 107/128] Potential fix for pull request finding 'Unused import' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- src/cdm_data_loaders/ncbi_ftp/assembly.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cdm_data_loaders/ncbi_ftp/assembly.py b/src/cdm_data_loaders/ncbi_ftp/assembly.py index 25ce7fd4..5d75f533 100644 --- a/src/cdm_data_loaders/ncbi_ftp/assembly.py +++ b/src/cdm_data_loaders/ncbi_ftp/assembly.py @@ -13,7 +13,7 @@ from cdm_data_loaders.ncbi_ftp.constants import ACCESSION_PARTS_REGEX, ASSEMBLY_PATH_REGEX, FTP_HOST from cdm_data_loaders.utils.cdm_logger import get_cdm_logger -from cdm_data_loaders.utils.checksums import compute_md5, verify_md5 +from cdm_data_loaders.utils.checksums import verify_md5 from cdm_data_loaders.utils.ftp_client import connect_ftp, ftp_noop_keepalive, ftp_retrieve_text logger = get_cdm_logger() From 8f7ac0989bb8da1c232468340889d818e0ae3245 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Tue, 2 Jun 2026 13:29:17 -0700 Subject: [PATCH 108/128] merge s3 modules --- scripts/s3_local.py | 97 ++++---------------------------- src/cdm_data_loaders/utils/s3.py | 82 +++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 87 deletions(-) diff --git a/scripts/s3_local.py b/scripts/s3_local.py index 2f3926de..c42a9300 100755 --- a/scripts/s3_local.py +++ b/scripts/s3_local.py @@ -16,109 +16,32 @@ AWS_SECRET_ACCESS_KEY minioadmin """ -import json import os import sys from pathlib import Path -import boto3 -from botocore.client import BaseClient +import cdm_data_loaders.utils.s3 as s3 -def _client() -> BaseClient: - return boto3.client( - "s3", - endpoint_url=os.environ.get("AWS_ENDPOINT_URL", "http://localhost:9000"), - aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID", "minioadmin"), - aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY", "minioadmin"), - ) - - -def _split(uri: str) -> tuple[str, str]: - """Split ``s3://bucket/key`` into ``(bucket, key)``.""" - if not uri.startswith("s3://"): - raise SystemExit(f"Expected s3:// URI, got: {uri}") - parts = uri[5:].split("/", 1) - return parts[0], parts[1] if len(parts) > 1 else "" - - -# subcommands - - -def cmd_mb(args: list[str]) -> None: - """Create a bucket: ``mb s3://bucket``.""" - if not args: - raise SystemExit("Usage: s3_local.py mb s3://BUCKET") - bucket, _ = _split(args[0]) - s3 = _client() - try: - s3.head_bucket(Bucket=bucket) - print(f"Bucket already exists: {bucket}") - except Exception: # noqa: BLE001 - s3.create_bucket(Bucket=bucket) - print(f"Created bucket: {bucket}") - - -def cmd_cp(args: list[str]) -> None: - """Recursive upload: ``cp LOCAL_DIR s3://bucket/prefix/``.""" - if len(args) < 2: # noqa: PLR2004 - raise SystemExit("Usage: s3_local.py cp LOCAL_DIR s3://BUCKET/PREFIX/") - local_dir = Path(args[0]) - bucket, prefix = _split(args[1]) - prefix = prefix.rstrip("/") + "/" if prefix else "" - s3 = _client() - count = 0 - for path in sorted(local_dir.rglob("*")): - if path.is_dir(): - continue - rel = path.relative_to(local_dir) - key = f"{prefix}{rel}" - s3.upload_file(Filename=str(path), Bucket=bucket, Key=key) - count += 1 - print(f" {key}") - print(f"Uploaded {count} files to s3://{bucket}/{prefix}") - - -def cmd_ls(args: list[str]) -> None: - """List objects: ``ls s3://bucket/prefix/ [--limit N]``.""" - if not args: - raise SystemExit("Usage: s3_local.py ls s3://BUCKET/PREFIX/ [--limit N]") - bucket, prefix = _split(args[0]) - limit = 20 - if "--limit" in args: - idx = args.index("--limit") - limit = int(args[idx + 1]) - s3 = _client() - paginator = s3.get_paginator("list_objects_v2") - shown = 0 - for page in paginator.paginate(Bucket=bucket, Prefix=prefix): - for obj in page.get("Contents", []): - print(f" {obj['Size']:>10} {obj['Key']}") - shown += 1 - if shown >= limit: - return - - -def cmd_head(args: list[str]) -> None: - """Show metadata: ``head s3://bucket/key``.""" - if not args: - raise SystemExit("Usage: s3_local.py head s3://BUCKET/KEY") - bucket, key = _split(args[0]) - s3 = _client() - resp = s3.head_object(Bucket=bucket, Key=key) - meta = resp.get("Metadata", {}) - print(json.dumps(meta, indent=2)) +def _client() -> None: + s3.reset_s3_client() + _ = s3.get_s3_client({ + "endpoint_url": os.environ.get("AWS_ENDPOINT_URL", "http://localhost:9000"), + "aws_access_key_id": os.environ.get("AWS_ACCESS_KEY_ID", "minioadmin"), + "aws_secret_access_key": os.environ.get("AWS_SECRET_ACCESS_KEY", "minioadmin"), + }) # dispatch -COMMANDS = {"mb": cmd_mb, "cp": cmd_cp, "ls": cmd_ls, "head": cmd_head} +COMMANDS = {"mb": s3.cmd_mb, "cp": s3.cmd_cp, "ls": s3.cmd_ls, "head": s3.cmd_head} def main() -> None: if len(sys.argv) < 2 or sys.argv[1] not in COMMANDS: # noqa: PLR2004 cmds = ", ".join(COMMANDS) raise SystemExit(f"Usage: s3_local.py <{cmds}> [args ...]\n\n{__doc__}") + _client() COMMANDS[sys.argv[1]](sys.argv[2:]) diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index 4101b98e..a196f447 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -7,8 +7,10 @@ import boto3 import botocore import botocore.client +import json import tqdm from botocore.config import Config +from botocore.exceptions import ClientError from cdm_data_loaders.utils.cdm_logger import get_cdm_logger @@ -124,6 +126,10 @@ def split_s3_path(s3_path: str) -> tuple[str | None, str]: raise ValueError(err_msg) path_parts = unprefixed_path.split("/", 1) + # return just the bucket if that is all that was passed + if len(path_parts) == 1: + return (path_parts[0], "") + # the first part should be the bucket and the second part the key if len(path_parts) != 2 or not path_parts[1]: # noqa: PLR2004 err_msg = f"Invalid path: '{s3_path}'\nCould not parse out bucket and key" @@ -572,3 +578,79 @@ def delete_objects(bucket: str, keys: list[str]) -> list[dict[str, Any]]: ) errors.extend(resp.get("Errors", [])) return errors + + +# Helper functions for command-line tool + + +def cmd_mb(args: list[str]) -> None: + """Create a bucket: ``mb s3://bucket``.""" + if not args: + raise SystemExit("Usage: s3_local.py mb s3://BUCKET") + bucket, _ = split_s3_path(args[0]) + s3 = get_s3_client() + try: + s3.head_bucket(Bucket=bucket) + print(f"Bucket already exists: {bucket}") + except Exception: # noqa: BLE001 + s3.create_bucket(Bucket=bucket) + print(f"Created bucket: {bucket}") + + +def cmd_cp(args: list[str]) -> None: + """Recursive upload: ``cp LOCAL_DIR s3://bucket/prefix/``.""" + if len(args) < 2: # noqa: PLR2004 + raise SystemExit("Usage: s3_local.py cp LOCAL_DIR s3://BUCKET/PREFIX/") + local_dir = Path(args[0]) + bucket, prefix = split_s3_path(args[1]) + prefix = prefix.rstrip("/") + "/" if prefix else "" + s3 = get_s3_client() + count = 0 + for path in sorted(local_dir.rglob("*")): + if path.is_dir(): + continue + rel = path.relative_to(local_dir) + key = f"{prefix}{rel}" + s3.upload_file(Filename=str(path), Bucket=bucket, Key=key) + count += 1 + print(f" {key}") + print(f"Uploaded {count} files to s3://{bucket}/{prefix}") + + +def cmd_ls(args: list[str]) -> None: + """List objects: ``ls s3://bucket/prefix/ [--limit N]``.""" + if not args: + raise SystemExit("Usage: s3_local.py ls s3://BUCKET/PREFIX/ [--limit N]") + bucket, prefix = split_s3_path(args[0]) + limit = 20 + if "--limit" in args: + idx = args.index("--limit") + limit = int(args[idx + 1]) + s3 = get_s3_client() + paginator = s3.get_paginator("list_objects_v2") + shown = 0 + for page in paginator.paginate(Bucket=bucket, Prefix=prefix): + for obj in page.get("Contents", []): + print(f" {obj['Size']:>10} {obj['Key']}") + shown += 1 + if shown >= limit: + return + + +def cmd_head(args: list[str]) -> None: + """Show metadata: ``head s3://bucket/key``.""" + if not args: + raise SystemExit("Usage: s3_local.py head s3://BUCKET/KEY") + bucket, key = split_s3_path(args[0]) + s3 = get_s3_client() + meta = {} + try: + resp = s3.head_object(Bucket=bucket, Key=key) + meta = resp.get("Metadata", {}) + except ClientError as e: + if e.response["Error"]["Code"] == "404": # type: ignore[union-attr] + print(f"File not found in store: {bucket}/{key}") + return + raise + print(f"Metadata for {bucket}/{key}:") + print(json.dumps(meta, indent=2)) From 4f6e7311ff87dd889fbc4d847863a85a8f9440b4 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Tue, 2 Jun 2026 13:31:03 -0700 Subject: [PATCH 109/128] formatting --- scripts/s3_local.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/s3_local.py b/scripts/s3_local.py index c42a9300..93b5320d 100755 --- a/scripts/s3_local.py +++ b/scripts/s3_local.py @@ -18,18 +18,19 @@ import os import sys -from pathlib import Path import cdm_data_loaders.utils.s3 as s3 def _client() -> None: s3.reset_s3_client() - _ = s3.get_s3_client({ - "endpoint_url": os.environ.get("AWS_ENDPOINT_URL", "http://localhost:9000"), - "aws_access_key_id": os.environ.get("AWS_ACCESS_KEY_ID", "minioadmin"), - "aws_secret_access_key": os.environ.get("AWS_SECRET_ACCESS_KEY", "minioadmin"), - }) + _ = s3.get_s3_client( + { + "endpoint_url": os.environ.get("AWS_ENDPOINT_URL", "http://localhost:9000"), + "aws_access_key_id": os.environ.get("AWS_ACCESS_KEY_ID", "minioadmin"), + "aws_secret_access_key": os.environ.get("AWS_SECRET_ACCESS_KEY", "minioadmin"), + } + ) # dispatch From 17a9c5cb4139beb4a7d985e36227353f0ebf8840 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Tue, 2 Jun 2026 13:43:24 -0700 Subject: [PATCH 110/128] update split path function for bucket-only requests --- src/cdm_data_loaders/utils/s3.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/cdm_data_loaders/utils/s3.py b/src/cdm_data_loaders/utils/s3.py index a196f447..5dbcae9a 100644 --- a/src/cdm_data_loaders/utils/s3.py +++ b/src/cdm_data_loaders/utils/s3.py @@ -100,13 +100,15 @@ def reset_s3_client() -> None: _s3_client = None -def split_s3_path(s3_path: str) -> tuple[str | None, str]: +def split_s3_path(s3_path: str, *, allow_bucket_only: bool = False) -> tuple[str | None, str]: """Convert a full s3 path (including bucket) into a bucket and key pair. Returns a tuple of bucket, key :param s3_path: an s3 path, including the bucket name :type s3_path: str + :param allow_bucket_only: Allow parsing of a path that only includes a bucket (no key) + :type allow_bucket_only: bool :return: tuple of (bucket, key) :rtype: tuple[str | None, str] """ @@ -127,7 +129,7 @@ def split_s3_path(s3_path: str) -> tuple[str | None, str]: path_parts = unprefixed_path.split("/", 1) # return just the bucket if that is all that was passed - if len(path_parts) == 1: + if allow_bucket_only and len(path_parts) == 1: return (path_parts[0], "") # the first part should be the bucket and the second part the key @@ -587,7 +589,7 @@ def cmd_mb(args: list[str]) -> None: """Create a bucket: ``mb s3://bucket``.""" if not args: raise SystemExit("Usage: s3_local.py mb s3://BUCKET") - bucket, _ = split_s3_path(args[0]) + bucket, _ = split_s3_path(args[0], allow_bucket_only=True) s3 = get_s3_client() try: s3.head_bucket(Bucket=bucket) From aaef5aa9a9d3d8dd0dc697e91849548dfd0c733a Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Tue, 2 Jun 2026 14:47:37 -0700 Subject: [PATCH 111/128] update docs --- README.md | 18 ++---------------- docs/ncbi_ftp_e2e_walkthrough.md | 24 ------------------------ 2 files changed, 2 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index edb8198e..2661475b 100644 --- a/README.md +++ b/README.md @@ -73,28 +73,14 @@ The `cdm-data-loaders` kernel should now be available from the dropdown list of #### Jupyter Kernel Environment Variables -Often you will need access to environment variables that are included in the default Lakehouse -Jupyter environment, but will not be automatically included in your custom Jupyter kernel. To address -this, first identify the needed variables and values, and add them to your new kernel configuration -with the following steps: - -Open a new Jupyter Notebook __with the default kernel__ and run this in a new cell: -```python -import os -for k, v in sorted(os.environ.items()): - if "AWS" in k or "S3" in k or "MINIO" in k: # replace with whatever keys you're interested in - print(f"{k}={v}") -``` -Take the output and add the environment vars to the `kernel.json` for your new kernel (e.g., in `cdm-data-loaders/.venv/share/jupyter/kernels/python3/kernel.json`): +If you would like to include environment variables for your kernel that are not present in your default environment, you can add them to the `kernel.json` for your new kernel (e.g., in `cdm-data-loaders/.venv/share/jupyter/kernels/python3/kernel.json`): ```json { "argv": ["..."], "display_name": "cdm-data-loaders", "language": "python", "env": { - "AWS_ACCESS_KEY_ID": "...", - "AWS_SECRET_ACCESS_KEY": "...", - "AWS_DEFAULT_REGION": "...", + "MY_CUSTOM_VAR": "...", ... } } diff --git a/docs/ncbi_ftp_e2e_walkthrough.md b/docs/ncbi_ftp_e2e_walkthrough.md index 83eb3e1e..7c4e7909 100644 --- a/docs/ncbi_ftp_e2e_walkthrough.md +++ b/docs/ncbi_ftp_e2e_walkthrough.md @@ -121,30 +121,6 @@ uv run python -m ipykernel install --user --name cdm-data-loaders --display-name ``` Then, when you open the manifest or promote notebooks, choose the `cdm-data-loaders` kernel. -#### Add the S3 Credentials to the Kernel - -Open a new Jupyter Notebook with the default kernel and run this in a new cell: -```python -import os -for k, v in sorted(os.environ.items()): - if "AWS" in k or "S3" in k or "MINIO" in k: - print(f"{k}={v}") -``` -Take the output and add the environment vars to the `kernel.json` for your new kernel (e.g., in `cdm-data-loaders/.venv/share/jupyter/kernels/python3/kernel.json`): -```json -{ - "argv": ["..."], - "display_name": "cdm-data-loaders", - "language": "python", - "env": { - "AWS_ACCESS_KEY_ID": "...", - "AWS_SECRET_ACCESS_KEY": "...", - "AWS_DEFAULT_REGION": "...", - ... - } -} -``` - --- From c642acff222fe89cdabf709f32a0b6d233dd5b53 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Jun 2026 14:36:40 +0000 Subject: [PATCH 112/128] Bump delta-spark from 4.1.0 to 4.2.0 Bumps [delta-spark](https://github.com/delta-io/delta) from 4.1.0 to 4.2.0. - [Release notes](https://github.com/delta-io/delta/releases) - [Commits](https://github.com/delta-io/delta/compare/v4.1.0...v4.2.0) --- updated-dependencies: - dependency-name: delta-spark dependency-version: 4.2.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- uv.lock | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ff083b48..dfafa092 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "boto3[crt]>=1.42.55", "click>=8.4.1", "defusedxml>=0.7.1", - "delta-spark>=4.1.0", + "delta-spark>=4.2.0", "dlt[deltalake,duckdb,filesystem,parquet]>=1.27.2", "frictionless[aws]>=5.19.0", "frozendict>=2.4.7", diff --git a/uv.lock b/uv.lock index 3981cb34..d1b8f3b1 100644 --- a/uv.lock +++ b/uv.lock @@ -471,7 +471,7 @@ requires-dist = [ { name = "boto3", extras = ["crt"], specifier = ">=1.42.55" }, { name = "click", specifier = ">=8.4.1" }, { name = "defusedxml", specifier = ">=0.7.1" }, - { name = "delta-spark", specifier = ">=4.1.0" }, + { name = "delta-spark", specifier = ">=4.2.0" }, { name = "dlt", extras = ["deltalake", "duckdb", "filesystem", "parquet"], specifier = ">=1.27.2" }, { name = "frictionless", extras = ["aws"], specifier = ">=5.19.0" }, { name = "frozendict", specifier = ">=2.4.7" }, @@ -867,15 +867,15 @@ wheels = [ [[package]] name = "delta-spark" -version = "4.1.0" +version = "4.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "importlib-metadata" }, { name = "pyspark" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2f/09/d394015eb956c4475f6a949fb5fccedf7af19f97e981acbc81629e868a5e/delta_spark-4.1.0.tar.gz", hash = "sha256:98f73c2744f972919e0472974467f85d157810b617341ebf586374d91b8eadc7", size = 36808, upload-time = "2026-02-20T18:37:59.8Z" } +sdist = { url = "https://files.pythonhosted.org/packages/37/e4/dcf0afa219bc454d5edab65a4defd4dc90e196fdb8c6fb5156ec6d7fc468/delta_spark-4.2.0.tar.gz", hash = "sha256:198079cf74ee40788895a73111900095ebaed2c631205315868b4406c521e0d5", size = 45896, upload-time = "2026-04-10T23:31:30.102Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/03/2e440efd4a49c8ecfbcb665dea7db94583cf33a365d8480e47fb0aa0dc39/delta_spark-4.1.0-py3-none-any.whl", hash = "sha256:d80f6ebca542df48f257f2535f9c8c21bbfda65771b6ec21be843c0087e1dece", size = 43877, upload-time = "2026-02-20T18:37:58.15Z" }, + { url = "https://files.pythonhosted.org/packages/0a/86/766c7cb36a5209c09222be74bad9c047da22690c8e562fab4be29b6a5776/delta_spark-4.2.0-py3-none-any.whl", hash = "sha256:46327eaeb50915aec3b66bb4ab8fb8db7f90774f2a3bb65bb859333cedc6ef8f", size = 53929, upload-time = "2026-04-10T23:31:28.884Z" }, ] [[package]] @@ -1391,14 +1391,14 @@ wheels = [ [[package]] name = "importlib-metadata" -version = "9.0.0" +version = "8.7.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "zipp" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/49/3b30cad09e7771a4982d9975a8cbf64f00d4a1ececb53297f1d9a7be1b10/importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb", size = 57107, upload-time = "2025-12-21T10:00:19.278Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl", hash = "sha256:2d21d1cc5a017bd0559e36150c21c830ab1dc304dedd1b7ea85d20f45ef3edd7", size = 27789, upload-time = "2026-03-20T06:42:55.665Z" }, + { url = "https://files.pythonhosted.org/packages/fa/5e/f8e9a1d23b9c20a551a8a02ea3637b4642e22c2626e3a13a9a29cdea99eb/importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151", size = 27865, upload-time = "2025-12-21T10:00:18.329Z" }, ] [[package]] @@ -2991,12 +2991,12 @@ crypto = [ [[package]] name = "pyspark" -version = "4.1.2" +version = "4.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "py4j" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5e/71/4dd20c69332a2a4bf7ece8a655c9da98e4bd9b6bcea235349c1a00399d57/pyspark-4.1.2.tar.gz", hash = "sha256:fa5d6159f700d0990a07f4f62df1b7449401dccee9cd7d5d6df8957530841602", size = 455428043, upload-time = "2026-05-21T14:49:21.785Z" } +sdist = { url = "https://files.pythonhosted.org/packages/19/bf/58ee13add151469c25825b7125bbf62c3bdcec05eec4d458fcb5c5516066/pyspark-4.1.1.tar.gz", hash = "sha256:77f78984aa84fbe865c717dd37b49913b4e5c97d76ef6824f932f1aefa6621ec", size = 455359625, upload-time = "2026-01-09T09:38:38.28Z" } [package.optional-dependencies] connect = [ From e45d43389c8383d40ef54603667f86c839c63dfb Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 3 Jun 2026 13:57:56 -0700 Subject: [PATCH 113/128] replace minio with ceph for testing --- README.md | 35 +++--- docker-compose.yml | 47 +++---- docs/ncbi_ftp_e2e_walkthrough.md | 56 ++++----- notebooks/ncbi_ftp_download.ipynb | 6 +- notebooks/ncbi_ftp_manifest.ipynb | 6 +- notebooks/ncbi_ftp_promote.ipynb | 6 +- pyproject.toml | 8 +- scripts/entrypoint.sh | 2 +- scripts/s3_local.py | 10 +- tests/integration/conftest.py | 50 ++++---- tests/integration/test_download_e2e.py | 14 +-- tests/integration/test_full_pipeline.py | 28 ++--- tests/integration/test_manifest_e2e.py | 36 +++--- tests/integration/test_promote_e2e.py | 160 ++++++++++++------------ 14 files changed, 226 insertions(+), 238 deletions(-) diff --git a/README.md b/README.md index 2661475b..115cb895 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Repo for CDM input data loading and wrangling - [Development](#development) - [Spark and other non-python dependencies](#spark-and-other-non-python-dependencies) - [Tests](#tests) - - [Integration tests (MinIO + NCBI FTP)](#integration-tests-minio--ncbi-ftp) + - [Integration tests (CEPH + NCBI FTP)](#integration-tests-ceph--ncbi-ftp) - [Loading genomes, contigs, and features](#loading-genomes-contigs-and-features) - [Running bbmap stats and checkm2 on genome or contigset files](#running-bbmap-stats-and-checkm2-on-genome-or-contigset-files) @@ -162,17 +162,17 @@ To generate coverage for the tests, run The standard python `coverage` package is used and coverage can be generated as html or other formats by changing the parameters. -#### Integration tests (MinIO + NCBI FTP) +#### Integration tests (CEPH + NCBI FTP) -End-to-end integration tests for the NCBI assembly pipeline live in `tests/integration/`. They exercise the full flow — manifest diffing, FTP download, S3 promote/archive — against a locally running [MinIO](https://min.io/) container and the real NCBI FTP server. +End-to-end integration tests for the NCBI assembly pipeline live in `tests/integration/`. They exercise the full flow — manifest diffing, FTP download, S3 promote/archive — against a locally running [CEPH](https://ceph.io/) container and the real NCBI FTP server. **Requirements:** -- Docker (for MinIO) +- Docker or Podman (for CEPH) - Network access to `ftp.ncbi.nlm.nih.gov` **Running with Docker Compose (easiest)** -The [docker-compose.yml](docker-compose.yml) at the repo root defines both a MinIO service and the integration test runner. To build the image, start MinIO, and run the integration tests in one command: +The [docker-compose.yml](docker-compose.yml) at the repo root defines both a CEPH service and the integration test runner. To build the image, start CEPH, and run the integration tests in one command: ```sh docker compose up --build --abort-on-container-exit @@ -186,18 +186,19 @@ docker compose down --volumes **Running manually** -If you prefer to run the tests directly against a local MinIO instance (e.g. for faster iteration during development), follow the steps below. +If you prefer to run the tests directly against a local CEPH instance (e.g. for faster iteration during development), follow the steps below. -**1. Start MinIO locally:** +**1. Start CEPH locally:** ```sh docker run -d \ - --name minio \ - -p 9000:9000 \ - -p 9001:9001 \ - -e MINIO_ROOT_USER=minioadmin \ - -e MINIO_ROOT_PASSWORD=minioadmin \ - minio/minio:RELEASE.2025-02-28T09-55-16Z server /data --console-address ":9001" + --name ceph \ + -p 9000:8080 \ + -p 9001:8443 \ + -e RGW_PORT=8080 \ + -e RGW_ACCESS_KEY=test_access_key \ + -e RGW_SECRET_KEY=test_access_secret \ + ghcr.io/kbasetest/ceph-rgw-test-image:0.1.5 ``` **2. Run the integration tests:** @@ -206,16 +207,16 @@ docker run -d \ > uv run pytest tests/integration/ -m integration -v ``` -Tests are automatically skipped when MinIO is not reachable, so the default `uv run pytest` will never fail due to a missing MinIO instance. +Tests are automatically skipped when CEPH is not reachable, so the default `uv run pytest` will never fail due to a missing CEPH instance. **3. Inspect results:** -Buckets are **not** cleaned up after tests. Browse the MinIO console at [http://localhost:9001](http://localhost:9001) (login: `minioadmin` / `minioadmin`) to inspect the final state of each test bucket. Each test method creates its own bucket (e.g. `integ-test-promote-dry-run`). +Buckets are **not** cleaned up after tests. Browse the CEPH console at [http://localhost:9001](http://localhost:9001) (login: `admin` / `admin`) to inspect the final state of each test bucket. Each test method creates its own bucket (e.g. `integ-test-promote-dry-run`). -**4. Stop MinIO when done:** +**4. Stop CEPH when done:** ```sh -docker stop minio && docker rm minio +docker stop ceph && docker rm ceph ``` > **Note:** These tests download real assemblies from NCBI FTP and are inherently slow (~30–60s per assembly). They are also marked `slow_test` so you can exclude them independently: `uv run pytest -m "not slow_test"`. diff --git a/docker-compose.yml b/docker-compose.yml index 39a0b3db..73239988 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,44 +1,31 @@ services: - minio: - image: quay.io/minio/minio:latest - command: server /data --console-address ":9001" + ceph: + image: ghcr.io/kbasetest/ceph-rgw-test-image:0.1.5 environment: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin + RGW_PORT: "8080" + RGW_ACCESS_KEY: test_access_key + RGW_SECRET_KEY: test_access_secret ports: - - "9000:9000" - - "9001:9001" + - "9000:8080" + - "9001:8443" healthcheck: - test: [ "CMD", "mc", "ready", "local" ] - interval: 5s + # Mark healthy only when the RGW/S3 HTTP endpoint is reachable. + test: [ "CMD-SHELL", "curl -fsS http://127.0.0.1:8080/ >/dev/null || wget -q -O- http://127.0.0.1:8080/ >/dev/null" ] + interval: 10s timeout: 5s - retries: 5 + retries: 3 + start_period: 30s integration-tests: image: cdm-data-loaders-integration-tests:latest build: context: . depends_on: - minio: + ceph: condition: service_healthy environment: - AWS_ENDPOINT_URL: http://minio:9000 - AWS_ACCESS_KEY_ID: minioadmin - AWS_SECRET_ACCESS_KEY: minioadmin - entrypoint: - - /bin/sh - - -c - - | - attempts=0 - until python3 -c " - import urllib.request, os - urllib.request.urlopen(os.environ['AWS_ENDPOINT_URL'] + '/minio/health/live', timeout=1) - " 2>/dev/null; do - attempts=$$((attempts + 1)) - if [ "$$attempts" -ge 30 ]; then - echo 'Timed out waiting for MinIO.' && exit 1 - fi - echo 'Waiting for MinIO...' && sleep 1 - done - exec /app/scripts/entrypoint.sh integration_test + AWS_ENDPOINT_URL: http://ceph:8080 + AWS_ACCESS_KEY_ID: test_access_key + AWS_SECRET_ACCESS_KEY: test_access_secret + entrypoint: [ "/app/scripts/entrypoint.sh", "integration_test" ] command: [] diff --git a/docs/ncbi_ftp_e2e_walkthrough.md b/docs/ncbi_ftp_e2e_walkthrough.md index 7c4e7909..ef9eeb87 100644 --- a/docs/ncbi_ftp_e2e_walkthrough.md +++ b/docs/ncbi_ftp_e2e_walkthrough.md @@ -1,7 +1,7 @@ # NCBI FTP Pipeline — Local End-to-End Walkthrough Step-by-step instructions for running a small (≤ 10 assembly) end-to-end sync -of NCBI RefSeq records against a local MinIO container. The walkthrough uses +of NCBI RefSeq records against a local CEPH container. The walkthrough uses the two existing Jupyter notebooks for Phases 1 and 3, and the project's Docker image for the Phase 2 download step. @@ -80,23 +80,24 @@ s3://{STAGING_BUCKET}/{STAGING_KEY_PREFIX}raw_data/{GCF|GCA}/{nnn}/{nnn}/{nnn}/{ ### Local testing -### Start MinIO +### Start CEPH ```sh docker run -d \ - --name minio \ - -p 9000:9000 \ - -p 9001:9001 \ - -e MINIO_ROOT_USER=minioadmin \ - -e MINIO_ROOT_PASSWORD=minioadmin \ - minio/minio:RELEASE.2025-02-28T09-55-16Z server /data --console-address ":9001" + --name ceph \ + -p 9000:8080 \ + -p 9001:8443 \ + -e RGW_PORT=8080 \ + -e RGW_ACCESS_KEY=test_access_key \ + -e RGW_SECRET_KEY=test_access_secret \ + ghcr.io/kbasetest/ceph-rgw-test-image:0.1.5 ``` (Note that a similar service is included in the `docker-compose` configuration file at the root of this repository that is used in CI test workflows.) -Create a test bucket via the [MinIO console](http://localhost:9001) -(login: `minioadmin` / `minioadmin`), or from the command line using the +Create a test bucket via the [CEPH console](http://localhost:9001) +(login: `admin` / `admin`), or from the command line using the included `scripts/s3_local.py` helper (requires no extra installs — only `boto3` which is already a project dependency): @@ -142,10 +143,10 @@ Open `notebooks/ncbi_ftp_manifest.ipynb` in JupyterLab or VS Code. | `STORE_KEY_PREFIX` | `"tenant-general-warehouse/kbase/datasets/ncbi/"` | S3 key prefix | default Lakehouse path prefix | | `OUTPUT_DIR` | `Path("output")` | local path | keep as-is (local directory) | -### Initialise the S3 client for MinIO +### Initialise the S3 client for CEPH If you set `PREVIOUS_SUMMARY_URI`, `SNAPSHOT_UPLOAD_URI`, `LAKEHOUSE_BUCKET`, -or `STAGING_URI` to point at your local MinIO, you must initialise +or `STAGING_URI` to point at your local CEPH, you must initialise the S3 client **before** running the cells that use them. Insert a new cell after Cell 1 (Imports) with: @@ -155,8 +156,8 @@ from cdm_data_loaders.utils.s3 import get_s3_client, reset_s3_client reset_s3_client() get_s3_client({ "endpoint_url": "http://localhost:9000", - "aws_access_key_id": "minioadmin", - "aws_secret_access_key": "minioadmin", + "aws_access_key_id": "test_access_key", + "aws_secret_access_key": "test_access_secret", }) ``` @@ -229,7 +230,7 @@ can stage them into the container. For local testing, set `STAGING_URI = None` (the default) and copy the manifest manually in Step 3b below. -If you are testing against MinIO and want to exercise the S3 upload path: +If you are testing against CEPH and want to exercise the S3 upload path: ```python STAGING_URI = "s3://cdm-lake/staging/run1/" @@ -321,10 +322,10 @@ the FTP server's `md5checksums.txt`. > --threads 2 --limit 10 > ``` -### 3d. Upload staged files to MinIO +### 3d. Upload staged files to CEPH The download step writes to the local filesystem. To feed Phase 3 we need -to upload the staged files into MinIO under a staging prefix: +to upload the staged files into CEPH under a staging prefix: ```sh uv run python scripts/s3_local.py cp notebooks/staging/raw_data/ s3://cts/staging/run1/raw_data/ @@ -356,10 +357,10 @@ Open `notebooks/ncbi_ftp_promote.ipynb`. | `LAKEHOUSE_KEY_PREFIX` | `"tenant-general-warehouse/kbase/datasets/ncbi/"` | S3 key prefix | keep default | | `DRY_RUN` | `True` | bool | **start with dry-run!** | -### Initialise the S3 client for MinIO +### Initialise the S3 client for CEPH The notebook calls `get_s3_client()` which, by default, tries to import -credentials from `berdl_notebook_utils`. For local MinIO you need to +credentials from `berdl_notebook_utils`. For local CEPH you need to initialise the client manually **before** running Cell 4. Insert a new cell after Cell 2 (Imports) with: @@ -369,8 +370,8 @@ from cdm_data_loaders.utils.s3 import get_s3_client, reset_s3_client reset_s3_client() # clear any cached client get_s3_client({ "endpoint_url": "http://localhost:9000", - "aws_access_key_id": "minioadmin", - "aws_secret_access_key": "minioadmin", + "aws_access_key_id": "test_access_key", + "aws_secret_access_key": "test_access_secret", }) ``` @@ -382,7 +383,7 @@ get_s3_client({ 3. If the dry-run looks correct, set `DRY_RUN = False` in Cell 3 and re-run from Cell 3. -After promotion the final Lakehouse layout in MinIO will look like: +After promotion the final Lakehouse layout in CEPH will look like: ``` cdm-lake/ @@ -395,9 +396,9 @@ cdm-lake/ --- -## 5. Inspect results in MinIO +## 5. Inspect results in CEPH -Browse the [MinIO console](http://localhost:9001) or use the CLI: +Browse the [CEPH console](http://localhost:9001) or use the CLI: ```sh # List final Lakehouse objects @@ -471,8 +472,8 @@ previous snapshot: ## 7. Cleanup ```sh -# Stop and remove MinIO -docker stop minio && docker rm minio +# Stop and remove CEPH +docker stop ceph && docker rm ceph # Remove local staging data rm -rf staging/ output/ @@ -484,9 +485,8 @@ rm -rf staging/ output/ | Symptom | Cause | Fix | |---------|-------|-----| -| `berdl_notebook_utils` import error in notebook | Missing local MinIO client init | Add the `get_s3_client({...})` cell described in Step 4 | +| `berdl_notebook_utils` import error in notebook | Missing local CEPH client init | Add the `get_s3_client({...})` cell described in Step 4 | | `connect_ftp() timeout` | NCBI FTP may be slow or rate-limited | Retry; reduce `--threads` to 1 | -| `CRC64NVME` errors uploading to MinIO | MinIO version too old (needs ≥ `2025-02-07`) | Pin to `minio/minio:RELEASE.2025-02-28T09-55-16Z` or newer | | Phase 3 shows 0 promoted | Staging prefix doesn't match or bucket is wrong | Verify `STAGING_KEY_PREFIX` matches the S3 upload path from Step 3d | | Container can't reach FTP | Docker network isolation | Use `--network host` or ensure DNS resolution works inside the container | diff --git a/notebooks/ncbi_ftp_download.ipynb b/notebooks/ncbi_ftp_download.ipynb index 2af2c757..286db5cb 100644 --- a/notebooks/ncbi_ftp_download.ipynb +++ b/notebooks/ncbi_ftp_download.ipynb @@ -118,15 +118,15 @@ "source": [ "from cdm_data_loaders.utils.s3 import get_s3_client, reset_s3_client\n", "\n", - "# Provide S3 credentials (use for local testing against MinIO test container)\n", + "# Provide S3 credentials (use for local testing against CEPH test container)\n", "PROVIDE_CREDENTIALS = False # Set to False to rely on environment credentials (e.g. IAM role)\n", "if PROVIDE_CREDENTIALS:\n", " reset_s3_client() # Clear any existing client to ensure new credentials are used\n", " get_s3_client(\n", " {\n", " \"endpoint_url\": \"http://localhost:9000\",\n", - " \"aws_access_key_id\": \"minioadmin\",\n", - " \"aws_secret_access_key\": \"minioadmin\",\n", + " \"aws_access_key_id\": \"test_access_key\",\n", + " \"aws_secret_access_key\": \"test_access_secret\",\n", " }\n", " )" ] diff --git a/notebooks/ncbi_ftp_manifest.ipynb b/notebooks/ncbi_ftp_manifest.ipynb index 19c00f2c..d6caa2a5 100644 --- a/notebooks/ncbi_ftp_manifest.ipynb +++ b/notebooks/ncbi_ftp_manifest.ipynb @@ -74,15 +74,15 @@ "source": [ "from cdm_data_loaders.utils.s3 import reset_s3_client\n", "\n", - "# Provide S3 credentials (use for local testing against MinIO test container)\n", + "# Provide S3 credentials (use for local testing against CEPH test container)\n", "PROVIDE_CREDENTIALS = False # Set to False to rely on environment credentials (e.g. IAM role)\n", "if PROVIDE_CREDENTIALS:\n", " reset_s3_client() # Clear any existing client to ensure new credentials are used\n", " get_s3_client(\n", " {\n", " \"endpoint_url\": \"http://localhost:9000\",\n", - " \"aws_access_key_id\": \"minioadmin\",\n", - " \"aws_secret_access_key\": \"minioadmin\",\n", + " \"aws_access_key_id\": \"test_access_key\",\n", + " \"aws_secret_access_key\": \"test_access_secret\",\n", " }\n", " )" ] diff --git a/notebooks/ncbi_ftp_promote.ipynb b/notebooks/ncbi_ftp_promote.ipynb index 7af5ba0f..1bd6e2b2 100644 --- a/notebooks/ncbi_ftp_promote.ipynb +++ b/notebooks/ncbi_ftp_promote.ipynb @@ -133,15 +133,15 @@ "source": [ "from cdm_data_loaders.utils.s3 import reset_s3_client\n", "\n", - "# Provide S3 credentials (use for local testing against MinIO test container)\n", + "# Provide S3 credentials (use for local testing against CEPH test container)\n", "PROVIDE_CREDENTIALS = False # Set to False to rely on environment credentials (e.g. IAM role)\n", "if PROVIDE_CREDENTIALS:\n", " reset_s3_client() # Clear any existing client to ensure new credentials are used\n", " get_s3_client(\n", " {\n", " \"endpoint_url\": \"http://localhost:9000\",\n", - " \"aws_access_key_id\": \"minioadmin\",\n", - " \"aws_secret_access_key\": \"minioadmin\",\n", + " \"aws_access_key_id\": \"test_access_key\",\n", + " \"aws_secret_access_key\": \"test_access_secret\",\n", " }\n", " )" ] diff --git a/pyproject.toml b/pyproject.toml index 6e220c58..b47c6afc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -191,7 +191,7 @@ log_cli = true log_cli_level = "INFO" log_level = "INFO" addopts = ["-v"] -markers = ["requires_spark: must be run in an environment where spark is available", "s3: tests that mock s3 interactions", "slow_test: does what it says on the tin", "integration: end-to-end tests requiring a running MinIO instance and network access", "external_request: tests that make real network requests to external services (e.g. NCBI FTP)"] +markers = ["requires_spark: must be run in an environment where spark is available", "s3: tests that mock s3 interactions", "slow_test: does what it says on the tin", "integration: end-to-end tests requiring a running CEPH instance and network access", "external_request: tests that make real network requests to external services (e.g. NCBI FTP)"] # environment settings for running tests [tool.pytest_env] @@ -209,10 +209,10 @@ S3_ENDPOINT_URL = "http://localhost:9000" AWS_CONFIG_FILE = "/dev/null" # ensure tests don't pick up creds from a real config file AWS_DEFAULT_REGION = "us-east-1" # default to use containerized S3 test store -AWS_ENDPOINT_URL = "http://localhost:9000" +AWS_ENDPOINT_URL = { value = "http://localhost:9000", skip_if_set = true } AWS_ENDPOINT_URL_S3 = { unset = true } -AWS_ACCESS_KEY_ID = "minioadmin" -AWS_SECRET_ACCESS_KEY = "minioadmin" +AWS_ACCESS_KEY_ID = "test_access_key" +AWS_SECRET_ACCESS_KEY = "test_access_secret" [tool.uv.sources] cdm-schema = { git = "https://github.com/kbase/cdm-schema.git" } diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index c8bf9606..91f35367 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -41,7 +41,7 @@ case "$cmd" in exec /usr/bin/tini -- uv run --no-sync pytest -m "not requires_spark" ;; integration_test) - # run the integration tests (requires a running MinIO instance) + # run the integration tests (requires a running CEPH instance) exec /usr/bin/tini -- uv run --no-sync pytest -m "integration" -v "$@" ;; bash) diff --git a/scripts/s3_local.py b/scripts/s3_local.py index 93b5320d..8302b91a 100755 --- a/scripts/s3_local.py +++ b/scripts/s3_local.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # ruff: noqa: T201, EM101, EM102, TRY003, D103 -"""Thin S3 CLI for local MinIO testing (no aws-cli install required). +"""Thin S3 CLI for local CEPH testing (no aws-cli install required). Usage (all commands assume ``uv run`` from the repo root): @@ -12,8 +12,8 @@ Environment variables (with defaults for the walkthrough): AWS_ENDPOINT_URL http://localhost:9000 - AWS_ACCESS_KEY_ID minioadmin - AWS_SECRET_ACCESS_KEY minioadmin + AWS_ACCESS_KEY_ID test_access_key + AWS_SECRET_ACCESS_KEY test_access_secret """ import os @@ -27,8 +27,8 @@ def _client() -> None: _ = s3.get_s3_client( { "endpoint_url": os.environ.get("AWS_ENDPOINT_URL", "http://localhost:9000"), - "aws_access_key_id": os.environ.get("AWS_ACCESS_KEY_ID", "minioadmin"), - "aws_secret_access_key": os.environ.get("AWS_SECRET_ACCESS_KEY", "minioadmin"), + "aws_access_key_id": os.environ.get("AWS_ACCESS_KEY_ID", "test_access_key"), + "aws_secret_access_key": os.environ.get("AWS_SECRET_ACCESS_KEY", "test_access_secret"), } ) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 35dfc0ac..0d010184 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,9 +1,9 @@ -"""Shared fixtures and helpers for MinIO-backed integration tests. +"""Shared fixtures and helpers for CEPH-backed integration tests. -Integration tests are auto-skipped when MinIO is not reachable. Each test +Integration tests are auto-skipped when CEPH is not reachable. Each test method gets its own bucket (derived from the test node name) that is emptied on re-run but **never deleted** after the test — this lets developers inspect -the final state of the object store via the MinIO console. +the final state of the object store via the CEPH console. """ import hashlib @@ -27,13 +27,13 @@ _MAX_BUCKET_LEN = 63 -# MinIO reachability check +# CEPH reachability check -_minio_available: bool | None = None +_ceph_available: bool | None = None -def _minio_reachable() -> bool: - """Return True if the MinIO endpoint accepts connections.""" +def _ceph_reachable() -> bool: + """Return True if the CEPH endpoint accepts connections.""" try: client = boto3.client( "s3", @@ -50,13 +50,13 @@ def _minio_reachable() -> bool: def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: # noqa: ARG001 - """Auto-skip ``@pytest.mark.integration`` tests when MinIO is unreachable.""" - global _minio_available # noqa: PLW0603 - if _minio_available is None: - _minio_available = _minio_reachable() - if _minio_available: + """Auto-skip ``@pytest.mark.integration`` tests when CEPH is unreachable.""" + global _ceph_available # noqa: PLW0603 + if _ceph_available is None: + _ceph_available = _ceph_reachable() + if _ceph_available: return - skip_marker = pytest.mark.skip(reason="MinIO not reachable — skipping integration tests") + skip_marker = pytest.mark.skip(reason="CEPH not reachable — skipping integration tests") for item in items: if "integration" in item.keywords: item.add_marker(skip_marker) @@ -66,11 +66,11 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item @pytest.fixture -def minio_s3_client() -> botocore.client.BaseClient: - """Session-scoped real boto3 S3 client pointed at the local MinIO instance. +def ceph_s3_client() -> botocore.client.BaseClient: + """Session-scoped real boto3 S3 client pointed at the local CEPH instance. Patches ``get_s3_client`` on every module that uses it so internal calls - are transparently routed to MinIO. + are transparently routed to CEPH. """ client = boto3.client("s3") @@ -105,14 +105,14 @@ def _bucket_name_from_node(node_id: str) -> str: @pytest.fixture -def test_bucket(minio_s3_client: botocore.client.BaseClient, request: pytest.FixtureRequest) -> str: - """Create a per-test-method bucket in MinIO and return its name. +def test_bucket(ceph_s3_client: botocore.client.BaseClient, request: pytest.FixtureRequest) -> str: + """Create a per-test-method bucket in CEPH and return its name. On re-run, any existing objects are deleted first so the test starts clean. The bucket is **not** deleted after the test. """ bucket = _bucket_name_from_node(request.node.nodeid) - s3 = minio_s3_client + s3 = ceph_s3_client try: s3.head_bucket(Bucket=bucket) @@ -133,8 +133,8 @@ def test_bucket(minio_s3_client: botocore.client.BaseClient, request: pytest.Fix @pytest.fixture -def staging_test_bucket(minio_s3_client: botocore.client.BaseClient, request: pytest.FixtureRequest) -> str: - """Create a per-test staging bucket in MinIO and return its name. +def staging_test_bucket(ceph_s3_client: botocore.client.BaseClient, request: pytest.FixtureRequest) -> str: + """Create a per-test staging bucket in CEPH and return its name. Mirrors ``test_bucket`` but uses a ``staging-`` prefix so staging and Lakehouse buckets are distinct within the same test. @@ -143,7 +143,7 @@ def staging_test_bucket(minio_s3_client: botocore.client.BaseClient, request: py if len(bucket) > _MAX_BUCKET_LEN: suffix = hashlib.md5(bucket.encode()).hexdigest()[:6] # noqa: S324 bucket = f"{bucket[: _MAX_BUCKET_LEN - 7]}-{suffix}" - s3 = minio_s3_client + s3 = ceph_s3_client try: s3.head_bucket(Bucket=bucket) @@ -165,13 +165,13 @@ def staging_test_bucket(minio_s3_client: botocore.client.BaseClient, request: py # Helpers -def stage_files_to_minio( +def stage_files_to_ceph( s3: botocore.client.BaseClient, bucket: str, local_dir: str | Path, staging_prefix: str, ) -> list[str]: - """Upload a local directory tree to a MinIO staging prefix. + """Upload a local directory tree to a CEPH staging prefix. :param s3: boto3 S3 client :param bucket: target bucket @@ -199,7 +199,7 @@ def seed_lakehouse( # noqa: PLR0913 path_prefix: str, assembly_dir: str | None = None, ) -> list[str]: - """Seed assembly files at the final Lakehouse path in MinIO. + """Seed assembly files at the final Lakehouse path in CEPH. :param s3: boto3 S3 client :param bucket: target bucket diff --git a/tests/integration/test_download_e2e.py b/tests/integration/test_download_e2e.py index 469c65b6..2128b6b5 100644 --- a/tests/integration/test_download_e2e.py +++ b/tests/integration/test_download_e2e.py @@ -1,7 +1,7 @@ """End-to-end tests for Phase 2 — FTP download of assemblies. These tests download real (small) assemblies from the NCBI FTP server. -Marked ``integration`` and ``slow_test``; auto-skipped when MinIO is +Marked ``integration`` and ``slow_test``; auto-skipped when CEPH is unreachable. """ @@ -133,7 +133,7 @@ def test_download_resume(self, tmp_path: Path) -> None: @pytest.mark.external_request def test_download_and_stage_e2e( tmp_path: Path, - minio_s3_client, + ceph_s3_client, test_bucket: str, ) -> None: """Download one assembly and verify it is staged under the expected S3 prefix.""" @@ -141,15 +141,15 @@ def test_download_and_stage_e2e( staging_prefix = "staging/e2e-test/" - # Seed the manifest in MinIO so download_and_stage can read it from S3 + # Seed the manifest in CEPH so download_and_stage can read it from S3 manifest_s3_key = f"{staging_prefix}input/transfer_manifest.txt" - minio_s3_client.put_object( + ceph_s3_client.put_object( Bucket=test_bucket, Key=manifest_s3_key, Body=manifest_path.read_bytes(), ) - with patch.object(s3_utils, "get_s3_client", return_value=minio_s3_client): + with patch.object(s3_utils, "get_s3_client", return_value=ceph_s3_client): report = download_and_stage( bucket=test_bucket, staging_key_prefix=staging_prefix, @@ -166,7 +166,7 @@ def test_download_and_stage_e2e( assert report["dry_run"] is False # Verify raw_data/ files and .md5 sidecars are staged - paginator = minio_s3_client.get_paginator("list_objects_v2") + paginator = ceph_s3_client.get_paginator("list_objects_v2") staged_keys = [ obj["Key"] for page in paginator.paginate(Bucket=test_bucket, Prefix=f"{staging_prefix}raw_data/") @@ -181,6 +181,6 @@ def test_download_and_stage_e2e( # Verify download_report.json was also uploaded report_key = f"{staging_prefix}download_report.json" - resp = minio_s3_client.get_object(Bucket=test_bucket, Key=report_key) + resp = ceph_s3_client.get_object(Bucket=test_bucket, Key=report_key) saved_report = json.loads(resp["Body"].read()) assert saved_report["succeeded"] >= 1 diff --git a/tests/integration/test_full_pipeline.py b/tests/integration/test_full_pipeline.py index 9c957df0..e524f16a 100644 --- a/tests/integration/test_full_pipeline.py +++ b/tests/integration/test_full_pipeline.py @@ -1,9 +1,9 @@ """End-to-end tests for the full NCBI assembly pipeline (Phase 1 → 2 → 3). Exercises the entire flow: download summary from real NCBI FTP, compute diff, -download a single assembly, stage in MinIO, promote to final Lakehouse path. +download a single assembly, stage in CEPH, promote to final Lakehouse path. -Marked ``integration`` and ``slow_test``; auto-skipped when MinIO is +Marked ``integration`` and ``slow_test``; auto-skipped when CEPH is unreachable. """ @@ -29,7 +29,7 @@ from cdm_data_loaders.ncbi_ftp.promote import DEFAULT_LAKEHOUSE_KEY_PREFIX, promote_from_s3 from cdm_data_loaders.pipelines.ncbi_ftp_download import download_batch -from .conftest import get_object_metadata, list_all_keys, stage_files_to_minio, staging_test_bucket # noqa: F401 +from .conftest import get_object_metadata, list_all_keys, stage_files_to_ceph, staging_test_bucket # noqa: F401 STABLE_PREFIX = "900" STAGING_PREFIX = "staging/run1/" @@ -44,13 +44,13 @@ class TestFullPipelineSmallBatch: def test_full_pipeline_small_batch( self, - minio_s3_client: object, + ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path, ) -> None: - """Single assembly flows through all three phases into MinIO.""" - s3 = minio_s3_client + """Single assembly flows through all three phases into CEPH.""" + s3 = ceph_s3_client # Step 1: Manifest generation raw = download_assembly_summary(database="refseq") @@ -76,9 +76,9 @@ def test_full_pipeline_small_batch( assert report["succeeded"] >= 1 assert report["failed"] == 0 - # Upload local output to MinIO staging - keys = stage_files_to_minio(s3, staging_test_bucket, output_dir, STAGING_PREFIX) - assert len(keys) > 0, "Expected files staged to MinIO" + # Upload local output to CEPH staging + keys = stage_files_to_ceph(s3, staging_test_bucket, output_dir, STAGING_PREFIX) + assert len(keys) > 0, "Expected files staged to CEPH" # Step 3: Promote from staging to final path promote_report = promote_from_s3( @@ -112,13 +112,13 @@ class TestFullPipelineIncrementalSync: def test_full_pipeline_incremental( self, - minio_s3_client: object, + ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path, ) -> None: """Second sync archives the old version and promotes the new one.""" - s3 = minio_s3_client + s3 = ceph_s3_client # First sync: Steps 1 -> 2 -> 3 raw = download_assembly_summary(database="refseq") @@ -136,9 +136,9 @@ def test_full_pipeline_incremental( report1 = download_batch(str(manifest1), str(output1), threads=1, limit=1) assert report1["succeeded"] >= 1 - stage_files_to_minio(s3, staging_test_bucket, output1, STAGING_PREFIX) + stage_files_to_ceph(s3, staging_test_bucket, output1, STAGING_PREFIX) - # Upload manifest to MinIO for trimming (manifest lives in staging bucket) + # Upload manifest to CEPH for trimming (manifest lives in staging bucket) manifest_key = "ncbi/transfer_manifest.txt" s3.upload_file(Filename=str(manifest1), Bucket=staging_test_bucket, Key=manifest_key) @@ -197,7 +197,7 @@ def test_full_pipeline_incremental( staging_keys = list_all_keys(s3, staging_test_bucket, STAGING_PREFIX) for key in staging_keys: s3.delete_object(Bucket=staging_test_bucket, Key=key) - stage_files_to_minio(s3, staging_test_bucket, output2, STAGING_PREFIX) + stage_files_to_ceph(s3, staging_test_bucket, output2, STAGING_PREFIX) # Phase 3 — promote with archival promote2 = promote_from_s3( diff --git a/tests/integration/test_manifest_e2e.py b/tests/integration/test_manifest_e2e.py index 6d3ef5dc..8c2ed8c7 100644 --- a/tests/integration/test_manifest_e2e.py +++ b/tests/integration/test_manifest_e2e.py @@ -1,12 +1,13 @@ """End-to-end tests for Phase 1 — manifest generation and diffing. These tests hit the real NCBI FTP server (with tight prefix filters) and -optionally use MinIO for checksum verification. Marked ``integration`` -and ``slow_test``; auto-skipped when MinIO is unreachable. +optionally use CEPH for checksum verification. Marked ``integration`` +and ``slow_test``; auto-skipped when CEPH is unreachable. """ from __future__ import annotations +from itertools import islice from typing import TYPE_CHECKING import pytest @@ -164,16 +165,17 @@ class TestVerifyTransferCandidatesPrunes: def test_prunes_existing_matching_md5( self, - minio_s3_client: object, + ceph_s3_client: object, test_bucket: str, ) -> None: - """Assemblies with matching MD5 metadata in MinIO are pruned from the transfer list.""" + """Assemblies with matching MD5 metadata in CEPH are pruned from the transfer list.""" _full, filtered = _download_and_filter() - latest = {a: r for a, r in filtered.items() if r.status == "latest"} + # keep just the first 100 items for testing + latest = dict(islice(((a, r) for a, r in filtered.items() if r.status == "latest"), 100)) if not latest: pytest.skip(f"No latest assemblies in prefix {STABLE_PREFIX}") - # Pick one assembly to pre-seed in MinIO with correct checksums + # Pick one assembly to pre-seed in CEPH with correct checksums # and one to partially pre-seed to ensure it doesn't get pruned i_latest = iter(sorted(latest)) acc = next(i_latest) @@ -191,8 +193,8 @@ def test_prunes_existing_matching_md5( checksums = parse_md5_checksums_file(md5_text) - # Seed MinIO with dummy files that have the right MD5 metadata - s3 = minio_s3_client + # Seed CEPH with dummy files that have the right MD5 metadata + s3 = ceph_s3_client path_prefix = DEFAULT_LAKEHOUSE_KEY_PREFIX rel = build_accession_path(rec.assembly_dir) for fname, md5 in checksums.items(): @@ -239,18 +241,18 @@ def test_prunes_existing_matching_md5( @pytest.mark.integration @pytest.mark.slow_test class TestScanStoreToSyntheticSummary: - """Test synthetic assembly summary generation from MinIO store.""" + """Test synthetic assembly summary generation from CEPH store.""" - def test_builds_summary_from_minio_store( + def test_builds_summary_from_ceph_store( self, - minio_s3_client: object, + ceph_s3_client: object, test_bucket: str, ) -> None: - """Verify synthetic summary captures assemblies from MinIO.""" - s3 = minio_s3_client + """Verify synthetic summary captures assemblies from CEPH.""" + s3 = ceph_s3_client path_prefix = DEFAULT_LAKEHOUSE_KEY_PREFIX - # Seed MinIO with a couple of assemblies + # Seed CEPH with a couple of assemblies assemblies = { "GCF_000001215.4_v1": ["_genomic.fna.gz", "_protein.faa.gz"], "GCF_000005845.2_v2": ["_genomic.fna.gz"], @@ -280,14 +282,14 @@ def test_builds_summary_from_minio_store( def test_synthetic_summary_diff_against_current( self, - minio_s3_client: object, + ceph_s3_client: object, test_bucket: str, ) -> None: """Verify synthetic summary can be used as baseline for diffing.""" - s3 = minio_s3_client + s3 = ceph_s3_client path_prefix = DEFAULT_LAKEHOUSE_KEY_PREFIX - # Seed MinIO with one assembly + # Seed CEPH with one assembly key1 = f"{path_prefix}refseq/GCF_000001215.4_old/GCF_000001215.4_old_genomic.fna.gz" s3.put_object(Bucket=test_bucket, Key=key1, Body=b"data") diff --git a/tests/integration/test_promote_e2e.py b/tests/integration/test_promote_e2e.py index f837d1c6..98520b1c 100644 --- a/tests/integration/test_promote_e2e.py +++ b/tests/integration/test_promote_e2e.py @@ -1,10 +1,10 @@ -"""End-to-end tests for Phase 3 — promote and archive in MinIO. +"""End-to-end tests for Phase 3 — promote and archive in CEPH. -Pre-stages fake assembly files in MinIO and exercises ``promote_from_s3`` +Pre-stages fake assembly files in CEPH and exercises ``promote_from_s3`` with various combinations of manifests, archive operations, dry-run mode, manifest trimming, and incomplete staging. -Marked ``integration`` and ``slow_test``; auto-skipped when MinIO is +Marked ``integration`` and ``slow_test``; auto-skipped when CEPH is unreachable. Each test method gets its own bucket. """ @@ -81,9 +81,9 @@ def _write_manifest(tmp_path: Path, accessions: list[str], name: str) -> Path: class TestPromoteFromStaging: """Promote staged files to final Lakehouse paths.""" - def test_promote_from_staging(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: + def test_promote_from_staging(self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: """Staged files appear at the final Lakehouse path with MD5 metadata.""" - s3 = minio_s3_client + s3 = ceph_s3_client _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) report = promote_from_s3( @@ -112,14 +112,14 @@ def test_promote_from_staging(self, minio_s3_client: object, test_bucket: str, s class TestPromoteIdempotent: """Promoting the same staging data twice should succeed without errors.""" - def test_promote_idempotent(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: + def test_promote_idempotent(self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: """Second promote on empty staging succeeds and leaves the lakehouse unchanged. After the first promote, staged files are deleted. A second run therefore finds nothing to promote — which is correct and expected. The lakehouse contents must be identical after both runs. """ - s3 = minio_s3_client + s3 = ceph_s3_client _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) report1 = promote_from_s3( @@ -151,10 +151,10 @@ class TestPromoteArchiveUpdated: """Archive existing assemblies before overwriting with updated versions.""" def test_archive_updated( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path ) -> None: """Updated assemblies are archived before being overwritten.""" - s3 = minio_s3_client + s3 = ceph_s3_client # Seed "old" version at the final Lakehouse path old_files = { @@ -197,10 +197,10 @@ class TestPromoteArchiveRemoved: """Archive and delete replaced/suppressed assemblies.""" def test_archive_removed( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path ) -> None: """Removed assemblies are archived and source objects are deleted.""" - s3 = minio_s3_client + s3 = ceph_s3_client # Seed assemblies at final path files = { @@ -242,9 +242,9 @@ def test_archive_removed( class TestPromoteDryRun: """Dry-run mode should not create any objects.""" - def test_promote_dry_run(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: + def test_promote_dry_run(self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: """Dry-run logs actions but creates no objects at the final path.""" - s3 = minio_s3_client + s3 = ceph_s3_client _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) report = promote_from_s3( @@ -269,12 +269,12 @@ class TestPromoteTrimsManifest: """Manifest trimming removes promoted accessions.""" def test_trims_manifest( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path ) -> None: - """Transfer manifest in MinIO is trimmed to exclude promoted accessions.""" - s3 = minio_s3_client + """Transfer manifest in CEPH is trimmed to exclude promoted accessions.""" + s3 = ceph_s3_client - # Upload a transfer manifest with 3 entries to MinIO (manifest lives in staging) + # Upload a transfer manifest with 3 entries to CEPH (manifest lives in staging) manifest_key = "ncbi/transfer_manifest.txt" manifest_lines = [ "/genomes/all/GCF/900/000/001/GCF_900000001.1_FakeAssemblyA/\n", @@ -297,7 +297,7 @@ def test_trims_manifest( assert report["failed"] == 0 - # Read back the manifest from MinIO (it lives in staging) + # Read back the manifest from CEPH (it lives in staging) resp = s3.get_object(Bucket=staging_test_bucket, Key=manifest_key) remaining = resp["Body"].read().decode() remaining_lines = [line.strip() for line in remaining.strip().splitlines() if line.strip()] @@ -312,9 +312,9 @@ def test_trims_manifest( class TestPromoteIncompleteStaging: """Incomplete staging (sidecar only, no data) should not promote anything.""" - def test_incomplete_staging(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: + def test_incomplete_staging(self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: """Only .md5 sidecars staged → nothing promoted.""" - s3 = minio_s3_client + s3 = ceph_s3_client # Stage only .md5 sidecars (no data files) rel = build_accession_path(ASSEMBLY_DIR_A) @@ -344,9 +344,9 @@ def test_incomplete_staging(self, minio_s3_client: object, test_bucket: str, sta class TestPromoteCreatesDescriptor: """Promote step writes a frictionless descriptor for each promoted assembly.""" - def test_descriptor_created(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: + def test_descriptor_created(self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: """After promote, a JSON descriptor exists under ``metadata/``.""" - s3 = minio_s3_client + s3 = ceph_s3_client _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) promote_from_s3( @@ -364,10 +364,10 @@ def test_descriptor_created(self, minio_s3_client: object, test_bucket: str, sta assert body["resource_type"] == "dataset" def test_descriptor_resources_include_promoted_files( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str ) -> None: """Descriptor's ``resources`` list references the final Lakehouse key.""" - s3 = minio_s3_client + s3 = ceph_s3_client _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) promote_from_s3( @@ -385,10 +385,10 @@ def test_descriptor_resources_include_promoted_files( assert any(PATH_PREFIX + "raw_data/" in p for p in resource_paths) def test_descriptor_resources_have_md5( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str ) -> None: """Resources with .md5 sidecars include the hash value.""" - s3 = minio_s3_client + s3 = ceph_s3_client _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) promote_from_s3( @@ -407,10 +407,10 @@ def test_descriptor_resources_have_md5( assert "hash" in resource, f"Expected hash in resource: {resource}" def test_multiple_assemblies_get_separate_descriptors( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str ) -> None: """Each assembly gets its own descriptor file.""" - s3 = minio_s3_client + s3 = ceph_s3_client _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_B) @@ -434,17 +434,17 @@ class TestPromoteArchiveUpdatedIncludesDescriptor: """Archiving updated assemblies also archives the descriptor.""" def test_archive_copies_descriptor( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path ) -> None: """After archiving an updated assembly, the descriptor appears under archive/.""" - s3 = minio_s3_client + s3 = ceph_s3_client # Seed old version at Lakehouse path *including* a live descriptor old_files = {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": "old content"} seed_lakehouse(s3, test_bucket, ACCESSION_A, old_files, PATH_PREFIX, ASSEMBLY_DIR_A) # Pre-upload a descriptor so archive_descriptor can find it descriptor = create_descriptor(ASSEMBLY_DIR_A, ACCESSION_A, []) - # Upload directly to MinIO (not via promote) + # Upload directly to CEPH (not via promote) descriptor_key = build_descriptor_key(ASSEMBLY_DIR_A, PATH_PREFIX) s3.put_object(Bucket=test_bucket, Key=descriptor_key, Body=json.dumps(descriptor).encode()) @@ -472,10 +472,10 @@ class TestPromoteArchiveRemovedIncludesDescriptor: """Archiving removed assemblies also archives the descriptor.""" def test_archive_removed_copies_descriptor( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path ) -> None: """After archiving a removed assembly, the descriptor is under archive/.""" - s3 = minio_s3_client + s3 = ceph_s3_client # Seed the assembly at final Lakehouse path files = {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": "content"} @@ -506,9 +506,9 @@ def test_archive_removed_copies_descriptor( class TestPromoteDryRunNoDescriptor: """Dry-run must not write any descriptor files.""" - def test_dry_run_no_descriptor(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: + def test_dry_run_no_descriptor(self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: """Dry-run does not upload a descriptor to the metadata/ prefix.""" - s3 = minio_s3_client + s3 = ceph_s3_client _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) promote_from_s3( @@ -532,12 +532,12 @@ class TestArchiveMultiFileConcurrent: """Verify parallel copy archives all files correctly with correct content.""" def test_all_files_archived_with_correct_content( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path ) -> None: """Every file is archived with byte-identical content when copied concurrently.""" from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies - s3 = minio_s3_client + s3 = ceph_s3_client # Seed many files for assembly A at final Lakehouse path many_files = { @@ -572,12 +572,12 @@ def test_all_files_archived_with_correct_content( assert actual_body == expected_body, f"Content mismatch for {fname}" def test_archive_key_paths_are_correct( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path ) -> None: """Archived keys follow the exact ``archive/{release}/{reason}/{rel_path}`` pattern.""" from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies - s3 = minio_s3_client + s3 = ceph_s3_client files = {f"{ASSEMBLY_DIR_B}_genomic.fna.gz": b"content"} seed_lakehouse(s3, test_bucket, ACCESSION_B, files, PATH_PREFIX, ASSEMBLY_DIR_B) @@ -603,12 +603,12 @@ class TestArchiveDeleteSourceBatch: """Verify batch delete removes all source objects after concurrent copy.""" def test_all_sources_deleted_after_archive( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path ) -> None: """After archive with delete_source=True, no source objects remain.""" from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies - s3 = minio_s3_client + s3 = ceph_s3_client many_files = { f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"genomic", f"{ASSEMBLY_DIR_A}_protein.faa.gz": b"protein", @@ -634,12 +634,12 @@ def test_all_sources_deleted_after_archive( assert len(remaining) == 0, f"Source not deleted: {key}" def test_archive_present_source_gone( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path ) -> None: """Archive destinations exist AND sources are gone after replaced_or_suppressed archive.""" from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies - s3 = minio_s3_client + s3 = ceph_s3_client files = { f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"genomic", f"{ASSEMBLY_DIR_A}_protein.faa.gz": b"protein", @@ -676,7 +676,7 @@ class TestPartialArchiveResume: """ def test_partial_updated_archive_resumes( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path ) -> None: """Re-running after a partial updated archive overwrites stale copies and archives missing files. @@ -686,7 +686,7 @@ def test_partial_updated_archive_resumes( """ from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies - s3 = minio_s3_client + s3 = ceph_s3_client rel = build_accession_path(ASSEMBLY_DIR_A) file_a = f"{ASSEMBLY_DIR_A}_genomic.fna.gz" @@ -724,7 +724,7 @@ def test_partial_updated_archive_resumes( assert len(source_keys) == len(current_content) def test_partial_replaced_archive_resumes_and_deletes( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path ) -> None: """Re-running replaced_or_suppressed archive after partial run completes and deletes all sources. @@ -734,7 +734,7 @@ def test_partial_replaced_archive_resumes_and_deletes( """ from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies - s3 = minio_s3_client + s3 = ceph_s3_client rel = build_accession_path(ASSEMBLY_DIR_A) file_a = f"{ASSEMBLY_DIR_A}_genomic.fna.gz" @@ -782,12 +782,12 @@ def test_partial_replaced_archive_resumes_and_deletes( assert resp_a["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 def test_full_rerun_after_complete_archive_is_idempotent( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path ) -> None: """Running archive again when all files already exist at archive paths is safe (no errors).""" from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies - s3 = minio_s3_client + s3 = ceph_s3_client files = { f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"genomic", f"{ASSEMBLY_DIR_A}_protein.faa.gz": b"protein", @@ -832,12 +832,12 @@ class TestArchiveMultiAccessionManifest: """Multiple accessions in a single manifest are all archived.""" def test_two_accessions_both_archived( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path ) -> None: """Both accessions are archived with correct keys when listed in one manifest.""" from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies - s3 = minio_s3_client + s3 = ceph_s3_client files_a = {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"genomic-A"} files_b = {f"{ASSEMBLY_DIR_B}_genomic.fna.gz": b"genomic-B"} @@ -867,12 +867,12 @@ def test_two_accessions_both_archived( assert len(list_all_keys(s3, test_bucket, f"{PATH_PREFIX}{rel_b}")) == 0 def test_three_accessions_correct_archive_reason_segment( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path ) -> None: """Archive keys for all three accessions include the archive_reason segment.""" from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies - s3 = minio_s3_client + s3 = ceph_s3_client accessions_and_dirs = [ (ACCESSION_A, ASSEMBLY_DIR_A), (ACCESSION_B, ASSEMBLY_DIR_B), @@ -911,12 +911,12 @@ class TestArchiveDryRunParallel: """Dry-run with many files leaves everything unchanged.""" def test_dry_run_no_copies_no_deletes( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path ) -> None: """Dry-run with multiple files per accession creates no archive keys and keeps sources.""" from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies - s3 = minio_s3_client + s3 = ceph_s3_client many_files = { f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"genomic", f"{ASSEMBLY_DIR_A}_protein.faa.gz": b"protein", @@ -972,10 +972,10 @@ class TestPromoteMultiFileConcurrent: """Verify concurrent promotion lands all files with correct content and MD5.""" def test_six_files_all_promoted_with_correct_content( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str ) -> None: """Every staged file arrives at the correct final key with byte-identical content.""" - s3 = minio_s3_client + s3 = ceph_s3_client many_files = { f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"GENOMIC", f"{ASSEMBLY_DIR_A}_protein.faa.gz": b"PROTEIN", @@ -1003,10 +1003,10 @@ def test_six_files_all_promoted_with_correct_content( assert obj["Body"].read() == expected_body, f"Content mismatch: {fname}" def test_md5_metadata_correct_per_file( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str ) -> None: """Each promoted file carries MD5 metadata matching its own content, not another file's.""" - s3 = minio_s3_client + s3 = ceph_s3_client files = { f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"GENOMIC_UNIQUE", f"{ASSEMBLY_DIR_A}_protein.faa.gz": b"PROTEIN_UNIQUE", @@ -1028,10 +1028,10 @@ def test_md5_metadata_correct_per_file( assert meta.get("md5") == _md5(content), f"Wrong MD5 metadata on {fname}" def test_file_without_sidecar_has_no_md5_metadata( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str ) -> None: """A file staged without a .md5 sidecar is promoted but has no md5 metadata key.""" - s3 = minio_s3_client + s3 = ceph_s3_client fname = f"{ASSEMBLY_DIR_A}_genomic.fna.gz" _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, {fname: FAKE_GENOMIC}, with_md5=False) @@ -1053,10 +1053,10 @@ class TestPromoteStagingCleanup: """After a fully successful promote, all staged files and sidecars are deleted.""" def test_staged_data_files_deleted( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str ) -> None: """Data files are removed from staging after a successful assembly promote.""" - s3 = minio_s3_client + s3 = ceph_s3_client files = { f"{ASSEMBLY_DIR_A}_genomic.fna.gz": FAKE_GENOMIC, f"{ASSEMBLY_DIR_A}_protein.faa.gz": FAKE_PROTEIN, @@ -1073,9 +1073,9 @@ def test_staged_data_files_deleted( remaining_staging = list_all_keys(s3, staging_test_bucket, STAGING_PREFIX) assert len(remaining_staging) == 0, f"Staging not cleaned: {remaining_staging}" - def test_md5_sidecars_deleted(self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: + def test_md5_sidecars_deleted(self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: """Both data files and .md5 sidecars are removed from staging after promote.""" - s3 = minio_s3_client + s3 = ceph_s3_client files = { f"{ASSEMBLY_DIR_A}_genomic.fna.gz": FAKE_GENOMIC, f"{ASSEMBLY_DIR_A}_protein.faa.gz": FAKE_PROTEIN, @@ -1097,10 +1097,10 @@ def test_md5_sidecars_deleted(self, minio_s3_client: object, test_bucket: str, s assert len(after_keys) == 0, f"Staging not fully cleaned (including sidecars): {after_keys}" def test_two_assemblies_staging_both_cleaned( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str ) -> None: """Staging for both assemblies is fully cleaned when both assemblies succeed.""" - s3 = minio_s3_client + s3 = ceph_s3_client _stage_many( s3, staging_test_bucket, @@ -1135,10 +1135,10 @@ class TestPromoteTwoAssembliesBothLand: """Both assemblies staged together are both promoted to correct Lakehouse paths.""" def test_both_assemblies_at_correct_final_paths( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str ) -> None: """Each assembly's files appear at distinct, correctly-routed final Lakehouse paths.""" - s3 = minio_s3_client + s3 = ceph_s3_client files_a = {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"genomic-A"} files_b = {f"{ASSEMBLY_DIR_B}_genomic.fna.gz": b"genomic-B"} _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, files_a) @@ -1162,10 +1162,10 @@ def test_both_assemblies_at_correct_final_paths( assert obj_b["Body"].read() == b"genomic-B" def test_final_path_keys_do_not_overlap( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str ) -> None: """Files for assembly A and assembly B land at distinct paths — no key collision.""" - s3 = minio_s3_client + s3 = ceph_s3_client _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"a"}) _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_B, {f"{ASSEMBLY_DIR_B}_genomic.fna.gz": b"b"}) @@ -1191,10 +1191,10 @@ class TestPromoteDryRunMultiFile: """dry_run leaves staging untouched and writes nothing to the Lakehouse.""" def test_dry_run_many_files_staging_untouched( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str ) -> None: """All staged files (data + .md5) survive a dry-run promote unchanged.""" - s3 = minio_s3_client + s3 = ceph_s3_client many_files = { f"{ASSEMBLY_DIR_A}_genomic.fna.gz": FAKE_GENOMIC, f"{ASSEMBLY_DIR_A}_protein.faa.gz": FAKE_PROTEIN, @@ -1223,10 +1223,10 @@ def test_dry_run_many_files_staging_untouched( assert len(final_keys) == 0, f"Dry-run created Lakehouse objects: {final_keys}" def test_dry_run_two_assemblies_nothing_written( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str ) -> None: """Dry-run with two staged assemblies creates no Lakehouse objects.""" - s3 = minio_s3_client + s3 = ceph_s3_client _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": FAKE_GENOMIC}) _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_B, {f"{ASSEMBLY_DIR_B}_genomic.fna.gz": FAKE_GENOMIC}) @@ -1247,11 +1247,9 @@ def test_dry_run_two_assemblies_nothing_written( class TestPromoteSecondRunOnEmptyStaging: """After staging is cleaned, a second promote run promotes 0 files without error.""" - def test_second_run_promoted_zero( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str - ) -> None: + def test_second_run_promoted_zero(self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: """Re-running promote on already-cleaned staging succeeds with promoted=0.""" - s3 = minio_s3_client + s3 = ceph_s3_client _stage_many( s3, staging_test_bucket, @@ -1283,10 +1281,10 @@ def test_second_run_promoted_zero( assert len(final_keys) == 1 def test_lakehouse_unchanged_on_second_run( - self, minio_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str ) -> None: """Lakehouse contents are identical before and after a second (no-op) promote run.""" - s3 = minio_s3_client + s3 = ceph_s3_client _stage_many( s3, staging_test_bucket, From 38e549d0cca5b30cb3c3c225b8fc1fed7805df0b Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 3 Jun 2026 14:29:08 -0700 Subject: [PATCH 114/128] add diagnostics and simplify integration action --- .github/workflows/integration_tests.yaml | 32 ++++++++++++++---------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml index 77962029..f9e3a506 100644 --- a/.github/workflows/integration_tests.yaml +++ b/.github/workflows/integration_tests.yaml @@ -24,25 +24,31 @@ jobs: - name: Checkout uses: actions/checkout@v6 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build integration test image - uses: docker/build-push-action@v6 - with: - context: . - load: true - tags: cdm-data-loaders-integration-tests:latest - cache-from: type=gha - cache-to: type=gha,mode=max - - name: Run integration tests run: | docker compose up \ - --no-build \ + --build \ --abort-on-container-exit \ --exit-code-from integration-tests + - name: Capture compose diagnostics on failure + if: failure() + run: | + echo "=== docker compose ps ===" + docker compose ps -a || true + + echo "=== ceph logs ===" + docker logs cdm-data-loaders-ceph-1 || true + + echo "=== integration test logs ===" + docker logs cdm-data-loaders-integration-tests-1 || true + + echo "=== ceph inspect state ===" + docker inspect cdm-data-loaders-ceph-1 --format '{{json .State}}' || true + + echo "=== ceph health state ===" + docker inspect cdm-data-loaders-ceph-1 --format '{{json .State.Health}}' || true + - name: Tear down if: always() run: docker compose down --volumes From abbcf372f73181f97abe74813d49e4131d85f9bb Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 3 Jun 2026 14:40:09 -0700 Subject: [PATCH 115/128] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 115cb895..34c89b15 100644 --- a/README.md +++ b/README.md @@ -211,7 +211,7 @@ Tests are automatically skipped when CEPH is not reachable, so the default `uv r **3. Inspect results:** -Buckets are **not** cleaned up after tests. Browse the CEPH console at [http://localhost:9001](http://localhost:9001) (login: `admin` / `admin`) to inspect the final state of each test bucket. Each test method creates its own bucket (e.g. `integ-test-promote-dry-run`). +Buckets are **not** cleaned up after tests. The CEPH dashboard at [http://localhost:9001](http://localhost:9001) (login: `admin` / `admin`) does not support inspecting bucket contents; use the CLI below (e.g. `scripts/s3_local.py ls/head`) to inspect the final state of each test bucket. Each test method creates its own bucket (e.g. `integ-test-promote-dry-run`). **4. Stop CEPH when done:** From bfecd45f2bc8118e28cdbf1d19e3ee88c7ae51cc Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 3 Jun 2026 14:40:18 -0700 Subject: [PATCH 116/128] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- docs/ncbi_ftp_e2e_walkthrough.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/ncbi_ftp_e2e_walkthrough.md b/docs/ncbi_ftp_e2e_walkthrough.md index ef9eeb87..bb3dd1c8 100644 --- a/docs/ncbi_ftp_e2e_walkthrough.md +++ b/docs/ncbi_ftp_e2e_walkthrough.md @@ -96,10 +96,7 @@ docker run -d \ (Note that a similar service is included in the `docker-compose` configuration file at the root of this repository that is used in CI test workflows.) -Create a test bucket via the [CEPH console](http://localhost:9001) -(login: `admin` / `admin`), or from the command line using the -included `scripts/s3_local.py` helper (requires no extra installs — only -`boto3` which is already a project dependency): +Create test buckets from the command line using the included `scripts/s3_local.py` helper (requires no extra installs — only `boto3` which is already a project dependency): ```sh uv run python scripts/s3_local.py mb s3://cdm-lake From c5cf36a5cb0112397f73f09fd0e2fd34e8f96997 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 3 Jun 2026 14:22:03 -0700 Subject: [PATCH 117/128] Add qsv to docker image for xSV linting/validation/etc. --- Dockerfile | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index a218a8bc..0593b281 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,14 +9,20 @@ FROM ghcr.io/ialarmedalien/xml_file_splitter:latest AS rust-app # Use a Python image with uv pre-installed FROM ghcr.io/astral-sh/uv:python3.13-trixie-slim +ARG QSV_VERSION="20.1.0" + # Set environment variable to noninteractive to prevent prompts during apt operations ENV DEBIAN_FRONTEND=noninteractive # add tini and git -RUN apt-get update -y && apt-get install -y --no-install-recommends tini git ca-certificates && \ - rm -rf /var/lib/apt/lists/* +RUN apt-get update -y && apt-get install -y --no-install-recommends tini git ca-certificates wget unzip && rm -rf /var/lib/apt/lists/* + +# download the qsv binary from the GitHub releases and copy it to /usr/local/bin +RUN wget https://github.com/dathere/qsv/releases/download/${QSV_VERSION}/qsv-${QSV_VERSION}-aarch64-unknown-linux-gnu.zip && \ + unzip qsv-${QSV_VERSION}-aarch64-unknown-linux-gnu.zip -d /usr/local/bin/ && \ + rm qsv-${QSV_VERSION}-aarch64-unknown-linux-gnu.zip -# copy only the compiled xml-file-splitter binary from the Rust image +# copy over the compiled xml-file-splitter binary from the Rust image COPY --from=rust-app /usr/local/bin/xml_file_splitter /usr/local/bin/xml_file_splitter # Setup a non-root user From 252c6dfbb71b2b9dc9971675fd9602cb96f8d9fc Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 3 Jun 2026 14:43:54 -0700 Subject: [PATCH 118/128] address copilot comments --- docs/ncbi_ftp_e2e_walkthrough.md | 2 +- tests/integration/conftest.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/ncbi_ftp_e2e_walkthrough.md b/docs/ncbi_ftp_e2e_walkthrough.md index bb3dd1c8..32d39bea 100644 --- a/docs/ncbi_ftp_e2e_walkthrough.md +++ b/docs/ncbi_ftp_e2e_walkthrough.md @@ -395,7 +395,7 @@ cdm-lake/ ## 5. Inspect results in CEPH -Browse the [CEPH console](http://localhost:9001) or use the CLI: +Use the CLI to inspect the final state of the store: ```sh # List final Lakehouse objects diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 0d010184..c8ab3354 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -3,7 +3,8 @@ Integration tests are auto-skipped when CEPH is not reachable. Each test method gets its own bucket (derived from the test node name) that is emptied on re-run but **never deleted** after the test — this lets developers inspect -the final state of the object store via the CEPH console. +the final state of the object store. The CEPH dashboard does not currently +allow inspection of the store, but the `s3_local` command-line tool can be used. """ import hashlib From 203f6c72d58ff44c5be3e3504716e7a0d223812d Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 3 Jun 2026 15:03:31 -0700 Subject: [PATCH 119/128] update s3 credential test conditions --- tests/utils/test_s3.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index d9191b2f..78455beb 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -218,6 +218,7 @@ def test_get_s3_client_incomplete_creds_via_args( reset_s3_client() assert s3_utils._s3_client is None # noqa: SLF001 args = { + "endpoint_url": None, "aws_access_key_id": aws_access_key_id, "aws_secret_access_key": aws_secret_access_key, } From 2837bb030d8756e315b0fa860a2fd8abb5f339b4 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 3 Jun 2026 15:24:19 -0700 Subject: [PATCH 120/128] reset s3 creds for testing in actions --- tests/utils/test_s3.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index 78455beb..e2c8536f 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -212,9 +212,16 @@ def test_get_s3_client_success_via_env(endpoint_url: str | None, monkeypatch: py ("aws_access_key_id", "aws_secret_access_key"), [("aws_access_key_id", None), (None, "aws_secret_access_key")] ) def test_get_s3_client_incomplete_creds_via_args( - aws_access_key_id: str | None, aws_secret_access_key: str | None + aws_access_key_id: str | None, aws_secret_access_key: str | None, monkeypatch: pytest.MonkeyPatch ) -> None: """Verify that get_s3_client raises ValueError when only one of aws_access_key_id or aws_secret_access_key is provided via arguments.""" + # Remove any real endpoint/credential env vars so moto intercepts all HTTP calls. + monkeypatch.delenv("AWS_ACCESS_KEY_ID", raising=False) + monkeypatch.delenv("AWS_SECRET_ACCESS_KEY", raising=False) + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + monkeypatch.delenv("AWS_ENDPOINT_URL", raising=False) + monkeypatch.delenv("AWS_ENDPOINT_URL_S3", raising=False) + boto3.DEFAULT_SESSION = None reset_s3_client() assert s3_utils._s3_client is None # noqa: SLF001 args = { From 84d7492b9e76b7f51e150d88e6396df47f759545 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 3 Jun 2026 15:40:17 -0700 Subject: [PATCH 121/128] update s3 test conditions --- tests/utils/test_s3.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index e2c8536f..127cfdec 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -215,17 +215,10 @@ def test_get_s3_client_incomplete_creds_via_args( aws_access_key_id: str | None, aws_secret_access_key: str | None, monkeypatch: pytest.MonkeyPatch ) -> None: """Verify that get_s3_client raises ValueError when only one of aws_access_key_id or aws_secret_access_key is provided via arguments.""" - # Remove any real endpoint/credential env vars so moto intercepts all HTTP calls. - monkeypatch.delenv("AWS_ACCESS_KEY_ID", raising=False) - monkeypatch.delenv("AWS_SECRET_ACCESS_KEY", raising=False) - monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") - monkeypatch.delenv("AWS_ENDPOINT_URL", raising=False) - monkeypatch.delenv("AWS_ENDPOINT_URL_S3", raising=False) - boto3.DEFAULT_SESSION = None + prep_client_init(monkeypatch) reset_s3_client() assert s3_utils._s3_client is None # noqa: SLF001 args = { - "endpoint_url": None, "aws_access_key_id": aws_access_key_id, "aws_secret_access_key": aws_secret_access_key, } From 80faf1840ae556ab4b286ac7220940554cf3311f Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Wed, 3 Jun 2026 15:58:02 -0700 Subject: [PATCH 122/128] reset s3 creds in test --- tests/utils/test_s3.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/utils/test_s3.py b/tests/utils/test_s3.py index 127cfdec..c2d508e0 100644 --- a/tests/utils/test_s3.py +++ b/tests/utils/test_s3.py @@ -256,6 +256,7 @@ def test_get_s3_client_incomplete_creds_via_env( monkeypatch: pytest.MonkeyPatch, ) -> None: """Verify that get_s3_client raises ValueError when only one of aws_access_key_id or aws_secret_access_key is provided.""" + prep_client_init(monkeypatch) reset_s3_client() assert s3_utils._s3_client is None # noqa: SLF001 args = { From aa85b92b63e2637b28be3d8ae2b1a1dcfcad7916 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Thu, 4 Jun 2026 09:05:11 -0700 Subject: [PATCH 123/128] Just download binaries of xml_file_splitter and qsv to install in container --- Dockerfile | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0593b281..0a7c8a78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,13 +3,11 @@ # Dockerfile is based heavily on the example uv dockerfile: # https://github.com/astral-sh/uv-docker-example -# Pull the pre-built Rust app from ghcr.io -FROM ghcr.io/ialarmedalien/xml_file_splitter:latest AS rust-app - # Use a Python image with uv pre-installed FROM ghcr.io/astral-sh/uv:python3.13-trixie-slim ARG QSV_VERSION="20.1.0" +ARG XML_FILE_SPLITTER_VERSION="v0.1.2" # Set environment variable to noninteractive to prevent prompts during apt operations ENV DEBIAN_FRONTEND=noninteractive @@ -17,13 +15,17 @@ ENV DEBIAN_FRONTEND=noninteractive # add tini and git RUN apt-get update -y && apt-get install -y --no-install-recommends tini git ca-certificates wget unzip && rm -rf /var/lib/apt/lists/* -# download the qsv binary from the GitHub releases and copy it to /usr/local/bin -RUN wget https://github.com/dathere/qsv/releases/download/${QSV_VERSION}/qsv-${QSV_VERSION}-aarch64-unknown-linux-gnu.zip && \ - unzip qsv-${QSV_VERSION}-aarch64-unknown-linux-gnu.zip -d /usr/local/bin/ && \ - rm qsv-${QSV_VERSION}-aarch64-unknown-linux-gnu.zip - -# copy over the compiled xml-file-splitter binary from the Rust image -COPY --from=rust-app /usr/local/bin/xml_file_splitter /usr/local/bin/xml_file_splitter +WORKDIR /tmp + +# download and install the xml_file_splitter and qsv binaries, and copy them to /usr/local/bin +RUN wget https://github.com/ialarmedalien/xml_file_splitter/releases/download/${XML_FILE_SPLITTER_VERSION}/xml_file_splitter-aarch64-unknown-linux-gnu.tar.gz && \ + tar -xvf xml_file_splitter-aarch64-unknown-linux-gnu.tar.gz && \ + mv xml_file_splitter-aarch64-unknown-linux-gnu/xml_file_splitter /usr/local/bin/ && \ + # qsv release -- only need the `qsv` binary from it + wget https://github.com/dathere/qsv/releases/download/${QSV_VERSION}/qsv-${QSV_VERSION}-aarch64-unknown-linux-gnu.zip && \ + unzip qsv-${QSV_VERSION}-aarch64-unknown-linux-gnu.zip -d /tmp/qsv && \ + mv /tmp/qsv/qsv /usr/local/bin/ && \ + rm -rf /tmp/* # Setup a non-root user RUN groupadd --system --gid 999 nonroot \ From 9ba7ab6d5c19972ce3ece4055b017167e35a0fb3 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 9 Jun 2026 10:30:59 -0700 Subject: [PATCH 124/128] Add decompression of gz file util --- src/cdm_data_loaders/utils/gz.py | 29 +++++++++ tests/utils/test_gz.py | 104 +++++++++++++++++++++++++++++-- 2 files changed, 128 insertions(+), 5 deletions(-) diff --git a/src/cdm_data_loaders/utils/gz.py b/src/cdm_data_loaders/utils/gz.py index 00bac119..83564f38 100644 --- a/src/cdm_data_loaders/utils/gz.py +++ b/src/cdm_data_loaders/utils/gz.py @@ -11,6 +11,35 @@ logger = get_cdm_logger() +def decompress_file(file: Path) -> None: + """Decompress a gzip file. + + :param file: file to decompress + :type file: Path + """ + if file.suffix != ".gz": + logger.info("File %s does not end with .gz: skipping decompression", str(file)) + return + + if not file.exists(): + logger.warning("File %s does not exist: skipping decompression", str(file)) + return + + if not file.is_file(): + logger.warning("%s is not a file: skipping decompression", str(file)) + return + + # remove .gz suffix + output_file = file.with_suffix("") + if output_file.exists(): + logger.info("Found existing file %s: skipping decompression", output_file) + return + + with gzip.open(file, "rb") as f_in, output_file.open("wb") as f_out: + shutil.copyfileobj(f_in, f_out) + logger.info("Created output file %s", output_file) + + def compress_files(directory: Path | str, file_glob: str) -> None: """Compress all files matching a certain pattern in a directory. diff --git a/tests/utils/test_gz.py b/tests/utils/test_gz.py index b3276dbc..0f8acda8 100644 --- a/tests/utils/test_gz.py +++ b/tests/utils/test_gz.py @@ -7,15 +7,25 @@ import pytest from click.testing import CliRunner -from cdm_data_loaders.utils.gz import compress_file, compress_files, main +from cdm_data_loaders.utils.gz import compress_file, compress_files, decompress_file, main + +TEMP_FILE_CONTENT = b"Hello, world!\nThis is a test." @pytest.fixture def temporary_file(tmp_path: Path) -> Path: """Create a temporary text file and return its path.""" file_path = tmp_path / "sample.txt" - content = b"Hello, world!\nThis is a test." - file_path.write_bytes(content) + file_path.write_bytes(TEMP_FILE_CONTENT) + return file_path + + +@pytest.fixture +def temporary_gz(tmp_path: Path) -> Path: + """Create a temporary gzipped text file and return its path.""" + file_path = tmp_path / "sample.txt.gz" + with gzip.open(file_path, "wb") as f: + f.write(TEMP_FILE_CONTENT) return file_path @@ -29,8 +39,7 @@ def test_compress_file_creates_gzip(temporary_file: Path, caplog: pytest.LogCapt """Test that running compress file actually compresses a file.""" # Ensure no .gz exists initially gz_path = temporary_file.with_suffix(temporary_file.suffix + ".gz") - if gz_path.exists(): - gz_path.unlink() + assert not gz_path.exists() compress_file(temporary_file) @@ -135,6 +144,91 @@ def test_compress_files_invalid_directory(tmp_path: Path) -> None: compress_files(non_existent, "*") +def test_decompress_file_creates_file(temporary_gz: Path, caplog: pytest.LogCaptureFixture) -> None: + """Test that running decompress file actually decompresses a file.""" + # Ensure no decompressed file exists initially + decompressed_file = temporary_gz.with_suffix("") + assert not decompressed_file.exists() + assert temporary_gz.exists() + + decompress_file(temporary_gz) + + # Verify the decompressed file was created + assert decompressed_file.exists() + + # compare content of the decompressed file + assert decompressed_file.read_bytes() == TEMP_FILE_CONTENT + + assert len(caplog.records) == 1 + assert caplog.records[0].levelno == logging.INFO + assert f"Created output file {decompressed_file!s}" in caplog.records[0].message + + +@pytest.mark.parametrize("file_name", ["data", "data.tar", "not_a_gz", "file.gz.bak"]) +def test_decompress_file_skips_non_gz_file(tmp_path: Path, file_name: str, caplog: pytest.LogCaptureFixture) -> None: + """Ensure that an existing decompressed file prevents decompress_file from running decompression.""" + file_path = tmp_path / file_name + # attempt to decompress data.bin.gz + decompress_file(file_path) + + assert len(caplog.records) == 1 + assert caplog.records[0].levelno == logging.INFO + assert f"File {file_path!s} does not end with .gz: skipping decompression" in caplog.records[0].message + + +def test_decompress_file_skips_directory(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + """Ensure that if the supplied path is a directory, decompression does not occur.""" + file_path = tmp_path / "some_directory.gz" + file_path.mkdir(parents=True, exist_ok=True) + assert file_path.exists() + assert file_path.is_dir() + + decompress_file(file_path) + + assert len(caplog.records) == 1 + assert caplog.records[0].levelno == logging.WARNING + assert f"{file_path!s} is not a file: skipping decompression" in caplog.records[0].message + + +def test_decompress_file_skips_missing_file(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + """Ensure that a missing file prevents decompress_file from running decompression.""" + missing_file = tmp_path / "missing.gz" + decompress_file(missing_file) + + assert len(caplog.records) == 1 + assert caplog.records[0].levelno == logging.WARNING + assert f"File {missing_file!s} does not exist: skipping decompression" in caplog.records[0].message + + +@pytest.mark.parametrize("file_name", ["data", "data.tar", "not_a_gz", "file.gz.bak"]) +def test_decompress_file_skips_existing_decompressed_file( + tmp_path: Path, file_name: str, caplog: pytest.LogCaptureFixture +) -> None: + """Ensure that an existing decompressed file prevents decompress_file from running decompression.""" + file_path = tmp_path / file_name + file_path.write_bytes(b"binary\x00data") + assert file_path.name == file_name + assert file_path.exists() + + original_mtime = file_path.stat().st_mtime + # fake gzip file + gz_path = file_path.with_suffix(file_path.suffix + ".gz") + gz_path.write_bytes(b"some old crap") + assert gz_path.name == file_name + ".gz" + assert gz_path.exists() + + # attempt to decompress data.bin.gz + decompress_file(gz_path) + + # Ensure the .gz file was not overwritten (mtime unchanged) + assert file_path.stat().st_mtime == original_mtime + assert file_path.read_bytes() == b"binary\x00data" + + assert len(caplog.records) == 1 + assert caplog.records[0].levelno == logging.INFO + assert f"Found existing file {file_path!s}: skipping decompression" in caplog.records[0].message + + @pytest.mark.skip("CliRunner conflicts with logging, causing a ValueError") def test_cli_single_file(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: """Test the CLI with a valid file.""" From 8b820b58aff7caf9433816f21fb6291dae5163e8 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Tue, 9 Jun 2026 12:00:05 -0700 Subject: [PATCH 125/128] Update CEPH-backed integration tests (#177) * relabel ceph tests; let fail if ceph not reachable * address copilot comments * update test script to exclude ceph tests * update test script to exclude ceph tests * fail fast when ceph unreachable * clean up test imports * formatting --------- Co-authored-by: i alarmed alien --- .github/workflows/integration_tests.yaml | 8 ++--- .github/workflows/tests.yml | 2 +- README.md | 14 ++++---- docker-compose.yml | 2 +- pyproject.toml | 2 +- scripts/entrypoint.sh | 10 +++--- scripts/run_tests.sh | 2 +- tests/integration/conftest.py | 26 +++++++------- tests/integration/test_download_e2e.py | 8 ++--- tests/integration/test_full_pipeline.py | 17 ++++----- tests/integration/test_manifest_e2e.py | 10 +++--- tests/integration/test_promote_e2e.py | 46 ++++++++++++------------ 12 files changed, 69 insertions(+), 78 deletions(-) diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml index f9e3a506..0f0e5177 100644 --- a/.github/workflows/integration_tests.yaml +++ b/.github/workflows/integration_tests.yaml @@ -1,4 +1,4 @@ -name: Integration tests +name: CEPH Integration tests on: workflow_call: @@ -17,14 +17,14 @@ permissions: contents: read jobs: - integration_tests: + ceph_integration_tests: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v6 - - name: Run integration tests + - name: Run CEPH-backed integration tests run: | docker compose up \ --build \ @@ -40,7 +40,7 @@ jobs: echo "=== ceph logs ===" docker logs cdm-data-loaders-ceph-1 || true - echo "=== integration test logs ===" + echo "=== ceph integration test logs ===" docker logs cdm-data-loaders-integration-tests-1 || true echo "=== ceph inspect state ===" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 98287e6b..0cf56526 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -91,4 +91,4 @@ jobs: - name: Run local tests shell: bash continue-on-error: false - run: uv run pytest -m "not requires_spark" + run: uv run pytest -m "not requires_spark and not requires_ceph" diff --git a/README.md b/README.md index 34c89b15..59813af5 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Repo for CDM input data loading and wrangling - [Development](#development) - [Spark and other non-python dependencies](#spark-and-other-non-python-dependencies) - [Tests](#tests) - - [Integration tests (CEPH + NCBI FTP)](#integration-tests-ceph--ncbi-ftp) + - [CEPH-Backed integration tests (CEPH + NCBI FTP)](#ceph-backed-integration-tests-ceph--ncbi-ftp) - [Loading genomes, contigs, and features](#loading-genomes-contigs-and-features) - [Running bbmap stats and checkm2 on genome or contigset files](#running-bbmap-stats-and-checkm2-on-genome-or-contigset-files) @@ -142,16 +142,16 @@ See the [BERDataLakehouse/spark_notebook](https://github.com/BERDataLakehouse/sp Tests are categorised using pytest markers to allow developers to execute some or all the tests. See [pyproject.toml](pyproject.toml) for the markers used. -To run all tests (requires a running Spark instance), execute the command: +To run all tests (requires a running Spark instance and a running CEPH test container), execute the command: ```sh > uv run pytest ``` -To run only tests that do not require Spark, run +To run only tests that do not require Spark or CEPH, run ```sh -> uv run pytest -m "not requires_spark" +> uv run pytest -m "not requires_spark and not requires_ceph" ``` To generate coverage for the tests, run @@ -162,7 +162,7 @@ To generate coverage for the tests, run The standard python `coverage` package is used and coverage can be generated as html or other formats by changing the parameters. -#### Integration tests (CEPH + NCBI FTP) +#### CEPH-Backed integration tests (CEPH + NCBI FTP) End-to-end integration tests for the NCBI assembly pipeline live in `tests/integration/`. They exercise the full flow — manifest diffing, FTP download, S3 promote/archive — against a locally running [CEPH](https://ceph.io/) container and the real NCBI FTP server. @@ -204,11 +204,9 @@ docker run -d \ **2. Run the integration tests:** ```sh -> uv run pytest tests/integration/ -m integration -v +> uv run pytest tests/integration/ -m requires_ceph -v ``` -Tests are automatically skipped when CEPH is not reachable, so the default `uv run pytest` will never fail due to a missing CEPH instance. - **3. Inspect results:** Buckets are **not** cleaned up after tests. The CEPH dashboard at [http://localhost:9001](http://localhost:9001) (login: `admin` / `admin`) does not support inspecting bucket contents; use the CLI below (e.g. `scripts/s3_local.py ls/head`) to inspect the final state of each test bucket. Each test method creates its own bucket (e.g. `integ-test-promote-dry-run`). diff --git a/docker-compose.yml b/docker-compose.yml index 73239988..6bef710c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,5 +27,5 @@ services: AWS_ENDPOINT_URL: http://ceph:8080 AWS_ACCESS_KEY_ID: test_access_key AWS_SECRET_ACCESS_KEY: test_access_secret - entrypoint: [ "/app/scripts/entrypoint.sh", "integration_test" ] + entrypoint: [ "/app/scripts/entrypoint.sh", "ceph_integration_test" ] command: [] diff --git a/pyproject.toml b/pyproject.toml index 323823d4..66c4f20f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -191,7 +191,7 @@ log_cli = true log_cli_level = "INFO" log_level = "INFO" addopts = ["-v"] -markers = ["requires_spark: must be run in an environment where spark is available", "s3: tests that mock s3 interactions", "slow_test: does what it says on the tin", "integration: end-to-end tests requiring a running CEPH instance and network access", "external_request: tests that make real network requests to external services (e.g. NCBI FTP)"] +markers = ["requires_spark: must be run in an environment where spark is available", "s3: tests that mock s3 interactions", "slow_test: does what it says on the tin", "requires_ceph: end-to-end tests requiring a running CEPH instance and network access", "external_request: tests that make real network requests to external services (e.g. NCBI FTP)"] # environment settings for running tests [tool.pytest_env] diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index 91f35367..60ea113d 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -VALID_COMMANDS=(all_the_bacteria ncbi_ftp_sync ncbi_rest_api uniprot uniref xml_split test integration_test bash) +VALID_COMMANDS=(all_the_bacteria ncbi_ftp_sync ncbi_rest_api uniprot uniref xml_split test ceph_integration_test bash) usage() { local joined @@ -38,11 +38,11 @@ case "$cmd" in exec /usr/bin/tini -- xml_file_splitter "$@" ;; test) - exec /usr/bin/tini -- uv run --no-sync pytest -m "not requires_spark" + exec /usr/bin/tini -- uv run --no-sync pytest -m "not requires_spark and not requires_ceph" ;; - integration_test) - # run the integration tests (requires a running CEPH instance) - exec /usr/bin/tini -- uv run --no-sync pytest -m "integration" -v "$@" + ceph_integration_test) + # run the CEPH-backed integration tests (requires a running CEPH instance) + exec /usr/bin/tini -- uv run --no-sync pytest -m "requires_ceph" -v "$@" ;; bash) exec /usr/bin/tini -- /bin/bash diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 0c55bb0c..8c1abf3c 100644 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -10,4 +10,4 @@ cd "$SCRIPT_DIR" uv venv --system-site-packages # run the tests using the active venv and with the dev dependencies installed. -uv run --active --frozen --group dev pytest --cov=src --cov-report=xml +uv run --active --frozen --group dev pytest -m "not requires_ceph" --cov=src --cov-report=xml diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index c8ab3354..48ce113e 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,9 +1,8 @@ """Shared fixtures and helpers for CEPH-backed integration tests. -Integration tests are auto-skipped when CEPH is not reachable. Each test -method gets its own bucket (derived from the test node name) that is emptied -on re-run but **never deleted** after the test — this lets developers inspect -the final state of the object store. The CEPH dashboard does not currently +Each test method gets its own bucket (derived from the test node name) that is +emptied on re-run but **never deleted** after the test — this lets developers +inspect the final state of the object store. The CEPH dashboard does not currently allow inspection of the store, but the `s3_local` command-line tool can be used. """ @@ -50,17 +49,20 @@ def _ceph_reachable() -> bool: return True -def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: # noqa: ARG001 - """Auto-skip ``@pytest.mark.integration`` tests when CEPH is unreachable.""" +def pytest_runtest_setup(item: pytest.Item) -> None: + """Fail CEPH-required tests early when CEPH is unavailable.""" + if "requires_ceph" not in item.keywords: + return + global _ceph_available # noqa: PLW0603 if _ceph_available is None: _ceph_available = _ceph_reachable() - if _ceph_available: - return - skip_marker = pytest.mark.skip(reason="CEPH not reachable — skipping integration tests") - for item in items: - if "integration" in item.keywords: - item.add_marker(skip_marker) + + if not _ceph_available: + pytest.fail( + "CEPH not reachable. Start a CEPH test store or deselect with -m 'not requires_ceph'.", + pytrace=False, + ) # Fixtures diff --git a/tests/integration/test_download_e2e.py b/tests/integration/test_download_e2e.py index 2128b6b5..74e39848 100644 --- a/tests/integration/test_download_e2e.py +++ b/tests/integration/test_download_e2e.py @@ -1,8 +1,8 @@ """End-to-end tests for Phase 2 — FTP download of assemblies. These tests download real (small) assemblies from the NCBI FTP server. -Marked ``integration`` and ``slow_test``; auto-skipped when CEPH is -unreachable. +Marked ``requires_ceph`` (if a running CEPH test store is required) and +``slow_test``. """ import json @@ -44,7 +44,6 @@ def _manifest_for_one_assembly(tmp_path: Path) -> tuple[Path, str]: return manifest_path, diff.new[0] -@pytest.mark.integration @pytest.mark.slow_test @pytest.mark.external_request class TestDownloadSmallBatch: @@ -87,7 +86,6 @@ def test_download_small_batch(self, tmp_path: Path) -> None: assert saved_report["succeeded"] >= 1 -@pytest.mark.integration @pytest.mark.slow_test @pytest.mark.external_request class TestDownloadResumeIncomplete: @@ -128,7 +126,7 @@ def test_download_resume(self, tmp_path: Path) -> None: assert files_after_first.issubset(files_after_second) -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test @pytest.mark.external_request def test_download_and_stage_e2e( diff --git a/tests/integration/test_full_pipeline.py b/tests/integration/test_full_pipeline.py index e524f16a..08a2c9ab 100644 --- a/tests/integration/test_full_pipeline.py +++ b/tests/integration/test_full_pipeline.py @@ -3,19 +3,14 @@ Exercises the entire flow: download summary from real NCBI FTP, compute diff, download a single assembly, stage in CEPH, promote to final Lakehouse path. -Marked ``integration`` and ``slow_test``; auto-skipped when CEPH is -unreachable. +Marked ``requires_ceph`` (when a runnning CEPH test store is required) and +``slow_test``. """ -from __future__ import annotations - -from typing import TYPE_CHECKING +from pathlib import Path import pytest -if TYPE_CHECKING: - from pathlib import Path - from cdm_data_loaders.ncbi_ftp.manifest import ( AssemblyRecord, compute_diff, @@ -29,14 +24,14 @@ from cdm_data_loaders.ncbi_ftp.promote import DEFAULT_LAKEHOUSE_KEY_PREFIX, promote_from_s3 from cdm_data_loaders.pipelines.ncbi_ftp_download import download_batch -from .conftest import get_object_metadata, list_all_keys, stage_files_to_ceph, staging_test_bucket # noqa: F401 +from .conftest import get_object_metadata, list_all_keys, stage_files_to_ceph STABLE_PREFIX = "900" STAGING_PREFIX = "staging/run1/" PATH_PREFIX = DEFAULT_LAKEHOUSE_KEY_PREFIX -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test @pytest.mark.external_request class TestFullPipelineSmallBatch: @@ -104,7 +99,7 @@ def test_full_pipeline_small_batch( assert has_md5, "Expected at least one file with MD5 metadata" -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test @pytest.mark.external_request class TestFullPipelineIncrementalSync: diff --git a/tests/integration/test_manifest_e2e.py b/tests/integration/test_manifest_e2e.py index 8c2ed8c7..6cf2851b 100644 --- a/tests/integration/test_manifest_e2e.py +++ b/tests/integration/test_manifest_e2e.py @@ -1,8 +1,8 @@ """End-to-end tests for Phase 1 — manifest generation and diffing. These tests hit the real NCBI FTP server (with tight prefix filters) and -optionally use CEPH for checksum verification. Marked ``integration`` -and ``slow_test``; auto-skipped when CEPH is unreachable. +optionally use CEPH for checksum verification. Marked ``requires_ceph`` +(if a running CEPH test store is required) and ``slow_test``. """ from __future__ import annotations @@ -56,7 +56,6 @@ def _download_and_filter() -> tuple[dict[str, AssemblyRecord], dict[str, Assembl # Tests -@pytest.mark.integration @pytest.mark.slow_test @pytest.mark.external_request class TestFreshSyncNoPrevious: @@ -94,7 +93,6 @@ def test_fresh_sync_no_previous(self, tmp_path: Path) -> None: assert summary_path.exists() -@pytest.mark.integration @pytest.mark.slow_test @pytest.mark.external_request class TestIncrementalDiffSyntheticPrevious: @@ -157,7 +155,7 @@ def test_incremental_diff(self, tmp_path: Path) -> None: assert updated_acc in updated_list -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test @pytest.mark.external_request class TestVerifyTransferCandidatesPrunes: @@ -238,7 +236,7 @@ def test_prunes_existing_matching_md5( assert c in result, f"Expected {c} to remain (not seeded)" -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestScanStoreToSyntheticSummary: """Test synthetic assembly summary generation from CEPH store.""" diff --git a/tests/integration/test_promote_e2e.py b/tests/integration/test_promote_e2e.py index 98520b1c..05d17f0a 100644 --- a/tests/integration/test_promote_e2e.py +++ b/tests/integration/test_promote_e2e.py @@ -4,8 +4,8 @@ with various combinations of manifests, archive operations, dry-run mode, manifest trimming, and incomplete staging. -Marked ``integration`` and ``slow_test``; auto-skipped when CEPH is -unreachable. Each test method gets its own bucket. +Marked ``requires_ceph`` (when a running CEPH test store is required) and +``slow_test``. Each test method gets its own bucket. """ import hashlib @@ -76,7 +76,7 @@ def _write_manifest(tmp_path: Path, accessions: list[str], name: str) -> Path: # Tests -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestPromoteFromStaging: """Promote staged files to final Lakehouse paths.""" @@ -107,7 +107,7 @@ def test_promote_from_staging(self, ceph_s3_client: object, test_bucket: str, st assert "md5" in meta, f"Missing md5 metadata on {key}" -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestPromoteIdempotent: """Promoting the same staging data twice should succeed without errors.""" @@ -145,7 +145,7 @@ def test_promote_idempotent(self, ceph_s3_client: object, test_bucket: str, stag assert keys_after_first == keys_after_second -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestPromoteArchiveUpdated: """Archive existing assemblies before overwriting with updated versions.""" @@ -191,7 +191,7 @@ def test_archive_updated( assert "/2024-01/" in key -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestPromoteArchiveRemoved: """Archive and delete replaced/suppressed assemblies.""" @@ -237,7 +237,7 @@ def test_archive_removed( assert len(source_keys) == 0, f"Expected source objects deleted, found: {source_keys}" -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestPromoteDryRun: """Dry-run mode should not create any objects.""" @@ -263,7 +263,7 @@ def test_promote_dry_run(self, ceph_s3_client: object, test_bucket: str, staging assert len(final_keys) == 0, f"Dry-run should not create objects, found: {final_keys}" -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestPromoteTrimsManifest: """Manifest trimming removes promoted accessions.""" @@ -307,7 +307,7 @@ def test_trims_manifest( assert "GCF_900000003" in remaining_lines[0] -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestPromoteIncompleteStaging: """Incomplete staging (sidecar only, no data) should not promote anything.""" @@ -339,7 +339,7 @@ def test_incomplete_staging(self, ceph_s3_client: object, test_bucket: str, stag assert len(final_keys) == 0 -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestPromoteCreatesDescriptor: """Promote step writes a frictionless descriptor for each promoted assembly.""" @@ -428,7 +428,7 @@ def test_multiple_assemblies_get_separate_descriptors( assert body["identifier"] == f"NCBI:{accession}" -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestPromoteArchiveUpdatedIncludesDescriptor: """Archiving updated assemblies also archives the descriptor.""" @@ -466,7 +466,7 @@ def test_archive_copies_descriptor( assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestPromoteArchiveRemovedIncludesDescriptor: """Archiving removed assemblies also archives the descriptor.""" @@ -501,7 +501,7 @@ def test_archive_removed_copies_descriptor( assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestPromoteDryRunNoDescriptor: """Dry-run must not write any descriptor files.""" @@ -526,7 +526,7 @@ def test_dry_run_no_descriptor(self, ceph_s3_client: object, test_bucket: str, s # Parallel archiving tests -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestArchiveMultiFileConcurrent: """Verify parallel copy archives all files correctly with correct content.""" @@ -597,7 +597,7 @@ def test_archive_key_paths_are_correct( assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestArchiveDeleteSourceBatch: """Verify batch delete removes all source objects after concurrent copy.""" @@ -665,7 +665,7 @@ def test_archive_present_source_gone( assert len(source_keys) == 0, f"Source objects remain: {source_keys}" -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestPartialArchiveResume: """Corner case: a prior archive run was interrupted mid-way. @@ -826,7 +826,7 @@ def test_full_rerun_after_complete_archive_is_idempotent( assert obj["Body"].read() == expected_body -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestArchiveMultiAccessionManifest: """Multiple accessions in a single manifest are all archived.""" @@ -905,7 +905,7 @@ def test_three_accessions_correct_archive_reason_segment( assert "/2024-03/" in key, f"Archive key missing release segment: {key}" -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestArchiveDryRunParallel: """Dry-run with many files leaves everything unchanged.""" @@ -966,7 +966,7 @@ def _stage_many( s3.put_object(Bucket=bucket, Key=f"{key}.md5", Body=_md5(content).encode()) -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestPromoteMultiFileConcurrent: """Verify concurrent promotion lands all files with correct content and MD5.""" @@ -1047,7 +1047,7 @@ def test_file_without_sidecar_has_no_md5_metadata( assert "md5" not in meta, f"Expected no md5 metadata, got: {meta}" -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestPromoteStagingCleanup: """After a fully successful promote, all staged files and sidecars are deleted.""" @@ -1129,7 +1129,7 @@ def test_two_assemblies_staging_both_cleaned( assert len(remaining) == 0, f"Staging not fully cleaned after two-assembly promote: {remaining}" -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestPromoteTwoAssembliesBothLand: """Both assemblies staged together are both promoted to correct Lakehouse paths.""" @@ -1185,7 +1185,7 @@ def test_final_path_keys_do_not_overlap( assert keys_a[0] != keys_b[0] -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestPromoteDryRunMultiFile: """dry_run leaves staging untouched and writes nothing to the Lakehouse.""" @@ -1242,7 +1242,7 @@ def test_dry_run_two_assemblies_nothing_written( assert len(final_keys) == 0, f"Dry-run created objects: {final_keys}" -@pytest.mark.integration +@pytest.mark.requires_ceph @pytest.mark.slow_test class TestPromoteSecondRunOnEmptyStaging: """After staging is cleaned, a second promote run promotes 0 files without error.""" From 62f0a1daf92759fed4e4efb08298b6619df39992 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Wed, 10 Jun 2026 13:40:47 -0700 Subject: [PATCH 126/128] Add option to NCBI REST pipeline to allow dataset and annotation endpoints to be run separately --- .../pipelines/ncbi_rest_api.py | 102 +++++++-- tests/pipelines/test_ncbi_rest_api.py | 208 +++++++++++++++--- 2 files changed, 258 insertions(+), 52 deletions(-) diff --git a/src/cdm_data_loaders/pipelines/ncbi_rest_api.py b/src/cdm_data_loaders/pipelines/ncbi_rest_api.py index f41d0acf..7d656988 100644 --- a/src/cdm_data_loaders/pipelines/ncbi_rest_api.py +++ b/src/cdm_data_loaders/pipelines/ncbi_rest_api.py @@ -2,6 +2,7 @@ import logging import os +import re from collections.abc import Generator from functools import partial from itertools import islice @@ -17,7 +18,7 @@ from dlt.sources.helpers.rest_client.paginators import ( JSONResponseCursorPaginator, ) -from pydantic import AliasChoices, Field +from pydantic import AliasChoices, Field, field_validator from pydantic_settings import SettingsConfigDict from requests.exceptions import HTTPError @@ -34,16 +35,22 @@ # Max number of items to request per page from the NCBI REST API (max allowed is 1000). MAX_RESULTS_PER_PAGE = 1000 +# Max number of IDs to send in a multi-ID query +# Max URL length seems to be 4611 (as of Jun 2026) +MAX_IDS_PER_QUERY = 250 DATASET = "dataset" ANNOTATION = "annotation" ERROR = "error" +QUERY_TYPE_REGEX = re.compile(r"^(" + DATASET + r"|" + ANNOTATION + r")$") + logger = logging.getLogger("dlt") REST_CLIENT_HOOKS = {} ARG_ALIAS_BATCH_SIZE = ["-b", "--batch-size", "--batch_size"] +ARG_ALIAS_QUERY_TYPE = ["-q", "--query-type", "--query_type"] class NcbiSettings(CtsSettings): @@ -52,13 +59,32 @@ class NcbiSettings(CtsSettings): model_config = SettingsConfigDict(**DEFAULT_SETTINGS_CONFIG_DICT, cli_prog_name="ncbi_rest_api") batch_size: int = Field( - default=MAX_RESULTS_PER_PAGE, + default=MAX_IDS_PER_QUERY, description="Number of IDs to send in each request to the NCBI REST API.", validation_alias=AliasChoices(*[alias.strip("-") for alias in ARG_ALIAS_BATCH_SIZE]), ge=1, - le=MAX_RESULTS_PER_PAGE, + le=MAX_IDS_PER_QUERY, + ) + + query_type: str | None = Field( + default=None, + description="The type of query to perform, dataset or annotation. By default, both are performed.", + validation_alias=AliasChoices(*[alias.strip("-") for alias in ARG_ALIAS_QUERY_TYPE]), + pattern=QUERY_TYPE_REGEX, ) + @field_validator("query_type", mode="before") + @classmethod + def trim_lc_query_type(cls, v: str | None) -> str | None: + """Prepare the query_type parameter for validation. + + :param v: query_type parameter + :type v: str | None + :return: cleaned query type param + :rtype: str | None + """ + return v.strip().lower() if v and v.strip() else None + def generate_file_path_name_from_url(settings: NcbiSettings, url_string: str) -> Path: """Given an NCBI URL, generate a save directory and file name for the raw HTTP responses. @@ -98,6 +124,32 @@ def save_raw_response(settings: NcbiSettings, response: Response, *_: Any, **__: return response +PIPELINE_SETTINGS: NcbiSettings | None = None + + +def set_settings(settings_obj: NcbiSettings) -> None: + """Set the global PIPELINE_SETTINGS object to the supplied obj. + + :param settings_obj: settings object + :type settings_obj: NcbiSettings + """ + # ugh, somewhat horrible + global PIPELINE_SETTINGS + PIPELINE_SETTINGS = settings_obj + + +def get_settings() -> NcbiSettings: + """Get the pipeline settings. + + return: pipeline settings + :rtype: NcbiSettings + """ + if PIPELINE_SETTINGS is None: + err_msg = "Pipeline settings have not been initialised" + raise RuntimeError(err_msg) + return PIPELINE_SETTINGS + + ncbi_genome_client = RESTClient( base_url="https://api.ncbi.nlm.nih.gov/datasets/v2/genome/accession/", headers={"accept": "application/json"}, @@ -161,28 +213,38 @@ def get_assembly_reports(assembly_id_list: list[str]) -> dict[str, Any]: if not assembly_id_list: return {} - errors = [] + settings = get_settings() + fetch_dataset: bool = settings.query_type in (None, DATASET) + fetch_annotation: bool = settings.query_type in (None, ANNOTATION) + errors: list[dict[str, Any]] = [] # N.b. invalid IDs will not be present in dataset_reports dataset_reports = {} - try: - dataset_reports = get_dataset_reports(assembly_id_list) - except Exception as e: # noqa: BLE001 - add_error(errors, e, "dataset_report", assembly_id_list=assembly_id_list) - - annotation_reports: dict[str, Any] = {} - for assembly_id in assembly_id_list: + if fetch_dataset: try: - annotation_reports[assembly_id] = get_annotation_report(assembly_id) + dataset_reports = get_dataset_reports(assembly_id_list) except Exception as e: # noqa: BLE001 - add_error(errors, e, "annotation_report", assembly_id=assembly_id) + add_error(errors, e, "dataset_report", assembly_id_list=assembly_id_list) + + annotation_reports: dict[str, Any] = {} + if fetch_annotation: + for assembly_id in assembly_id_list: + try: + annotation_reports[assembly_id] = get_annotation_report(assembly_id) + except Exception as e: # noqa: BLE001 + add_error(errors, e, "annotation_report", assembly_id=assembly_id) # ensure every assembly_id in the list has either the downloaded dataset_report or None - return { - DATASET: {assembly_id: dataset_reports.get(assembly_id) for assembly_id in assembly_id_list}, - ANNOTATION: {assembly_id: annotation_reports.get(assembly_id) for assembly_id in assembly_id_list}, + output: dict[str, Any] = { ERROR: errors, } + if fetch_annotation: + output[ANNOTATION] = {assembly_id: annotation_reports.get(assembly_id) for assembly_id in assembly_id_list} + + if fetch_dataset: + output[DATASET] = {assembly_id: dataset_reports.get(assembly_id) for assembly_id in assembly_id_list} + + return output def get_dataset_reports(assembly_id_list: list[str]) -> dict[str, None | dict[str, Any]]: @@ -268,8 +330,9 @@ def assemble_assembly_reports( @dlt.resource(name="assembly_list") -def assembly_list(settings: NcbiSettings) -> Generator[list[str], Any, Any]: +def assembly_list() -> Generator[list[str], Any, Any]: """List of assemblies to fetch.""" + settings = get_settings() batcher = BatchCursor(settings.input_dir, batch_size=1) while files := batcher.get_batch(): for file_path in files: @@ -300,12 +363,11 @@ def run_ncbi_pipeline(settings: NcbiSettings) -> None: :param settings: configuration for the pipeline :type settings: NcbiSettings """ + set_settings(settings) + if settings.dev_mode: REST_CLIENT_HOOKS["response"] = [partial(save_raw_response, settings)] - # ensure that assembly_list has the correct settings bound before running the pipeline - assembly_list.bind(settings) - pipeline_kwargs = { "pipeline_name": DATASET_NAME, "dataset_name": DATASET_NAME, diff --git a/tests/pipelines/test_ncbi_rest_api.py b/tests/pipelines/test_ncbi_rest_api.py index a5029f3a..5a744fa4 100644 --- a/tests/pipelines/test_ncbi_rest_api.py +++ b/tests/pipelines/test_ncbi_rest_api.py @@ -1,9 +1,10 @@ """Tests for the NCBI datasets API pipeline functions.""" +import re from pathlib import Path from typing import Any from unittest import mock -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch import pytest from dlt.extract.items import DataItemWithMeta @@ -18,9 +19,11 @@ from cdm_data_loaders.pipelines.ncbi_rest_api import ( ANNOTATION, ARG_ALIAS_BATCH_SIZE, + ARG_ALIAS_QUERY_TYPE, DATASET, DATASET_NAME, ERROR, + MAX_IDS_PER_QUERY, NcbiSettings, assemble_assembly_reports, assembly_list, @@ -28,7 +31,9 @@ get_annotation_report, get_assembly_reports, get_dataset_reports, + get_settings, run_ncbi_pipeline, + set_settings, ) from tests.pipelines.conftest import ( DEFAULT_VCR_CONFIG, @@ -51,17 +56,23 @@ def patch_rest_client_hooks(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr("cdm_data_loaders.pipelines.ncbi_rest_api.REST_CLIENT_HOOKS", {}) -BATCH_SIZE_STRING = "500" -BATCH_SIZE_INT = 500 +ID_WITH_2K_ANNOTS = "GCF_000003135.1" +ID_WITH_500_ANNOTS = "GCF_000007725.1" +ID_TRIGGERS_500_ERR = "GCF_500_ERROR" +VALID_IDS = [ID_WITH_500_ANNOTS, ID_WITH_2K_ANNOTS] +INVALID_ID = "invalid_id" +ALL_IDS = [*VALID_IDS, INVALID_ID] +BATCH_SIZE = 200 +BATCH_SIZE_STRING = str(BATCH_SIZE) -TEST_NCBI_SETTINGS = frozendict( - **TEST_CTS_SETTINGS, - batch_size=BATCH_SIZE_STRING, -) +TEST_NCBI_SETTINGS = frozendict(**TEST_CTS_SETTINGS, batch_size=BATCH_SIZE_STRING, query_type="") + +TEST_NCBI_SETTINGS_RECONCILED = frozendict(**TEST_CTS_SETTINGS_RECONCILED, batch_size=BATCH_SIZE, query_type=None) + +TEST_NCBI_SETTINGS_V1 = frozendict(**TEST_CTS_SETTINGS, batch_size=BATCH_SIZE_STRING, query_type=" ANNOTATION ") -TEST_NCBI_SETTINGS_RECONCILED = frozendict( - **TEST_CTS_SETTINGS_RECONCILED, - batch_size=BATCH_SIZE_INT, +TEST_NCBI_SETTINGS_RECONCILED_V1 = frozendict( + **TEST_CTS_SETTINGS_RECONCILED, batch_size=BATCH_SIZE, query_type=ANNOTATION ) @@ -71,16 +82,6 @@ def vcr_config() -> dict[str, Any]: return {**DEFAULT_VCR_CONFIG} -ID_WITH_2K_ANNOTS = "GCF_000003135.1" -ID_WITH_500_ANNOTS = "GCF_000007725.1" -ID_TRIGGERS_500_ERR = "GCF_500_ERROR" -VALID_IDS = [ID_WITH_500_ANNOTS, ID_WITH_2K_ANNOTS] -INVALID_ID = "invalid_id" -ALL_IDS = [*VALID_IDS, INVALID_ID] -BATCH_SIZE = 500 -BATCH_SIZE_STRING = "500" - - @pytest.fixture(scope="module") def valid_assembly_ids() -> list[str]: """A list of assembly IDs.""" @@ -105,19 +106,69 @@ def assembly_ids(valid_assembly_ids: list[str], invalid_assembly_id: str) -> lis return [*valid_assembly_ids, invalid_assembly_id] +@pytest.fixture(scope="module") +def test_settings() -> NcbiSettings: + """Generate a test settings class.""" + return make_settings_autofill_config(NcbiSettings) # type: ignore[reportReturnType] + + def make_settings(**kwargs: str | int | bool) -> NcbiSettings: """Generate a validated NcbiSettings object.""" return NcbiSettings.model_validate(kwargs) +@pytest.fixture(autouse=True) +def reset_settings(monkeypatch: pytest.MonkeyPatch) -> None: + """Reset the pipeline settings prior to running tests. + + :param monkeypatch: monkeypatch + :type monkeypatch: pytest.MonkeyPatch + """ + monkeypatch.setattr(ncbi_module, "PIPELINE_SETTINGS", None) + + +def test_get_set_settings(test_settings: NcbiSettings) -> None: + """Test setting and retrieval of settings.""" + assert ncbi_module.PIPELINE_SETTINGS is None + with pytest.raises(RuntimeError, match="Pipeline settings have not been initialised"): + get_settings() + + set_settings(test_settings) + assert test_settings == ncbi_module.PIPELINE_SETTINGS + assert get_settings() == test_settings + + set_settings(None) # type: ignore[reportArgumentType] + assert ncbi_module.PIPELINE_SETTINGS is None + with pytest.raises(RuntimeError, match="Pipeline settings have not been initialised"): + get_settings() + + +@pytest.mark.parametrize( + ("batch_size", "parsed_batch_size"), + [ + ("10", 10), + (10, 10), + (None, MAX_IDS_PER_QUERY), + ], +) +def test_settings_valid_batch_size(batch_size: int | str | None, parsed_batch_size: int) -> None: + """Ensure that a valid batch size is correctly parsed.""" + if batch_size is None: + settings: NcbiSettings = make_settings_autofill_config(NcbiSettings) # type: ignore[reportReturnType] + else: + settings: NcbiSettings = make_settings_autofill_config(NcbiSettings, batch_size=batch_size) # type: ignore[reportReturnType] + assert settings.batch_size == parsed_batch_size + + @pytest.mark.parametrize( ("bad_batch_size", "message"), [ ("0", "Input should be greater than or equal to 1"), ("-1", "Input should be greater than or equal to 1"), - ("1001", "Input should be less than or equal to 1000"), + ("1001", f"Input should be less than or equal to {MAX_IDS_PER_QUERY}"), ("notanint", "Input should be a valid integer"), ("", "Input should be a valid integer"), + ("1.2345", "Input should be a valid integer"), ], ) @pytest.mark.parametrize("use_cliapp", [True, False]) @@ -131,12 +182,61 @@ def test_cli_invalid_batch_size_via_cli_raises(bad_batch_size: str, message: str make_settings_autofill_config(NcbiSettings, batch_size=bad_batch_size) -def test_settings_all_params_set() -> None: +@pytest.mark.parametrize( + ("query_type", "parsed_query_type"), + [ + (ANNOTATION, ANNOTATION), + (DATASET, DATASET), + (" ANNOTATION ", ANNOTATION), + ("\n\nDataset\t\n", DATASET), + (" ", None), + ("", None), + (None, None), + ], +) +def test_cli_valid_query_type(query_type: str | None, parsed_query_type: str | None) -> None: + """Ensure that a valid batch size is correctly parsed.""" + settings: NcbiSettings = make_settings_autofill_config(NcbiSettings, query_type=query_type) # type: ignore[reportReturnType] + assert settings.query_type == parsed_query_type + + if query_type is None: + alt_settings: NcbiSettings = make_settings_autofill_config(NcbiSettings) # type: ignore[reportReturnType] + assert alt_settings.query_type == parsed_query_type + + +STRING_MATCH_MESSAGE = re.escape("String should match pattern '^(dataset|annotation)$'") + + +@pytest.mark.parametrize( + "query_type", + [ + "annot", + "anotation", + "data set", + ], +) +@pytest.mark.parametrize("use_cliapp", [True, False]) +def test_cli_invalid_query_type(query_type: str, use_cliapp: bool) -> None: + """Ensure that an invalid batch size passed via CLI raises an error.""" + if use_cliapp: + with pytest.raises(ValidationError, match=STRING_MATCH_MESSAGE): + CliApp.run(NcbiSettings, cli_args=["--query-type", query_type]) + else: + with pytest.raises(ValidationError, match=STRING_MATCH_MESSAGE): + make_settings_autofill_config(NcbiSettings, query_type=query_type) + + +@pytest.mark.parametrize( + ("settings", "reconciled"), + [(TEST_NCBI_SETTINGS, TEST_NCBI_SETTINGS_RECONCILED), (TEST_NCBI_SETTINGS_V1, TEST_NCBI_SETTINGS_RECONCILED_V1)], +) +def test_settings_all_params_set(settings: frozendict, reconciled: frozendict) -> None: """Ensure that settings are set correctly when all args are specified.""" - s = make_settings_autofill_config(NcbiSettings, **TEST_NCBI_SETTINGS) - check_settings(s, TEST_NCBI_SETTINGS_RECONCILED) + s = make_settings_autofill_config(NcbiSettings, **settings) + check_settings(s, reconciled) +@pytest.mark.parametrize("query_type", ARG_ALIAS_QUERY_TYPE) @pytest.mark.parametrize("batch_size", ARG_ALIAS_BATCH_SIZE) @pytest.mark.parametrize("dev_mode", ARG_ALIASES["dev_mode"]) @pytest.mark.parametrize("input_dir", ARG_ALIASES["input_dir"]) @@ -147,6 +247,7 @@ def test_settings_all_params_set() -> None: ARG_ALIASES["use_output_dir_for_pipeline_metadata"], ) def test_cli_all_variants( # noqa: PLR0913 + query_type: str, batch_size: str, dev_mode: str, input_dir: str, @@ -160,6 +261,8 @@ def test_cli_all_variants( # noqa: PLR0913 NcbiSettings, dlt_config=dlt_config, cli_args=[ + query_type, + "", batch_size, BATCH_SIZE_STRING, dev_mode, @@ -224,8 +327,9 @@ def check_annotation_report(annotation_report: list[dict[str, Any]] | None, asse def test_assembly_list_resource() -> None: """Test that the assembly list resource yields the expected assembly IDs.""" settings: NcbiSettings = make_settings_autofill_config(NcbiSettings, input_dir="tests/data/ncbi_rest_api/input") # type: ignore[reportAssignmentType] + set_settings(settings) - ass_list = list(assembly_list(settings)) + ass_list = list(assembly_list()) assert ass_list == [ "GCF_029958545.3", "GCF_029958565.3", @@ -270,7 +374,8 @@ def test_run_ncbi_pipeline_sets_core_run_pipeline_args_correctly( "raw_data_dir": "/some/dir/raw_data", "use_destination": "local_fs", "use_output_dir_for_pipeline_metadata": bool(use_pipeline_dir), - "batch_size": 1000, + "batch_size": MAX_IDS_PER_QUERY, + "query_type": None, }, ) @@ -280,7 +385,6 @@ def test_run_ncbi_pipeline_sets_core_run_pipeline_args_correctly( mock_dlt.destination.assert_called_once() assert mock_dlt.destination.call_args_list[0].kwargs == {"max_table_nesting": 0} assert mock_dlt.destination.call_args_list[0].args == ("local_fs",) - mock_assembly_list.bind.assert_called_once_with(settings) mock_dlt.pipeline.assert_called_once() assert mock_dlt.pipeline.call_args.kwargs["destination"] == mock_dlt.destination.return_value @@ -346,14 +450,16 @@ def test_get_annotation_report_invalid_id() -> None: assert get_annotation_report(INVALID_ID) is None -def test_get_assembly_reports_empty_id_list() -> None: +def test_get_assembly_reports_empty_id_list(test_settings: NcbiSettings) -> None: """Ensure that getting reports for an empty list returns nothing.""" + set_settings(test_settings) assert get_assembly_reports([]) == {} @pytest.mark.vcr -def test_get_assembly_reports() -> None: +def test_get_assembly_reports(test_settings: NcbiSettings) -> None: """Test the retrieval of annotation and dataset reports.""" + set_settings(test_settings) assembly_reports = get_assembly_reports(ALL_IDS) assert set(assembly_reports) == {DATASET, ANNOTATION, ERROR} for datatype in [DATASET, ANNOTATION]: @@ -365,6 +471,41 @@ def test_get_assembly_reports() -> None: assert assembly_reports[ERROR] == [] +@pytest.mark.parametrize("query_type", [None, DATASET, ANNOTATION]) +def test_get_assembly_reports_mock_subs(query_type: str | None, monkeypatch: pytest.MonkeyPatch) -> None: + """Ensure that the correct subs are called and the correct subs are not called with the query_type parameter.""" + settings: NcbiSettings = make_settings_autofill_config(NcbiSettings, query_type=query_type) # type: ignore[reportAssignmentType] + set_settings(settings) + mock_get_annotation_report = MagicMock(return_value={"this": "that"}) + mock_get_dataset_reports = MagicMock(return_value=dict.fromkeys(ALL_IDS, "blob")) + + monkeypatch.setattr(ncbi_module, "get_annotation_report", mock_get_annotation_report) + monkeypatch.setattr(ncbi_module, "get_dataset_reports", mock_get_dataset_reports) + + output = get_assembly_reports(ALL_IDS) + assert output[ERROR] == [] + + if query_type == ANNOTATION: + mock_get_dataset_reports.assert_not_called() + assert DATASET not in output + + else: + mock_get_dataset_reports.assert_called_once_with(ALL_IDS) + assert output[DATASET] == dict.fromkeys(ALL_IDS, "blob") + + if query_type == DATASET: + mock_get_annotation_report.assert_not_called() + assert ANNOTATION not in output + else: + assert mock_get_annotation_report.call_args_list == [ + call( + n, + ) + for n in ALL_IDS + ] + assert output[ANNOTATION] == {this_id: {"this": "that"} for this_id in ALL_IDS} + + RECORDED_ERRORS = { "dataset_404": { "assembly_id": None, @@ -412,8 +553,9 @@ def test_get_assembly_reports() -> None: @mock.patch("tenacity.nap.time.sleep", MagicMock()) @pytest.mark.default_cassette("test_get_assembly_reports_annotation_report_errors.yaml") @pytest.mark.vcr -def test_get_assembly_reports_annotation_report_errors() -> None: +def test_get_assembly_reports_annotation_report_errors(test_settings: NcbiSettings) -> None: """Test the retrieval of assembly data when errors occur fetching annotation reports.""" + set_settings(test_settings) original_get_annotation_report = get_annotation_report def patched_get_annotation_report(assembly_id: str) -> list[dict[str, Any]] | None: @@ -451,8 +593,9 @@ def patched_get_annotation_report(assembly_id: str) -> list[dict[str, Any]] | No @mock.patch("tenacity.nap.time.sleep", MagicMock()) @pytest.mark.vcr -def test_get_assembly_reports_dataset_report_errors() -> None: +def test_get_assembly_reports_dataset_report_errors(test_settings: NcbiSettings) -> None: """Test the retrieval of assembly data when an error occurs fetching dataset reports.""" + set_settings(test_settings) assembly_reports = get_assembly_reports(ALL_IDS) assert set(assembly_reports) == {DATASET, ANNOTATION, ERROR} for datatype in [DATASET, ANNOTATION]: @@ -467,8 +610,9 @@ def test_get_assembly_reports_dataset_report_errors() -> None: @mock.patch("tenacity.nap.time.sleep", MagicMock()) @pytest.mark.vcr -def test_get_assembly_reports_total_wipeout() -> None: +def test_get_assembly_reports_total_wipeout(test_settings: NcbiSettings) -> None: """Test the retrieval of assembly data when all queries fail.""" + set_settings(test_settings) original_get_annotation_report = get_annotation_report def patched_get_annotation_report(assembly_id: str) -> list[dict[str, Any]] | None: From aa99930c7c35e469c3efd9addedd032b0c3f4e59 Mon Sep 17 00:00:00 2001 From: Matt Dawson Date: Mon, 15 Jun 2026 14:58:15 -0700 Subject: [PATCH 127/128] NCBI FTP Pipeline Clean-Up (#184) * use path types for s3, ftp, local paths for ncbi ftp * Potential fix for pull request finding 'Statement has no effect' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> * Potential fix for pull request finding 'Unused import' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * address copilot comments * Update src/cdm_data_loaders/ncbi_ftp/manifest.py Co-authored-by: i alarmed alien * address review comments * address review comments * address review comments * clean up settings tests * clean up settings tests more --------- Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: i alarmed alien --- notebooks/ncbi_ftp_download.ipynb | 21 +- src/cdm_data_loaders/ncbi_ftp/assembly.py | 52 +- src/cdm_data_loaders/ncbi_ftp/manifest.py | 168 +++-- src/cdm_data_loaders/ncbi_ftp/metadata.py | 61 +- src/cdm_data_loaders/ncbi_ftp/promote.py | 189 +++--- .../pipelines/ncbi_ftp_download.py | 62 +- src/cdm_data_loaders/utils/ftp_client.py | 24 +- tests/integration/conftest.py | 68 +- tests/integration/test_download_e2e.py | 33 +- tests/integration/test_full_pipeline.py | 45 +- tests/integration/test_manifest_e2e.py | 66 +- tests/integration/test_promote_e2e.py | 520 +++++++-------- tests/ncbi_ftp/conftest.py | 34 +- tests/ncbi_ftp/test_assembly.py | 35 +- tests/ncbi_ftp/test_manifest.py | 221 ++++--- tests/ncbi_ftp/test_metadata.py | 99 +-- tests/ncbi_ftp/test_notebooks.py | 25 +- tests/ncbi_ftp/test_promote.py | 594 +++++++++++------- tests/pipelines/test_ncbi_ftp_download.py | 110 ++-- tests/utils/test_ftp_client.py | 20 +- 20 files changed, 1358 insertions(+), 1089 deletions(-) diff --git a/notebooks/ncbi_ftp_download.ipynb b/notebooks/ncbi_ftp_download.ipynb index 286db5cb..35d61030 100644 --- a/notebooks/ncbi_ftp_download.ipynb +++ b/notebooks/ncbi_ftp_download.ipynb @@ -50,6 +50,7 @@ "source": [ "\"\"\"Imports and S3 client initialisation.\"\"\"\n", "\n", + "from pathlib import Path, PurePosixPath\n", "\n", "from cdm_data_loaders.pipelines.ncbi_ftp_download import (\n", " download_and_stage,\n", @@ -75,21 +76,21 @@ "\n", "# S3 bucket where the manifest lives and where staged files will be written\n", "# format: bucket name (no s3:// scheme)\n", - "STAGING_BUCKET = \"cts\"\n", + "STAGING_BUCKET: PurePosixPath = PurePosixPath(\"cts\")\n", "\n", "# S3 object key of the transfer manifest written by Phase 1\n", "# format: S3 object key within STAGING_BUCKET (no scheme, no bucket)\n", "# Set to None to use MANIFEST_LOCAL_PATH instead\n", - "MANIFEST_S3_KEY: str | None = \"io/matt-cohere/staging/run1/input/transfer_manifest.txt\"\n", + "MANIFEST_S3_KEY: PurePosixPath | None = PurePosixPath(\"io/matt-cohere/staging/run1/input/transfer_manifest.txt\")\n", "\n", "# Local path to the transfer manifest (alternative to MANIFEST_S3_KEY)\n", "# format: local filesystem path\n", "# Set to None to use MANIFEST_S3_KEY instead\n", - "MANIFEST_LOCAL_PATH: str | None = None\n", + "MANIFEST_LOCAL_PATH: Path | None = None\n", "\n", "# S3 key prefix for staged output files (must match what Phase 3 expects)\n", "# format: S3 key prefix within STAGING_BUCKET (no scheme, no bucket)\n", - "STAGING_KEY_PREFIX = \"io/matt-cohere/staging/run1/output/\"\n", + "STAGING_KEY_PREFIX: PurePosixPath = PurePosixPath(\"io/matt-cohere/staging/run1/output/\")\n", "\n", "# Number of parallel download and upload threads\n", "THREADS = 4\n", @@ -142,13 +143,15 @@ "\n", "if MANIFEST_S3_KEY is not None:\n", " s3 = get_s3_client()\n", - " response = s3.get_object(Bucket=STAGING_BUCKET, Key=MANIFEST_S3_KEY)\n", + " response = s3.get_object(Bucket=str(STAGING_BUCKET), Key=str(MANIFEST_S3_KEY))\n", " manifest_lines = response[\"Body\"].read().decode().splitlines()\n", - "else:\n", - " with open(MANIFEST_LOCAL_PATH) as f:\n", + "elif MANIFEST_LOCAL_PATH:\n", + " with MANIFEST_LOCAL_PATH.open() as f:\n", " manifest_lines = f.read().splitlines()\n", + "else:\n", + " manifest_lines = []\n", "\n", - "data_lines = [l for l in manifest_lines if l.strip() and not l.startswith(\"#\")]\n", + "data_lines = [line for line in manifest_lines if line.strip() and not line.startswith(\"#\")]\n", "\n", "print(f\"Total entries: {len(data_lines)}\")\n", "print(\"First 10:\")\n", @@ -214,7 +217,7 @@ ], "metadata": { "kernelspec": { - "display_name": "cdm-data-loaders (3.13.11)", + "display_name": "cdm-data-loaders (3.13.11.final.0)", "language": "python", "name": "python3" }, diff --git a/src/cdm_data_loaders/ncbi_ftp/assembly.py b/src/cdm_data_loaders/ncbi_ftp/assembly.py index 5d75f533..8368cc75 100644 --- a/src/cdm_data_loaders/ncbi_ftp/assembly.py +++ b/src/cdm_data_loaders/ncbi_ftp/assembly.py @@ -8,7 +8,7 @@ import contextlib import time from ftplib import FTP -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any from cdm_data_loaders.ncbi_ftp.constants import ACCESSION_PARTS_REGEX, ASSEMBLY_PATH_REGEX, FTP_HOST @@ -32,7 +32,7 @@ ] -def parse_md5_checksums_file(text: str) -> dict[str, str]: +def parse_md5_checksums_file(text: str) -> dict[PurePosixPath, str]: """Parse an NCBI ``md5checksums.txt`` file into a filename-to-hash mapping. Each line has the format `` ./`` (two-space separator). @@ -40,43 +40,42 @@ def parse_md5_checksums_file(text: str) -> dict[str, str]: :param text: raw text of the md5checksums.txt file :return: dict mapping filename to MD5 hex digest """ - checksums: dict[str, str] = {} + checksums: dict[PurePosixPath, str] = {} for raw_line in text.strip().splitlines(): parts = raw_line.strip().split(" ", maxsplit=1) if len(parts) == 2: # noqa: PLR2004 md5_hash, filename = parts - checksums[filename.removeprefix("./")] = md5_hash.strip() + checksums[PurePosixPath(filename.removeprefix("./"))] = md5_hash.strip() return checksums # Path helpers -def build_accession_path(assembly_dir: str) -> str: +def build_accession_path(assembly_dir: PurePosixPath) -> PurePosixPath: """Build the relative output path for an assembly directory. Produces ``raw_data/{GCF|GCA}/{000}/{001}/{215}/{assembly_dir}/``. :param assembly_dir: full assembly directory name (e.g. ``GCF_000001215.4_Release_6...``) - :return: relative path string + :return: relative path :raises ValueError: if the assembly directory name cannot be parsed """ - if m := ACCESSION_PARTS_REGEX.match(assembly_dir): - db, p1, p2, p3 = m.groups() - return f"raw_data/{db}/{p1}/{p2}/{p3}/{assembly_dir}/" + if m := ACCESSION_PARTS_REGEX.match(f"{assembly_dir}"): # returns e.g., "GCF", "000", "001", "215" captured groups + return PurePosixPath("raw_data", *m.groups(), assembly_dir) msg = f"Cannot parse accession: {assembly_dir}" raise ValueError(msg) -def parse_assembly_path(assembly_path: str) -> tuple[str, str, str]: +def parse_assembly_path(assembly_path: PurePosixPath) -> tuple[str, PurePosixPath, str]: """Extract database, assembly_dir, and accession from an FTP assembly path. :param assembly_path: FTP directory path (e.g. ``/genomes/all/GCF/000/.../GCF_000001215.4_Rel.../``) :return: tuple of ``(database, assembly_dir, accession)`` :raises ValueError: if the path cannot be parsed """ - if m := ASSEMBLY_PATH_REGEX.search(assembly_path.rstrip("/")): - return m.group(3), m.group(1), m.group(2) + if m := ASSEMBLY_PATH_REGEX.search(str(assembly_path)): + return m.group(3), PurePosixPath(m.group(1)), m.group(2) msg = f"Cannot parse assembly path: {assembly_path}" raise ValueError(msg) @@ -86,9 +85,9 @@ def parse_assembly_path(assembly_path: str) -> tuple[str, str, str]: def _download_and_verify( # noqa: PLR0913 ftp: FTP, - filename: str, + filename: PurePosixPath, dest_dir: Path, - md5_checksums: dict[str, str], + md5_checksums: dict[PurePosixPath, str], stats: dict[str, Any], last_activity: float, ) -> float: @@ -132,8 +131,8 @@ def validate_file(local_file: Path) -> bool: def download_assembly_to_local( - assembly_path: str, - output_dir: str | Path, + assembly_path: PurePosixPath, + output_dir: Path, ftp_host: str = FTP_HOST, ftp: FTP | None = None, ) -> dict[str, Any]: @@ -147,11 +146,11 @@ def download_assembly_to_local( :param output_dir: base output directory :param ftp_host: FTP hostname :param ftp: optional existing FTP connection (caller manages lifecycle) - :return: dict with download statistics + :return: json parsable dict with download statistics """ _database, assembly_dir, accession = parse_assembly_path(assembly_path) rel_path = build_accession_path(assembly_dir) - dest_dir = Path(output_dir) / rel_path + dest_dir: Path = output_dir / rel_path dest_dir.mkdir(parents=True, exist_ok=True) logger.debug("Downloading %s -> %s", accession, dest_dir) @@ -161,27 +160,28 @@ def download_assembly_to_local( ftp = connect_ftp(ftp_host) stats: dict[str, Any] = { "accession": accession, - "assembly_dir": assembly_dir, + "assembly_dir": str(assembly_dir), "files_downloaded": 0, "files_skipped_checksum_mismatch": 0, "files_without_checksum": 0, } try: - ftp.cwd(assembly_path.rstrip("/")) + ftp.cwd(str(assembly_path)) - files: list[str] = [] - ftp.retrlines("NLST", files.append) + file_strings: list[str] = [] + ftp.retrlines("NLST", file_strings.append) + files: list[PurePosixPath] = [PurePosixPath(f) for f in file_strings] # Download and parse md5checksums.txt - md5_checksums: dict[str, str] = {} - if "md5checksums.txt" in files: - md5_text = ftp_retrieve_text(ftp, "md5checksums.txt") + md5_checksums: dict[PurePosixPath, str] = {} + if PurePosixPath("md5checksums.txt") in files: + md5_text = ftp_retrieve_text(ftp, PurePosixPath("md5checksums.txt")) md5_checksums = parse_md5_checksums_file(md5_text) (dest_dir / "md5checksums.txt").write_text(md5_text) stats["files_downloaded"] += 1 - target_files = [f for f in files if any(f.endswith(s) for s in FILE_FILTERS)] + target_files = [f for f in files if any(f.match(f"**{s}") for s in FILE_FILTERS)] last_activity = time.monotonic() for filename in target_files: diff --git a/src/cdm_data_loaders/ncbi_ftp/manifest.py b/src/cdm_data_loaders/ncbi_ftp/manifest.py index 59908fbe..bb7acf79 100644 --- a/src/cdm_data_loaders/ncbi_ftp/manifest.py +++ b/src/cdm_data_loaders/ncbi_ftp/manifest.py @@ -14,8 +14,10 @@ from collections.abc import Callable, Iterable from dataclasses import dataclass, field from datetime import UTC, datetime -from pathlib import Path +from http import HTTPStatus +from pathlib import Path, PurePosixPath from typing import Any +from urllib.parse import urlsplit from botocore.exceptions import ClientError @@ -37,24 +39,38 @@ "genbank": "GCA_", } -SUMMARY_FTP_PATHS: dict[str, str] = { - "refseq": "/genomes/ASSEMBLY_REPORTS/assembly_summary_refseq.txt", - "genbank": "/genomes/ASSEMBLY_REPORTS/assembly_summary_genbank.txt", +SUMMARY_FTP_PATHS: dict[str, PurePosixPath] = { + "refseq": PurePosixPath("/") / "genomes" / "ASSEMBLY_REPORTS" / "assembly_summary_refseq.txt", + "genbank": PurePosixPath("/") / "genomes" / "ASSEMBLY_REPORTS" / "assembly_summary_genbank.txt", } +# Assembly summary file columns of interest +_ACCESSION_COL = 0 +_STATUS_COL = 10 +_SEQ_REL_DATE_COL = 14 +_FTP_URL_COL = 19 +_MIN_COL = 20 # Data structures @dataclass class AssemblyRecord: - """Parsed row from an NCBI assembly summary file.""" + """Parsed row from an NCBI assembly summary file. + + Attributes: + accession: Assembly accession (e.g., "GCF_000001215.4"). + status: "latest", "replaced", or "suppressed". + seq_rel_date: Release date. + ftp_url: Full FTP URL. + assembly_dir: Assembly directory name (final path segment from the FTP URL). + """ accession: str status: str seq_rel_date: str - ftp_path: str - assembly_dir: str + ftp_url: str + assembly_dir: PurePosixPath @dataclass @@ -116,18 +132,19 @@ def _parse_lines(lines: Iterable[str]) -> None: delimiter="\t", ) for row in reader: - if len(row) < 20: # noqa: PLR2004 + if len(row) < _MIN_COL: continue - accession = row[0] - ftp_path = row[19] - if ftp_path == "na": + accession = row[_ACCESSION_COL] + ftp_url = row[_FTP_URL_COL] + if ftp_url == "na": continue + assembly_dir = PurePosixPath(_ftp_dir_from_url(ftp_url).name) assemblies[accession] = AssemblyRecord( accession=accession, - status=row[10], - seq_rel_date=row[14], - ftp_path=ftp_path, - assembly_dir=ftp_path.rstrip("/").split("/")[-1], + status=row[_STATUS_COL], + seq_rel_date=row[_SEQ_REL_DATE_COL], + ftp_url=ftp_url, + assembly_dir=assembly_dir, ) if isinstance(source, Path) or (isinstance(source, str) and "\n" not in source and Path(source).is_file()): @@ -142,29 +159,33 @@ def _parse_lines(lines: Iterable[str]) -> None: return assemblies -def get_latest_assembly_paths(assemblies: dict[str, AssemblyRecord], ftp_host: str = FTP_HOST) -> list[tuple[str, str]]: +def get_latest_assembly_paths( + assemblies: dict[str, AssemblyRecord], ftp_host: str = FTP_HOST +) -> list[tuple[str, PurePosixPath]]: """Extract FTP directory paths for all assemblies with ``latest`` status. :param assemblies: parsed assembly records :param ftp_host: FTP hostname for URL stripping :return: list of ``(accession, ftp_dir_path)`` tuples """ - paths: list[tuple[str, str]] = [] + paths: list[tuple[str, PurePosixPath]] = [] for accession, rec in assemblies.items(): if rec.status != "latest": continue - ftp_path = _ftp_dir_from_url(rec.ftp_path, ftp_host) - paths.append((accession, ftp_path.rstrip("/") + "/")) + ftp_path = _ftp_dir_from_url(rec.ftp_url, ftp_host) + paths.append((accession, ftp_path)) return paths # Prefix filtering -def accession_prefix(accession: str) -> str | None: +def accession_prefix(accession: str) -> str: """Extract the 3-digit prefix from an accession (e.g. ``GCF_000005845.2`` → ``"000"``).""" - m = ACCESSION_PARTS_REGEX.match(accession) - return m.group(2) if m else None + if m := ACCESSION_PARTS_REGEX.match(accession): + return m.group(2) + msg = f"Could not parse accession: {accession}" + raise ValueError(msg) def filter_by_prefix_range( @@ -244,31 +265,40 @@ def compute_diff( # FTP URL helpers -def _ftp_dir_from_url(ftp_url: str, ftp_host: str = FTP_HOST) -> str: +def _ftp_dir_from_url(ftp_url: str, ftp_host: str = FTP_HOST) -> PurePosixPath: """Convert an FTP URL from the assembly summary to an FTP directory path.""" - if ftp_url.startswith("https://"): - return ftp_url.replace(f"https://{ftp_host}", "") - if ftp_url.startswith("ftp://"): - return ftp_url.replace(f"ftp://{ftp_host}", "") - return ftp_url + parsed = urlsplit(ftp_url) + + # Full URL + if parsed.scheme in {"ftp", "https"}: + if parsed.netloc and parsed.netloc != ftp_host: + msg = f"Unexpected FTP host: {parsed.netloc}" + raise ValueError(msg) + return PurePosixPath(parsed.path) + + # unparsable url + msg = f"Could not parse FTP URL: {ftp_url}" + raise ValueError(msg) # Synthetic summary from S3 store scan -def _extract_accession_dir_and_id_from_s3_key(key: str) -> tuple[str | None, str | None]: +def _extract_accession_dir_and_id_from_s3_key(key: PurePosixPath) -> tuple[str, str]: """Extract both accession and assembly directory from an S3 object key. e.g. "some/prefix/GCF_000001215.4_Release_6_plus_ISO1_MT/file.gz" → ("GCF_000001215.4_Release_6_plus_ISO1_MT", "GCF_000001215.4") """ - m = ASSEMBLY_PATH_REGEX.search(key) - return (m.group(1), m.group(2)) if m else (None, None) + if m := ASSEMBLY_PATH_REGEX.search(str(key)): + return (m.group(1), m.group(2)) + msg = f"Could not parse S3 key for accession info: {key}" + raise ValueError(msg) def scan_store_to_synthetic_summary( - bucket: str, - key_prefix: str, + bucket: PurePosixPath, + key_prefix: PurePosixPath, release_date: str, database: str = "refseq", progress_callback: Callable[[int, str], None] | None = None, @@ -314,14 +344,15 @@ def scan_store_to_synthetic_summary( processed_count = 0 try: - objs = list_matching_objects(f"{bucket}/{key_prefix}") + objs = list_matching_objects(str(bucket / key_prefix)) for obj in objs: - assembly_dir, acc = _extract_accession_dir_and_id_from_s3_key(obj["Key"]) - if not acc or not acc.startswith(acc_prefix): + try: + assembly_dir, acc = _extract_accession_dir_and_id_from_s3_key(obj["Key"]) + except ValueError: continue - if not assembly_dir: + if not acc.startswith(acc_prefix): continue if acc not in assemblies: @@ -335,8 +366,8 @@ def scan_store_to_synthetic_summary( accession=acc, status="latest", seq_rel_date=release_date, - ftp_path=fake_ftp_path, - assembly_dir=assembly_dir, + ftp_url=fake_ftp_path, + assembly_dir=PurePosixPath(assembly_dir), ) processed_count += 1 if progress_callback is not None: @@ -355,7 +386,7 @@ def scan_store_to_synthetic_summary( def _fetch_accession_checksums_from_ftp( ftp: FTP | None, ftp_host: str, last_activity: float, current_accession: str -) -> tuple[dict[str, str], float]: +) -> tuple[dict[PurePosixPath, str], float]: """Fetch and parse the md5checksums.txt file for a given accession from FTP. :param ftp: FTP connection object @@ -369,21 +400,21 @@ def _fetch_accession_checksums_from_ftp( last_activity = ftp_noop_keepalive(ftp, last_activity) ftp_dir = _ftp_dir_from_url(current_accession, ftp_host) try: - md5_text = ftp_retrieve_text(ftp, ftp_dir.rstrip("/") + "/md5checksums.txt") + md5_text = ftp_retrieve_text(ftp, ftp_dir / "md5checksums.txt") last_activity = time.monotonic() ftp_checksums = parse_md5_checksums_file(md5_text) except Exception: # noqa: BLE001 logger.warning("Cannot fetch md5checksums.txt for %s, keeping in transfer list", current_accession) return {}, last_activity return { - fname: md5 for fname, md5 in ftp_checksums.items() if any(fname.endswith(suffix) for suffix in FILE_FILTERS) + fname: md5 for fname, md5 in ftp_checksums.items() if any(fname.match(f"**{suffix}") for suffix in FILE_FILTERS) }, last_activity def _does_accession_need_update( - target_checksums: dict[str, str], - bucket: str, - s3_prefix: str, + target_checksums: dict[PurePosixPath, str], + bucket: PurePosixPath, + s3_prefix: PurePosixPath, ) -> bool: """Check if any file for an accession needs updating by comparing FTP checksums to S3 metadata. @@ -393,13 +424,20 @@ def _does_accession_need_update( :return: True if any file is missing or has a checksum mismatch, False otherwise """ for fname, expected_md5 in target_checksums.items(): - s3_path = f"{bucket}/{s3_prefix}{fname}" + s3_path = bucket / s3_prefix / fname s3_md5 = "" try: - obj_info = head_object(s3_path) + obj_info = head_object(str(s3_path)) s3_md5 = obj_info.get("Metadata", {}).get("md5", "") except ClientError as e: - if e.response["Error"]["Code"] == "404": # type: ignore[union-attr] + code = e.response.get("Error", {}).get("Code") + # boto3 error codes are not always ints, so check the conversion first + try: + code_int = int(code) + except (TypeError, ValueError): + code_int = None + + if code_int == HTTPStatus.NOT_FOUND: logger.debug("File missing from store: %s", s3_path) return True raise @@ -414,8 +452,8 @@ def _does_accession_need_update( def verify_transfer_candidates( # noqa: PLR0913 accessions: list[str], current_assemblies: dict[str, AssemblyRecord], - bucket: str, - key_prefix: str, + bucket: PurePosixPath, + key_prefix: PurePosixPath, ftp_host: str = FTP_HOST, progress_callback: Callable[[int, int, str], None] | None = None, ) -> list[str]: @@ -462,10 +500,10 @@ def _progress(done: int, total: int, acc: str) -> None: # Build S3 prefix for this assembly s3_rel = build_accession_path(rec.assembly_dir) - s3_prefix = f"{key_prefix}{s3_rel}" + s3_prefix = key_prefix / s3_rel # Quick check: does *anything* exist under this prefix? - resp = list_matching_objects(f"{bucket}/{s3_prefix}", max_keys=1) + resp = list_matching_objects(str(bucket / s3_prefix), max_keys=1) if not resp: # Nothing in the store — definitely needs downloading confirmed.append(acc) @@ -475,7 +513,7 @@ def _progress(done: int, total: int, acc: str) -> None: # Objects exist — need FTP md5 checksums to decide target_checksums, last_activity = _fetch_accession_checksums_from_ftp( - ftp, ftp_host, last_activity, rec.ftp_path + ftp, ftp_host, last_activity, rec.ftp_url ) if not target_checksums: @@ -514,9 +552,9 @@ def _progress(done: int, total: int, acc: str) -> None: def write_transfer_manifest( diff: DiffResult, current_assemblies: dict[str, AssemblyRecord], - output_path: str | Path, + output_path: Path, ftp_host: str = FTP_HOST, -) -> list[str]: +) -> list[PurePosixPath]: """Write the transfer manifest (new + updated assemblies). Each line is an FTP directory path suitable for Phase 2 download. @@ -528,22 +566,22 @@ def write_transfer_manifest( :return: list of FTP paths written """ to_transfer = diff.new + diff.updated - paths: list[str] = [] + paths: list[PurePosixPath] = [] for acc in sorted(to_transfer): rec = current_assemblies.get(acc) if not rec: continue - ftp_path = _ftp_dir_from_url(rec.ftp_path, ftp_host) - paths.append(ftp_path.rstrip("/") + "/") + ftp_path = _ftp_dir_from_url(rec.ftp_url, ftp_host) + paths.append(ftp_path) with Path(output_path).open("w") as f: - f.writelines(p + "\n" for p in paths) + f.writelines(f"{p}\n" for p in paths) logger.info("Wrote %d entries to transfer manifest: %s", len(paths), output_path) return paths -def write_removed_manifest(diff: DiffResult, output_path: str | Path) -> list[str]: +def write_removed_manifest(diff: DiffResult, output_path: Path) -> list[str]: """Write the removed manifest (replaced + suppressed accessions). :param diff: computed diff result @@ -551,13 +589,13 @@ def write_removed_manifest(diff: DiffResult, output_path: str | Path) -> list[st :return: list of accessions written """ removed = sorted(diff.replaced + diff.suppressed) - with Path(output_path).open("w") as f: + with output_path.open("w") as f: f.writelines(acc + "\n" for acc in removed) logger.info("Wrote %d entries to removed manifest: %s", len(removed), output_path) return removed -def write_updated_manifest(diff: DiffResult, output_path: str | Path) -> list[str]: +def write_updated_manifest(diff: DiffResult, output_path: Path) -> list[str]: """Write the updated manifest (accessions whose content changed). This file is consumed by Phase 3 to archive existing S3 objects @@ -568,7 +606,7 @@ def write_updated_manifest(diff: DiffResult, output_path: str | Path) -> list[st :return: list of accessions written """ updated = sorted(diff.updated) - with Path(output_path).open("w") as f: + with output_path.open("w") as f: f.writelines(acc + "\n" for acc in updated) logger.info("Wrote %d entries to updated manifest: %s", len(updated), output_path) return updated @@ -576,7 +614,7 @@ def write_updated_manifest(diff: DiffResult, output_path: str | Path) -> list[st def write_diff_summary( diff: DiffResult, - output_path: str | Path, + output_path: Path, database: str, prefix_from: str | None = None, prefix_to: str | None = None, @@ -612,7 +650,7 @@ def write_diff_summary( "suppressed": diff.suppressed, }, } - with Path(output_path).open("w") as f: + with output_path.open("w") as f: json.dump(summary, f, indent=2) logger.info("Wrote diff summary to: %s", output_path) return summary diff --git a/src/cdm_data_loaders/ncbi_ftp/metadata.py b/src/cdm_data_loaders/ncbi_ftp/metadata.py index 35f15d2a..511d6ea8 100644 --- a/src/cdm_data_loaders/ncbi_ftp/metadata.py +++ b/src/cdm_data_loaders/ncbi_ftp/metadata.py @@ -19,7 +19,7 @@ import json import tempfile from datetime import UTC, datetime -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any, TypedDict from frictionless import Package @@ -47,7 +47,7 @@ class DescriptorResource(TypedDict, total=False): """A single resource entry in the frictionless descriptor ``resources`` list.""" name: str - path: str + path: PurePosixPath format: str bytes: int | None hash: str | None @@ -56,20 +56,19 @@ class DescriptorResource(TypedDict, total=False): # Public helpers -def build_descriptor_key(assembly_dir: str, key_prefix: str) -> str: +def build_descriptor_key(assembly_dir: PurePosixPath, key_prefix: PurePosixPath) -> PurePosixPath: """Return the S3 key for the live descriptor of *assembly_dir*. :param assembly_dir: full assembly directory name, e.g. ``GCF_000001215.4_Release_6_plus_ISO1_MT`` :param key_prefix: Lakehouse key prefix (trailing slash optional) :return: S3 key, e.g. ``tenant-general-warehouse/.../ncbi/metadata/GCF_..._datapackage.json`` """ - prefix = key_prefix.rstrip("/") + "/" - return f"{prefix}metadata/{assembly_dir}_datapackage.json" + return key_prefix / "metadata" / assembly_dir.with_name(f"{assembly_dir.name}_datapackage.json") def build_archive_descriptor_key( - assembly_dir: str, release_tag: str, key_prefix: str, archive_reason: str = "unknown" -) -> str: + assembly_dir: PurePosixPath, release_tag: str, key_prefix: PurePosixPath, archive_reason: str = "unknown" +) -> PurePosixPath: """Return the S3 key for the archived descriptor of *assembly_dir*. :param assembly_dir: full assembly directory name @@ -78,12 +77,18 @@ def build_archive_descriptor_key( :param archive_reason: reason for archival, encoded as a path segment :return: S3 key under ``archive/{release_tag}/{archive_reason}/metadata/`` """ - prefix = key_prefix.rstrip("/") + "/" - return f"{prefix}archive/{release_tag}/{archive_reason}/metadata/{assembly_dir}_datapackage.json" + return ( + key_prefix + / "archive" + / release_tag + / archive_reason + / "metadata" + / assembly_dir.with_name(f"{assembly_dir.name}_datapackage.json") + ) def create_descriptor( - assembly_dir: str, + assembly_dir: PurePosixPath, accession_full: str, resources: list[DescriptorResource], *, @@ -112,14 +117,20 @@ def create_descriptor( normalised: list[dict[str, Any]] = [] for res in resources: entry: dict[str, Any] = { - "name": res["name"].lower(), - "path": res["path"], + "name": res["name"].lower() if "name" in res else None, + "path": str(res["path"]) if "path" in res else None, "format": res.get("format", ""), } + if entry["name"] is None: + msg = f"Missing name for file descriptor in {accession_full}" + raise ValueError(msg) + if entry["path"] is None: + msg = f"Missing path for file descriptor in {accession_full}" + raise ValueError(msg) if res.get("bytes") is not None: - entry["bytes"] = res["bytes"] + entry["bytes"] = res.get("bytes") if res.get("hash") is not None: - entry["hash"] = res["hash"] + entry["hash"] = res.get("hash") normalised.append(entry) return { @@ -167,12 +178,12 @@ def validate_descriptor(descriptor: dict[str, Any], accession_full: str) -> None def upload_descriptor( descriptor: dict[str, Any], - assembly_dir: str, - bucket: str, - key_prefix: str, + assembly_dir: PurePosixPath, + bucket: PurePosixPath, + key_prefix: PurePosixPath, *, dry_run: bool = False, -) -> str: +) -> PurePosixPath: """Serialise and upload a descriptor to the live ``metadata/`` path. :param descriptor: descriptor dict from :func:`create_descriptor` @@ -196,7 +207,7 @@ def upload_descriptor( tmp.write(body) try: - s3.upload_file(Filename=tmp_path, Bucket=bucket, Key=key) + s3.upload_file(Filename=tmp_path, Bucket=str(bucket), Key=str(key)) logger.debug("Uploaded descriptor: s3://%s/%s", bucket, key) finally: Path(tmp_path).unlink() @@ -205,9 +216,9 @@ def upload_descriptor( def archive_descriptor( # noqa: PLR0913 - assembly_dir: str, - bucket: str, - key_prefix: str, + assembly_dir: PurePosixPath, + bucket: PurePosixPath, + key_prefix: PurePosixPath, release_tag: str, *, archive_reason: str = "unknown", @@ -235,7 +246,7 @@ def archive_descriptor( # noqa: PLR0913 s3 = get_s3_client() try: - s3.head_object(Bucket=bucket, Key=source_key) + s3.head_object(Bucket=str(bucket), Key=str(source_key)) except s3.exceptions.NoSuchKey: logger.warning("Descriptor not found, skipping archive: s3://%s/%s", bucket, source_key) return False @@ -247,8 +258,8 @@ def archive_descriptor( # noqa: PLR0913 raise _ = copy_object( - f"{bucket}/{source_key}", - f"{bucket}/{archive_key}", + str(bucket / source_key), + str(bucket / archive_key), ) logger.debug("Archived descriptor: s3://%s/%s -> %s", bucket, source_key, archive_key) return True diff --git a/src/cdm_data_loaders/ncbi_ftp/promote.py b/src/cdm_data_loaders/ncbi_ftp/promote.py index fe6fd24d..38b63111 100644 --- a/src/cdm_data_loaders/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/ncbi_ftp/promote.py @@ -36,7 +36,7 @@ logger = get_cdm_logger() -DEFAULT_LAKEHOUSE_KEY_PREFIX = "tenant-general-warehouse/kbase/datasets/ncbi/" +DEFAULT_LAKEHOUSE_KEY_PREFIX: PurePosixPath = PurePosixPath("tenant-general-warehouse/kbase/datasets/ncbi") _MAX_DRY_RUN_LOGS = 10 @@ -45,15 +45,15 @@ def promote_from_s3( # noqa: PLR0913 - staging_key_prefix: str, - staging_bucket: str, - lakehouse_bucket: str, - removed_manifest_path: str | Path | None = None, - updated_manifest_path: str | Path | None = None, - ncbi_release: str | None = None, - manifest_s3_key: str | None = None, - lakehouse_key_prefix: str = DEFAULT_LAKEHOUSE_KEY_PREFIX, *, + staging_bucket: PurePosixPath, + staging_key_prefix: PurePosixPath, + lakehouse_bucket: PurePosixPath, + lakehouse_key_prefix: PurePosixPath = DEFAULT_LAKEHOUSE_KEY_PREFIX, + removed_manifest_path: Path | None = None, + updated_manifest_path: Path | None = None, + manifest_s3_key: PurePosixPath | None = None, + ncbi_release: str | None = None, dry_run: bool = False, ) -> dict[str, Any]: """Promote files from an S3 staging prefix to the final Lakehouse path. @@ -61,24 +61,23 @@ def promote_from_s3( # noqa: PLR0913 Downloads each file to a temp location and re-uploads to the final path with MD5 metadata from ``.md5`` sidecar files. - :param staging_key_prefix: S3 key prefix where CTS output was written :param staging_bucket: S3 bucket containing the staged files (e.g. ``"cts"``) + :param staging_key_prefix: S3 key prefix where CTS output was written :param lakehouse_bucket: S3 bucket for the final Lakehouse destination (e.g. ``"cdm-lake"``) + :param lakehouse_key_prefix: S3 key prefix for final Lakehouse locations :param removed_manifest_path: local path to the removed_manifest file :param updated_manifest_path: local path to the updated_manifest file - :param ncbi_release: NCBI release version tag for archiving :param manifest_s3_key: S3 object key for transfer_manifest.txt (for trimming) - :param lakehouse_key_prefix: S3 key prefix for final Lakehouse locations + :param ncbi_release: NCBI release version tag for archiving :param dry_run: if True, log actions without side effects :return: report dict with counts """ # Get list of objects under the staging prefix - normalized_staging_key_prefix = staging_key_prefix.rstrip("/") + "/" - staged_objects: list[dict[str, Any]] = list_matching_objects(f"{staging_bucket}/{normalized_staging_key_prefix}") + staged_objects: list[dict[str, Any]] = list_matching_objects(f"{staging_bucket / staging_key_prefix}/") # Separate data files from sidecars - sidecars = {k["Key"] for k in staged_objects if k["Key"].endswith((".crc64nvme", ".md5"))} - data_files = [k["Key"] for k in staged_objects if k["Key"] not in sidecars] + sidecars = {PurePosixPath(k["Key"]) for k in staged_objects if k["Key"].endswith((".crc64nvme", ".md5"))} + data_files = [PurePosixPath(k["Key"]) for k in staged_objects if PurePosixPath(k["Key"]) not in sidecars] logger.info("Found %d data files and %d sidecars in staging", len(data_files), len(sidecars)) @@ -90,7 +89,7 @@ def promote_from_s3( # noqa: PLR0913 ]: if manifest_file and Path(str(manifest_file)).is_file(): archived += _archive_assemblies( - str(manifest_file), + manifest_file, lakehouse_bucket=lakehouse_bucket, ncbi_release=ncbi_release, lakehouse_key_prefix=lakehouse_key_prefix, @@ -102,7 +101,7 @@ def promote_from_s3( # noqa: PLR0913 promoted, failed, descriptors_written, promoted_accessions = _promote_data_files( data_files, sidecars, - normalized_staging_key_prefix, + staging_key_prefix, lakehouse_key_prefix, staging_bucket, lakehouse_bucket, @@ -138,39 +137,39 @@ def promote_from_s3( # noqa: PLR0913 def _group_files_by_assembly( - data_files: list[str], normalized_staging_prefix: str -) -> defaultdict[tuple[str, str], list[str]]: + data_files: list[PurePosixPath], staging_prefix: PurePosixPath +) -> defaultdict[tuple[PurePosixPath, str], list[PurePosixPath]]: """Group files by assembly; skip download_report.json and non-raw_data paths. :param data_files: list of S3 keys for staged data files - :param normalized_staging_prefix: normalized staging prefix in S3 + :param staging_prefix: staging prefix in S3 :return: dict mapping (assembly_dir, accession) to list of staged keys for that assembly """ - assembly_files: defaultdict[tuple[str, str], list[str]] = defaultdict(list) + assembly_files: defaultdict[tuple[PurePosixPath, str], list[PurePosixPath]] = defaultdict(list) for staged_key in data_files: - if staged_key.endswith("download_report.json"): + if staged_key.match("**/download_report.json"): continue - rel_path = staged_key[len(normalized_staging_prefix) :] - if not rel_path.startswith("raw_data/"): + rel_path = staged_key.relative_to(staging_prefix) + if not rel_path.is_relative_to("raw_data/"): continue - m = ASSEMBLY_PATH_REGEX.search(staged_key) + m = ASSEMBLY_PATH_REGEX.search(str(staged_key)) if m: - assembly_files[(m.group(1), m.group(2))].append(staged_key) + assembly_files[(PurePosixPath(m.group(1)), m.group(2))].append(staged_key) return assembly_files def _promote_file( # noqa: PLR0913 - staged_key: str, - normalized_staging_prefix: str, - lakehouse_key_prefix: str, - staging_bucket: str, - lakehouse_bucket: str, - sidecars: set[str], -) -> tuple[DescriptorResource, str]: + staged_key: PurePosixPath, + staging_prefix: PurePosixPath, + lakehouse_key_prefix: PurePosixPath, + staging_bucket: PurePosixPath, + lakehouse_bucket: PurePosixPath, + sidecars: set[PurePosixPath], +) -> tuple[DescriptorResource, PurePosixPath]: """Download one staged file, re-upload to Lakehouse with MD5 metadata. :param staged_key: S3 key of the staged file - :param normalized_staging_prefix: normalized staging prefix in S3 + :param staging_prefix: staging prefix in S3 :param lakehouse_key_prefix: S3 key prefix for final Lakehouse locations :param staging_bucket: S3 bucket containing the staged file :param lakehouse_bucket: S3 bucket for the final Lakehouse destination @@ -178,24 +177,24 @@ def _promote_file( # noqa: PLR0913 :return: ``(resource_dict, staged_key)`` on success; raises on failure. """ s3 = get_s3_client() - rel_path = staged_key[len(normalized_staging_prefix) :] - final_key = lakehouse_key_prefix + rel_path + rel_path = staged_key.relative_to(staging_prefix) + final_key = lakehouse_key_prefix / rel_path final_key_path = PurePosixPath(final_key) with tempfile.NamedTemporaryFile(delete=False) as tmp: tmp_path = tmp.name try: - s3.download_file(Bucket=staging_bucket, Key=staged_key, Filename=tmp_path) + s3.download_file(Bucket=str(staging_bucket), Key=str(staged_key), Filename=tmp_path) metadata: dict[str, str] = {} - md5_key = staged_key + ".md5" + md5_key = staged_key.with_name(f"{staged_key.name}.md5") if md5_key in sidecars: - md5_obj = s3.get_object(Bucket=staging_bucket, Key=md5_key) + md5_obj = s3.get_object(Bucket=str(staging_bucket), Key=str(md5_key)) metadata["md5"] = md5_obj["Body"].read().decode().strip() upload_succeeded = upload_file( tmp_path, - f"{lakehouse_bucket}/{final_key_path.parent}", + str(lakehouse_bucket / final_key_path.parent), user_metadata=metadata, object_name=final_key_path.name, show_progress=False, @@ -219,11 +218,11 @@ def _promote_file( # noqa: PLR0913 def _write_descriptor_for_assembly( - assembly_dir: str, + assembly_dir: PurePosixPath, accession: str, resources: list[DescriptorResource], - lakehouse_bucket: str, - lakehouse_key_prefix: str, + lakehouse_bucket: PurePosixPath, + lakehouse_key_prefix: PurePosixPath, ) -> bool: """Create and upload a frictionless descriptor for an assembly. @@ -236,7 +235,7 @@ def _write_descriptor_for_assembly( """ try: descriptor_key = build_descriptor_key(assembly_dir, lakehouse_key_prefix) - if object_exists(f"{lakehouse_bucket}/{descriptor_key}"): + if object_exists(str(lakehouse_bucket / descriptor_key)): logger.debug("Descriptor already exists, skipping: %s", descriptor_key) else: descriptor = create_descriptor(assembly_dir, accession, resources) @@ -251,9 +250,9 @@ def _write_descriptor_for_assembly( def _batch_delete( - promoted_keys: list[str], - sidecars: set[str], - staging_bucket: str, + promoted_keys: list[PurePosixPath], + sidecars: set[PurePosixPath], + staging_bucket: PurePosixPath, ) -> None: """Batch-delete all staged data files and their sidecars in one API call. @@ -263,23 +262,24 @@ def _batch_delete( """ keys_to_delete = list(promoted_keys) keys_to_delete.extend( - key + sidecar_ext + key.with_name(f"{key.name}{sidecar_ext}") for key in promoted_keys for sidecar_ext in (".md5", ".crc64nvme") - if key + sidecar_ext in sidecars + if key.with_name(f"{key.name}{sidecar_ext}") in sidecars ) - del_errors = delete_objects(staging_bucket, keys_to_delete) + string_keys_to_delete = [str(key) for key in keys_to_delete] + del_errors = delete_objects(str(staging_bucket), string_keys_to_delete) for err in del_errors: logger.warning("Failed to delete staged file %s: %s", err.get("Key"), err.get("Message")) def _promote_data_files( # noqa: PLR0913 - data_files: list[str], - sidecars: set[str], - normalized_staging_prefix: str, - lakehouse_key_prefix: str, - staging_bucket: str, - lakehouse_bucket: str, + data_files: list[PurePosixPath], + sidecars: set[PurePosixPath], + staging_prefix: PurePosixPath, + lakehouse_key_prefix: PurePosixPath, + staging_bucket: PurePosixPath, + lakehouse_bucket: PurePosixPath, *, dry_run: bool, ) -> tuple[int, int, int, set[str]]: @@ -297,12 +297,12 @@ def _promote_data_files( # noqa: PLR0913 failed = 0 descriptors_written = 0 promoted_accessions: set[str] = set() - assembly_files = _group_files_by_assembly(data_files, normalized_staging_prefix) + assembly_files = _group_files_by_assembly(data_files, staging_prefix) - def _promote_one(staged_key: str) -> tuple[DescriptorResource, str]: + def _promote_one(staged_key: PurePosixPath) -> tuple[DescriptorResource, PurePosixPath]: return _promote_file( staged_key, - normalized_staging_prefix, + staging_prefix, lakehouse_key_prefix, staging_bucket, lakehouse_bucket, @@ -315,12 +315,12 @@ def _promote_one(staged_key: str) -> tuple[DescriptorResource, str]: for (adir, acc), files in assembly_files.items(): assembly_failed = 0 resources: list[DescriptorResource] = [] - promoted_keys: list[str] = [] + promoted_keys: list[PurePosixPath] = [] if dry_run: for staged_key in files: - rel_path = staged_key[len(normalized_staging_prefix) :] - final_key = lakehouse_key_prefix + rel_path + rel_path = staged_key.relative_to(staging_prefix) + final_key = lakehouse_key_prefix / rel_path if _dry_run_log_count < _MAX_DRY_RUN_LOGS: logger.info("[dry-run] would promote: %s -> %s", staged_key, final_key) else: @@ -363,29 +363,27 @@ def _promote_one(staged_key: str) -> tuple[DescriptorResource, str]: # Archive assemblies -def _get_accession_path_prefix(accession: str, lakehouse_key_prefix: str) -> str | None: +def _get_accession_path_prefix(accession: str, lakehouse_key_prefix: PurePosixPath) -> PurePosixPath | None: """Get the S3 key prefix for all files related to an accession. :param accession: assembly accession (e.g. "GCF_000001405.39_Some_description") :param lakehouse_key_prefix: S3 key prefix for the Lakehouse dataset root :return: S3 key prefix under which all files for the accession are stored, or None if the accession format is invalid """ - m = ACCESSION_PARTS_REGEX.match(accession) + m = ACCESSION_PARTS_REGEX.match(accession) # returns, e.g., "GCF", "000", "001"', "405" captured groups if not m: logger.warning("Invalid accession format: %s", accession) return None - db = m.group(1) - p1, p2, p3 = m.group(2), m.group(3), m.group(4) - return f"{lakehouse_key_prefix}raw_data/{db}/{p1}/{p2}/{p3}/{accession}" + return PurePosixPath(lakehouse_key_prefix, "raw_data", *m.groups(), accession) def _get_source_dest_pairs_for_accession( accession: str, - lakehouse_bucket: str, - lakehouse_key_prefix: str, + lakehouse_bucket: PurePosixPath, + lakehouse_key_prefix: PurePosixPath, release_tag: str, archive_reason: str, -) -> list[tuple[str, str]]: +) -> list[tuple[PurePosixPath, PurePosixPath]]: """Get list of (source_key, archive_key) pairs for all objects related to an accession. :param accession: assembly accession (e.g. "GCF_000001405.39_Some_description") @@ -398,17 +396,21 @@ def _get_source_dest_pairs_for_accession( source_prefix = _get_accession_path_prefix(accession, lakehouse_key_prefix) if not source_prefix: return [] - matching_objs: list[dict[str, Any]] = list_matching_objects(f"{lakehouse_bucket}/{source_prefix}") + matching_objs: list[dict[str, Any]] = list_matching_objects(f"{lakehouse_bucket / source_prefix}") return [ ( - obj["Key"], - f"{lakehouse_key_prefix}archive/{release_tag}/{archive_reason}/{obj['Key'][len(lakehouse_key_prefix) :]}", + PurePosixPath(obj["Key"]), + lakehouse_key_prefix + / "archive" + / release_tag + / archive_reason + / PurePosixPath(obj["Key"]).relative_to(lakehouse_key_prefix), ) for obj in matching_objs ] -def _dry_run_output(key_pairs: list[tuple[str, str]], log_count: int) -> int: +def _dry_run_output(key_pairs: list[tuple[PurePosixPath, PurePosixPath]], log_count: int) -> int: """Log source and archive key pairs for a dry run, with a limit on how many are logged at INFO level. :param key_pairs: list of (source_key, archive_key) pairs @@ -425,7 +427,9 @@ def _dry_run_output(key_pairs: list[tuple[str, str]], log_count: int) -> int: return _dry_run_log_count -def _archive_objects(key_pairs: list[tuple[str, str]], lakehouse_bucket: str, *, delete_source: bool) -> int: +def _archive_objects( + key_pairs: list[tuple[PurePosixPath, PurePosixPath]], lakehouse_bucket: PurePosixPath, *, delete_source: bool +) -> int: """Copy objects from source keys to archive keys, optionally deleting the source objects. :param key_pairs: list of (source_key, archive_key) pairs @@ -436,14 +440,14 @@ def _archive_objects(key_pairs: list[tuple[str, str]], lakehouse_bucket: str, *, archived = 0 if not key_pairs: return archived - keys_to_delete: list[str] = [] + keys_to_delete: list[str] = [] # strings so they can be passed to delete_objects() in s3 module n_workers = min(32, len(key_pairs)) with ThreadPoolExecutor(max_workers=n_workers) as executor: futures = { executor.submit( copy_object, - f"{lakehouse_bucket}/{src}", - f"{lakehouse_bucket}/{arch}", + str(lakehouse_bucket / src), + str(lakehouse_bucket / arch), ): src for src, arch in key_pairs } @@ -453,13 +457,13 @@ def _archive_objects(key_pairs: list[tuple[str, str]], lakehouse_bucket: str, *, future.result() archived += 1 if delete_source: - keys_to_delete.append(src) + keys_to_delete.append(str(src)) logger.debug(" Archived: %s", src) except Exception: logger.exception("Failed to archive %s", src) if delete_source and keys_to_delete: - del_errors = delete_objects(lakehouse_bucket, keys_to_delete) + del_errors = delete_objects(str(lakehouse_bucket), keys_to_delete) for err in del_errors: logger.warning("Failed to delete %s: %s", err.get("Key"), err.get("Message")) @@ -467,10 +471,10 @@ def _archive_objects(key_pairs: list[tuple[str, str]], lakehouse_bucket: str, *, def _archive_assemblies( # noqa: PLR0913 - manifest_local_path: str, - lakehouse_bucket: str, + manifest_local_path: Path, + lakehouse_bucket: PurePosixPath, ncbi_release: str | None = None, - lakehouse_key_prefix: str = DEFAULT_LAKEHOUSE_KEY_PREFIX, + lakehouse_key_prefix: PurePosixPath = DEFAULT_LAKEHOUSE_KEY_PREFIX, archive_reason: str = "unknown", *, delete_source: bool = False, @@ -501,7 +505,7 @@ def _archive_assemblies( # noqa: PLR0913 _dry_run_log_count = 0 for accession in tqdm.tqdm(accessions, unit="accession", desc="Archiving"): # get list of (source_key, archive_key) pairs for all objects related to this accession - key_pairs: list[tuple[str, str]] = _get_source_dest_pairs_for_accession( + key_pairs: list[tuple[PurePosixPath, PurePosixPath]] = _get_source_dest_pairs_for_accession( accession, lakehouse_bucket, lakehouse_key_prefix, @@ -520,11 +524,10 @@ def _archive_assemblies( # noqa: PLR0913 archived += _archive_objects(key_pairs, lakehouse_bucket, delete_source=delete_source) # Infer assembly_dir from key paths for descriptor archival - assembly_dir: str | None = None + assembly_dir: PurePosixPath | None = None for src, _ in key_pairs: - adir_match = ASSEMBLY_PATH_REGEX.search(src) - if adir_match: - assembly_dir = adir_match.group(1) + if adir_match := ASSEMBLY_PATH_REGEX.search(f"{src}"): + assembly_dir = PurePosixPath(adir_match.group(1)) break # Archive the frictionless descriptor alongside raw data @@ -550,7 +553,9 @@ def _archive_assemblies( # noqa: PLR0913 # Manifest trimming -def _trim_manifest(manifest_s3_key: str, staging_bucket: str, promoted_accessions: set[str]) -> None: +def _trim_manifest( + manifest_s3_key: PurePosixPath, staging_bucket: PurePosixPath, promoted_accessions: set[str] +) -> None: """Remove promoted accessions from the transfer manifest in S3. :param manifest_s3_key: S3 object key of the transfer_manifest.txt @@ -564,7 +569,7 @@ def _trim_manifest(manifest_s3_key: str, staging_bucket: str, promoted_accession try: try: - s3.download_file(Bucket=staging_bucket, Key=manifest_s3_key, Filename=tmp_path) + s3.download_file(Bucket=str(staging_bucket), Key=str(manifest_s3_key), Filename=tmp_path) except s3.exceptions.NoSuchKey: logger.warning("Manifest not found in S3 (s3://%s/%s) — skipping trim", staging_bucket, manifest_s3_key) return @@ -582,7 +587,7 @@ def _trim_manifest(manifest_s3_key: str, staging_bucket: str, promoted_accession with Path(tmp_path).open("w") as f: f.writelines(remaining) - s3.upload_file(Filename=tmp_path, Bucket=staging_bucket, Key=manifest_s3_key) + s3.upload_file(Filename=tmp_path, Bucket=str(staging_bucket), Key=str(manifest_s3_key)) logger.info( "Trimmed manifest: %d -> %d entries (%d promoted)", len(lines), diff --git a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py index 52e42891..a309aa70 100644 --- a/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py +++ b/src/cdm_data_loaders/pipelines/ncbi_ftp_download.py @@ -13,7 +13,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import UTC, datetime from ftplib import error_temp -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any import tqdm @@ -36,7 +36,7 @@ logger = get_cdm_logger() -DEFAULT_STAGING_KEY_PREFIX = "staging/" +DEFAULT_STAGING_KEY_PREFIX: PurePosixPath = PurePosixPath("staging") class DownloadSettings(BaseSettings): @@ -44,13 +44,13 @@ class DownloadSettings(BaseSettings): model_config = SettingsConfigDict(**DEFAULT_SETTINGS_CONFIG_DICT) - manifest: str = Field( - default=f"{INPUT_MOUNT}/transfer_manifest.txt", + manifest: Path = Field( + default=Path(INPUT_MOUNT) / "transfer_manifest.txt", description="Path to the transfer manifest file listing FTP paths to download", validation_alias=AliasChoices("m", "manifest"), ) - output_dir: str = Field( - default=OUTPUT_MOUNT, + output_dir: Path = Field( + default=Path(OUTPUT_MOUNT), description="Output directory for downloaded assembly files", validation_alias=AliasChoices("output-dir", "output_dir"), ) @@ -80,8 +80,8 @@ class DownloadSettings(BaseSettings): def _upload_assembly_dir( assembly_dir: Path, tmp_root: Path, - bucket: str, - staging_key_prefix: str, + bucket: PurePosixPath, + staging_key_prefix: PurePosixPath, ) -> int: """Upload all files under *assembly_dir* to S3, deleting each file immediately after upload. @@ -101,8 +101,8 @@ def _upload_assembly_dir( for f in sorted(assembly_dir.rglob("*")): if f.is_file(): relative = f.relative_to(tmp_root) - dest_prefix = f"{bucket}/{staging_key_prefix.rstrip('/')}/{relative.parent}" - if upload_file(f, dest_prefix, show_progress=False): + dest_prefix = bucket / staging_key_prefix / relative.parent + if upload_file(f, str(dest_prefix), show_progress=False): count += 1 else: logger.warning("Failed to upload %s to %s", f, dest_prefix) @@ -115,8 +115,8 @@ def _upload_assembly_dir( def download_batch( - manifest_path: str | Path, - output_dir: str | Path, + manifest_path: Path, + output_dir: Path, threads: int = 4, ftp_host: str = FTP_HOST, limit: int | None = None, @@ -130,8 +130,8 @@ def download_batch( :param limit: optional limit for testing :return: report dict with overall stats """ - with Path(manifest_path).open() as f: - assembly_paths = [line.strip() for line in f if line.strip() and not line.startswith("#")] + with manifest_path.open() as f: + assembly_paths = [PurePosixPath(line.strip()) for line in f if line.strip() and not line.startswith("#")] if limit: assembly_paths = assembly_paths[:limit] @@ -144,7 +144,7 @@ def download_batch( failed: list[dict[str, str]] = [] all_stats: list[dict[str, Any]] = [] - def _download_one(path: str) -> tuple[str, Exception | None]: + def _download_one(path: PurePosixPath) -> tuple[str, Exception | None]: nonlocal success_count @retry( @@ -160,12 +160,12 @@ def _attempt() -> dict[str, Any]: try: stats = _attempt() except Exception as e: # noqa: BLE001 - return path, e + return str(path), e else: with lock: success_count += 1 all_stats.append(stats) - return path, None + return str(path), None try: with ( @@ -194,7 +194,7 @@ def _attempt() -> dict[str, Any]: "assembly_stats": all_stats, } - report_path = Path(output_dir) / "download_report.json" + report_path = output_dir / "download_report.json" report_path.parent.mkdir(parents=True, exist_ok=True) with report_path.open("w") as f: json.dump(report, f, indent=2) @@ -238,12 +238,12 @@ def cli() -> None: # Notebook / interactive entry point -def download_and_stage( +def download_and_stage( # noqa: PLR0913, PLR0915 *, - bucket: str, - staging_key_prefix: str, - manifest_s3_key: str | None = None, - manifest_local_path: str | Path | None = None, + bucket: PurePosixPath, + staging_key_prefix: PurePosixPath, + manifest_s3_key: PurePosixPath | None = None, + manifest_local_path: Path | None = None, threads: int = 4, ftp_host: str = FTP_HOST, limit: int | None = None, @@ -281,15 +281,15 @@ def download_and_stage( if manifest_s3_key is not None: s3 = get_s3_client() - response = s3.get_object(Bucket=bucket, Key=manifest_s3_key) + response = s3.get_object(Bucket=str(bucket), Key=str(manifest_s3_key)) manifest_dest.write_bytes(response["Body"].read()) logger.info("Manifest read from S3: s3://%s/%s", bucket, manifest_s3_key) - else: - manifest_dest.write_bytes(Path(manifest_local_path).read_bytes()) + elif manifest_local_path: + manifest_dest.write_bytes(manifest_local_path.read_bytes()) logger.info("Manifest read from local path: %s", manifest_local_path) with manifest_dest.open() as f: - assembly_paths = [line.strip() for line in f if line.strip() and not line.startswith("#")] + assembly_paths = [PurePosixPath(line.strip()) for line in f if line.strip() and not line.startswith("#")] if limit: assembly_paths = assembly_paths[:limit] @@ -303,7 +303,7 @@ def download_and_stage( failed: list[dict[str, str]] = [] all_stats: list[dict[str, Any]] = [] - def _download_upload_one(path: str) -> tuple[str, Exception | None]: + def _download_upload_one(path: PurePosixPath) -> tuple[str, Exception | None]: nonlocal success_count, staged_objects @retry( @@ -319,7 +319,7 @@ def _attempt() -> dict[str, Any]: try: stats = _attempt() except Exception as e: # noqa: BLE001 - return path, e + return str(path), e if not dry_run: _db, assembly_dir_name, _accession = parse_assembly_path(path) @@ -331,7 +331,7 @@ def _attempt() -> dict[str, Any]: with lock: success_count += 1 all_stats.append(stats) - return path, None + return str(path), None try: with ( @@ -364,7 +364,7 @@ def _attempt() -> dict[str, Any]: report_path = tmp / "download_report.json" with report_path.open("w") as f: json.dump(report, f, indent=2) - if upload_file(report_path, f"{bucket}/{staging_key_prefix.rstrip('/')}", show_progress=False): + if upload_file(report_path, str(bucket / staging_key_prefix), show_progress=False): staged_objects += 1 else: logger.warning("Failed to upload download report to s3://%s/%s", bucket, staging_key_prefix) diff --git a/src/cdm_data_loaders/utils/ftp_client.py b/src/cdm_data_loaders/utils/ftp_client.py index 8bf5f1f7..6eab9cd4 100644 --- a/src/cdm_data_loaders/utils/ftp_client.py +++ b/src/cdm_data_loaders/utils/ftp_client.py @@ -11,7 +11,7 @@ import threading import time from ftplib import FTP, error_temp -from pathlib import Path +from pathlib import Path, PurePosixPath from tenacity import ( before_sleep_log, @@ -49,6 +49,8 @@ def _set_keepalive(ftp: FTP, idle: int = 30, interval: int = 10, count: int = 3) verification. """ sock = ftp.sock + if sock is None: + return sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) if hasattr(socket, "TCP_KEEPIDLE"): sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, idle) @@ -73,7 +75,7 @@ def ftp_noop_keepalive(ftp: FTP, last_activity: float, interval: int = 25) -> fl return last_activity -def ftp_list_dir(ftp: FTP, path: str, retries: int = 3) -> list[str]: +def ftp_list_dir(ftp: FTP, path: PurePosixPath, retries: int = 3) -> list[PurePosixPath]: """List files in an FTP directory with retry on transient errors. :param ftp: active FTP connection @@ -81,7 +83,7 @@ def ftp_list_dir(ftp: FTP, path: str, retries: int = 3) -> list[str]: :param retries: number of retry attempts :return: list of filenames """ - ftp.cwd(path) + ftp.cwd(str(path)) @retry( retry=retry_if_exception_type(error_temp), @@ -90,15 +92,15 @@ def ftp_list_dir(ftp: FTP, path: str, retries: int = 3) -> list[str]: reraise=True, before_sleep=before_sleep_log(logger, logging.WARNING), ) - def _list() -> list[str]: - files: list[str] = [] - ftp.retrlines("NLST", files.append) - return files + def _list() -> list[PurePosixPath]: + file_strings: list[str] = [] + ftp.retrlines("NLST", file_strings.append) + return [PurePosixPath(f) for f in file_strings] return _list() -def ftp_download_file(ftp: FTP, remote_path: str, local_path: str, retries: int = 3) -> None: +def ftp_download_file(ftp: FTP, remote_path: PurePosixPath, local_path: Path, retries: int = 3) -> None: """Download a single file from FTP with retry on transient errors. :param ftp: active FTP connection @@ -115,13 +117,13 @@ def ftp_download_file(ftp: FTP, remote_path: str, local_path: str, retries: int before_sleep=before_sleep_log(logger, logging.WARNING), ) def _download() -> None: - with Path(local_path).open("wb") as f: + with local_path.open("wb") as f: ftp.retrbinary(f"RETR {remote_path}", f.write) _download() -def ftp_retrieve_text(ftp: FTP, remote_path: str) -> str: +def ftp_retrieve_text(ftp: FTP, remote_path: PurePosixPath) -> str: """Retrieve a text file from FTP, returning its content as a string. :param ftp: active FTP connection @@ -164,7 +166,7 @@ def get(self) -> FTP: if ftp is not None: try: ftp.voidcmd("NOOP") - except Exception: + except Exception: # noqa: BLE001 # Connection is stale — discard it and reconnect below with contextlib.suppress(Exception): ftp.quit() diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 48ce113e..34377a44 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -8,7 +8,8 @@ import hashlib import re -from pathlib import Path +from collections.abc import Generator +from pathlib import Path, PurePosixPath from typing import Any from unittest.mock import patch @@ -16,6 +17,7 @@ import botocore.client import botocore.config import pytest +from botocore.exceptions import ClientError import cdm_data_loaders.ncbi_ftp.manifest as manifest_mod import cdm_data_loaders.ncbi_ftp.promote as promote_mod @@ -69,7 +71,7 @@ def pytest_runtest_setup(item: pytest.Item) -> None: @pytest.fixture -def ceph_s3_client() -> botocore.client.BaseClient: +def ceph_s3_client() -> Generator[botocore.client.BaseClient]: """Session-scoped real boto3 S3 client pointed at the local CEPH instance. Patches ``get_s3_client`` on every module that uses it so internal calls @@ -108,7 +110,7 @@ def _bucket_name_from_node(node_id: str) -> str: @pytest.fixture -def test_bucket(ceph_s3_client: botocore.client.BaseClient, request: pytest.FixtureRequest) -> str: +def test_bucket(ceph_s3_client: botocore.client.BaseClient, request: pytest.FixtureRequest) -> PurePosixPath: """Create a per-test-method bucket in CEPH and return its name. On re-run, any existing objects are deleted first so the test starts clean. @@ -126,17 +128,17 @@ def test_bucket(ceph_s3_client: botocore.client.BaseClient, request: pytest.Fixt s3.delete_object(Bucket=bucket, Key=obj["Key"]) except s3.exceptions.NoSuchBucket: s3.create_bucket(Bucket=bucket) - except botocore.exceptions.ClientError as e: + except ClientError as e: if e.response["Error"]["Code"] in ("404", "NoSuchBucket"): s3.create_bucket(Bucket=bucket) else: raise - return bucket + return PurePosixPath(bucket) @pytest.fixture -def staging_test_bucket(ceph_s3_client: botocore.client.BaseClient, request: pytest.FixtureRequest) -> str: +def staging_test_bucket(ceph_s3_client: botocore.client.BaseClient, request: pytest.FixtureRequest) -> PurePosixPath: """Create a per-test staging bucket in CEPH and return its name. Mirrors ``test_bucket`` but uses a ``staging-`` prefix so staging and @@ -156,13 +158,13 @@ def staging_test_bucket(ceph_s3_client: botocore.client.BaseClient, request: pyt s3.delete_object(Bucket=bucket, Key=obj["Key"]) except s3.exceptions.NoSuchBucket: s3.create_bucket(Bucket=bucket) - except botocore.exceptions.ClientError as e: + except ClientError as e: if e.response["Error"]["Code"] in ("404", "NoSuchBucket"): s3.create_bucket(Bucket=bucket) else: raise - return bucket + return PurePosixPath(bucket) # Helpers @@ -170,10 +172,10 @@ def staging_test_bucket(ceph_s3_client: botocore.client.BaseClient, request: pyt def stage_files_to_ceph( s3: botocore.client.BaseClient, - bucket: str, - local_dir: str | Path, - staging_prefix: str, -) -> list[str]: + bucket: PurePosixPath, + local_dir: Path, + staging_prefix: PurePosixPath, +) -> list[PurePosixPath]: """Upload a local directory tree to a CEPH staging prefix. :param s3: boto3 S3 client @@ -183,25 +185,25 @@ def stage_files_to_ceph( :return: list of S3 keys uploaded """ local_dir = Path(local_dir) - keys: list[str] = [] + keys: list[PurePosixPath] = [] for path in sorted(local_dir.rglob("*")): if path.is_dir(): continue rel = path.relative_to(local_dir) - key = f"{staging_prefix.rstrip('/')}/{rel}" - s3.upload_file(Filename=str(path), Bucket=bucket, Key=key) + key = staging_prefix / rel + s3.upload_file(Filename=str(path), Bucket=str(bucket), Key=str(key)) keys.append(key) return keys def seed_lakehouse( # noqa: PLR0913 s3: botocore.client.BaseClient, - bucket: str, + bucket: PurePosixPath, accession: str, - files: dict[str, str | bytes], - path_prefix: str, - assembly_dir: str | None = None, -) -> list[str]: + files: dict[PurePosixPath, str | bytes], + path_prefix: PurePosixPath, + assembly_dir: PurePosixPath | None = None, +) -> list[PurePosixPath]: """Seed assembly files at the final Lakehouse path in CEPH. :param s3: boto3 S3 client @@ -212,19 +214,22 @@ def seed_lakehouse( # noqa: PLR0913 :param assembly_dir: full assembly dir name; if None, uses ``accession`` :return: list of S3 keys created """ - adir = assembly_dir or accession + adir = assembly_dir or PurePosixPath(accession) rel = build_accession_path(adir) - keys: list[str] = [] + keys: list[PurePosixPath] = [] + prefix = PurePosixPath(path_prefix) for fname, content in files.items(): - key = f"{path_prefix}{rel}{fname}" + key = prefix / rel / fname body = content.encode() if isinstance(content, str) else content md5 = hashlib.md5(body).hexdigest() # noqa: S324 - s3.put_object(Bucket=bucket, Key=key, Body=body, Metadata={"md5": md5}) + s3.put_object(Bucket=str(bucket), Key=str(key), Body=body, Metadata={"md5": md5}) keys.append(key) return keys -def list_all_keys(s3: botocore.client.BaseClient, bucket: str, prefix: str = "") -> list[str]: +def list_all_keys( + s3: botocore.client.BaseClient, bucket: PurePosixPath, prefix: PurePosixPath | None = None +) -> list[PurePosixPath]: """List all object keys in a bucket under a prefix. :param s3: boto3 S3 client @@ -232,14 +237,17 @@ def list_all_keys(s3: botocore.client.BaseClient, bucket: str, prefix: str = "") :param prefix: optional key prefix filter :return: sorted list of keys """ - keys: list[str] = [] + keys: list[PurePosixPath] = [] paginator = s3.get_paginator("list_objects_v2") - for page in paginator.paginate(Bucket=bucket, Prefix=prefix): - keys.extend(obj["Key"] for obj in page.get("Contents", [])) + paginate_kwargs: dict[str, str] = {"Bucket": str(bucket)} + if prefix is not None: + paginate_kwargs["Prefix"] = str(prefix) + for page in paginator.paginate(**paginate_kwargs): + keys.extend(PurePosixPath(obj["Key"]) for obj in page.get("Contents", [])) return sorted(keys) -def get_object_metadata(s3: botocore.client.BaseClient, bucket: str, key: str) -> dict[str, Any]: +def get_object_metadata(s3: botocore.client.BaseClient, bucket: PurePosixPath, key: PurePosixPath) -> dict[str, Any]: """Return the S3 user metadata dict for an S3 object (from HeadObject). :param s3: boto3 S3 client @@ -247,5 +255,5 @@ def get_object_metadata(s3: botocore.client.BaseClient, bucket: str, key: str) - :param key: object key :return: user metadata dict """ - resp = s3.head_object(Bucket=bucket, Key=key) + resp = s3.head_object(Bucket=str(bucket), Key=str(key)) return resp.get("Metadata", {}) diff --git a/tests/integration/test_download_e2e.py b/tests/integration/test_download_e2e.py index 74e39848..b40a3acf 100644 --- a/tests/integration/test_download_e2e.py +++ b/tests/integration/test_download_e2e.py @@ -6,10 +6,11 @@ """ import json -from pathlib import Path +from pathlib import Path, PurePosixPath from unittest.mock import patch import pytest +from botocore.client import BaseClient import cdm_data_loaders.utils.s3 as s3_utils from cdm_data_loaders.ncbi_ftp.manifest import ( @@ -57,8 +58,8 @@ def test_download_small_batch(self, tmp_path: Path) -> None: output_dir.mkdir() report = download_batch( - manifest_path=str(manifest_path), - output_dir=str(output_dir), + manifest_path=manifest_path, + output_dir=output_dir, threads=1, limit=1, ) @@ -100,8 +101,8 @@ def test_download_resume(self, tmp_path: Path) -> None: # First download report1 = download_batch( - manifest_path=str(manifest_path), - output_dir=str(output_dir), + manifest_path=manifest_path, + output_dir=output_dir, threads=1, limit=1, ) @@ -111,8 +112,8 @@ def test_download_resume(self, tmp_path: Path) -> None: # Second download — same manifest report2 = download_batch( - manifest_path=str(manifest_path), - output_dir=str(output_dir), + manifest_path=manifest_path, + output_dir=output_dir, threads=1, limit=1, ) @@ -131,19 +132,19 @@ def test_download_resume(self, tmp_path: Path) -> None: @pytest.mark.external_request def test_download_and_stage_e2e( tmp_path: Path, - ceph_s3_client, - test_bucket: str, + ceph_s3_client: BaseClient, + test_bucket: PurePosixPath, ) -> None: """Download one assembly and verify it is staged under the expected S3 prefix.""" manifest_path, _acc = _manifest_for_one_assembly(tmp_path) - staging_prefix = "staging/e2e-test/" + staging_prefix = PurePosixPath("staging") / "e2e-test" # Seed the manifest in CEPH so download_and_stage can read it from S3 - manifest_s3_key = f"{staging_prefix}input/transfer_manifest.txt" + manifest_s3_key = staging_prefix / "input" / "transfer_manifest.txt" ceph_s3_client.put_object( - Bucket=test_bucket, - Key=manifest_s3_key, + Bucket=str(test_bucket), + Key=str(manifest_s3_key), Body=manifest_path.read_bytes(), ) @@ -167,7 +168,7 @@ def test_download_and_stage_e2e( paginator = ceph_s3_client.get_paginator("list_objects_v2") staged_keys = [ obj["Key"] - for page in paginator.paginate(Bucket=test_bucket, Prefix=f"{staging_prefix}raw_data/") + for page in paginator.paginate(Bucket=str(test_bucket), Prefix=str(staging_prefix / "raw_data")) for obj in page.get("Contents", []) ] assert len(staged_keys) > 0, "Expected staged files under raw_data/" @@ -178,7 +179,7 @@ def test_download_and_stage_e2e( assert len(md5_files) > 0, "Expected .md5 sidecar files" # Verify download_report.json was also uploaded - report_key = f"{staging_prefix}download_report.json" - resp = ceph_s3_client.get_object(Bucket=test_bucket, Key=report_key) + report_key = staging_prefix / "download_report.json" + resp = ceph_s3_client.get_object(Bucket=str(test_bucket), Key=str(report_key)) saved_report = json.loads(resp["Body"].read()) assert saved_report["succeeded"] >= 1 diff --git a/tests/integration/test_full_pipeline.py b/tests/integration/test_full_pipeline.py index 08a2c9ab..19e88521 100644 --- a/tests/integration/test_full_pipeline.py +++ b/tests/integration/test_full_pipeline.py @@ -7,9 +7,10 @@ ``slow_test``. """ -from pathlib import Path +from pathlib import Path, PurePosixPath import pytest +from botocore.client import BaseClient from cdm_data_loaders.ncbi_ftp.manifest import ( AssemblyRecord, @@ -27,7 +28,7 @@ from .conftest import get_object_metadata, list_all_keys, stage_files_to_ceph STABLE_PREFIX = "900" -STAGING_PREFIX = "staging/run1/" +STAGING_PREFIX = PurePosixPath("staging") / "run1" PATH_PREFIX = DEFAULT_LAKEHOUSE_KEY_PREFIX @@ -39,9 +40,9 @@ class TestFullPipelineSmallBatch: def test_full_pipeline_small_batch( self, - ceph_s3_client: object, - test_bucket: str, - staging_test_bucket: str, + ceph_s3_client: BaseClient, + test_bucket: PurePosixPath, + staging_test_bucket: PurePosixPath, tmp_path: Path, ) -> None: """Single assembly flows through all three phases into CEPH.""" @@ -63,8 +64,8 @@ def test_full_pipeline_small_batch( output_dir.mkdir() report = download_batch( - manifest_path=str(manifest_path), - output_dir=str(output_dir), + manifest_path=manifest_path, + output_dir=output_dir, threads=1, limit=1, ) @@ -86,7 +87,7 @@ def test_full_pipeline_small_batch( assert promote_report["failed"] == 0 # Verify final state - final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX / "raw_data") assert len(final_keys) >= 1, "Expected files at final Lakehouse path" # At least one file should have MD5 metadata @@ -107,9 +108,9 @@ class TestFullPipelineIncrementalSync: def test_full_pipeline_incremental( self, - ceph_s3_client: object, - test_bucket: str, - staging_test_bucket: str, + ceph_s3_client: BaseClient, + test_bucket: PurePosixPath, + staging_test_bucket: PurePosixPath, tmp_path: Path, ) -> None: """Second sync archives the old version and promotes the new one.""" @@ -128,14 +129,14 @@ def test_full_pipeline_incremental( output1 = tmp_path / "output1" output1.mkdir() - report1 = download_batch(str(manifest1), str(output1), threads=1, limit=1) + report1 = download_batch(manifest1, output1, threads=1, limit=1) assert report1["succeeded"] >= 1 stage_files_to_ceph(s3, staging_test_bucket, output1, STAGING_PREFIX) # Upload manifest to CEPH for trimming (manifest lives in staging bucket) - manifest_key = "ncbi/transfer_manifest.txt" - s3.upload_file(Filename=str(manifest1), Bucket=staging_test_bucket, Key=manifest_key) + manifest_key = PurePosixPath("ncbi") / "transfer_manifest.txt" + s3.upload_file(Filename=str(manifest1), Bucket=str(staging_test_bucket), Key=str(manifest_key)) promote1 = promote_from_s3( staging_key_prefix=STAGING_PREFIX, @@ -146,7 +147,7 @@ def test_full_pipeline_incremental( ) assert promote1["promoted"] >= 1 - first_sync_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + first_sync_keys = list_all_keys(s3, test_bucket, PATH_PREFIX / "raw_data") assert len(first_sync_keys) >= 1 # Second sync: Manufacture "previous" with a tweak @@ -158,7 +159,7 @@ def test_full_pipeline_incremental( accession=rec.accession, status=rec.status, seq_rel_date=rec.seq_rel_date, - ftp_path=rec.ftp_path, + ftp_url=rec.ftp_url, assembly_dir=rec.assembly_dir, ) @@ -185,13 +186,13 @@ def test_full_pipeline_incremental( # Phase 2 — re-download the updated assembly output2 = tmp_path / "output2" output2.mkdir() - report2 = download_batch(str(manifest2), str(output2), threads=1, limit=1) + report2 = download_batch(manifest2, output2, threads=1, limit=1) assert report2["succeeded"] >= 1 # Clean staging and re-stage staging_keys = list_all_keys(s3, staging_test_bucket, STAGING_PREFIX) for key in staging_keys: - s3.delete_object(Bucket=staging_test_bucket, Key=key) + s3.delete_object(Bucket=str(staging_test_bucket), Key=str(key)) stage_files_to_ceph(s3, staging_test_bucket, output2, STAGING_PREFIX) # Phase 3 — promote with archival @@ -199,19 +200,19 @@ def test_full_pipeline_incremental( staging_key_prefix=STAGING_PREFIX, staging_bucket=staging_test_bucket, lakehouse_bucket=test_bucket, - updated_manifest_path=str(updated_manifest), + updated_manifest_path=updated_manifest, ncbi_release="test-incremental", lakehouse_key_prefix=PATH_PREFIX, ) assert promote2["failed"] == 0 # Verify archive exists - archive_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "archive/test-incremental/") + archive_keys = list_all_keys(s3, PurePosixPath(test_bucket), PATH_PREFIX / "archive/test-incremental") if promote2["archived"] > 0: assert len(archive_keys) >= 1 for key in archive_keys: - assert "/updated/" in key + assert "updated" in key.parts # Final Lakehouse path should still have files - final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX / "raw_data") assert len(final_keys) >= 1 diff --git a/tests/integration/test_manifest_e2e.py b/tests/integration/test_manifest_e2e.py index 6cf2851b..bacefd0d 100644 --- a/tests/integration/test_manifest_e2e.py +++ b/tests/integration/test_manifest_e2e.py @@ -5,12 +5,11 @@ (if a running CEPH test store is required) and ``slow_test``. """ -from __future__ import annotations - from itertools import islice -from typing import TYPE_CHECKING +from pathlib import Path, PurePosixPath import pytest +from botocore.client import BaseClient from cdm_data_loaders.ncbi_ftp.assembly import FILE_FILTERS, FTP_HOST, build_accession_path, parse_md5_checksums_file from cdm_data_loaders.ncbi_ftp.manifest import ( @@ -29,9 +28,6 @@ from cdm_data_loaders.ncbi_ftp.promote import DEFAULT_LAKEHOUSE_KEY_PREFIX from cdm_data_loaders.utils.ftp_client import connect_ftp, ftp_retrieve_text -if TYPE_CHECKING: - from pathlib import Path - # Use a high-numbered prefix range that typically has only a handful of # assemblies, keeping FTP traffic minimal. STABLE_PREFIX = "900" @@ -113,7 +109,7 @@ def test_incremental_diff(self, tmp_path: Path) -> None: accession=rec.accession, status=rec.status, seq_rel_date=rec.seq_rel_date, - ftp_path=rec.ftp_path, + ftp_url=rec.ftp_url, assembly_dir=rec.assembly_dir, ) @@ -131,8 +127,8 @@ def test_incremental_diff(self, tmp_path: Path) -> None: accession=fake_suppressed, status="latest", seq_rel_date="2020/01/01", - ftp_path="https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/900/999/999/GCF_900999999.1_FakeAsm", - assembly_dir="GCF_900999999.1_FakeAsm", + ftp_url="https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/900/999/999/GCF_900999999.1_FakeAsm", + assembly_dir=PurePosixPath("GCF_900999999.1_FakeAsm"), ) diff = compute_diff(filtered, previous_assemblies=previous) @@ -163,8 +159,8 @@ class TestVerifyTransferCandidatesPrunes: def test_prunes_existing_matching_md5( self, - ceph_s3_client: object, - test_bucket: str, + ceph_s3_client: BaseClient, + test_bucket: PurePosixPath, ) -> None: """Assemblies with matching MD5 metadata in CEPH are pruned from the transfer list.""" _full, filtered = _download_and_filter() @@ -180,12 +176,12 @@ def test_prunes_existing_matching_md5( acc_part = next(i_latest) rec = latest[acc] rec_part = latest[acc_part] - ftp_dir = rec.ftp_path.replace("https://ftp.ncbi.nlm.nih.gov", "") + ftp_dir = PurePosixPath(rec.ftp_url.replace("https://ftp.ncbi.nlm.nih.gov", "")) # Fetch the real md5checksums.txt from FTP ftp = connect_ftp(FTP_HOST) try: - md5_text = ftp_retrieve_text(ftp, ftp_dir.rstrip("/") + "/md5checksums.txt") + md5_text = ftp_retrieve_text(ftp, ftp_dir / "md5checksums.txt") finally: ftp.quit() @@ -196,22 +192,22 @@ def test_prunes_existing_matching_md5( path_prefix = DEFAULT_LAKEHOUSE_KEY_PREFIX rel = build_accession_path(rec.assembly_dir) for fname, md5 in checksums.items(): - if any(fname.endswith(suffix) for suffix in FILE_FILTERS): - key = f"{path_prefix}{rel}{fname}" + if any(fname.match(f"**{suffix}") for suffix in FILE_FILTERS): + key = path_prefix / rel / fname s3.put_object( - Bucket=test_bucket, - Key=key, + Bucket=str(test_bucket), + Key=str(key), Body=b"placeholder", Metadata={"md5": md5}, ) rel = build_accession_path(rec_part.assembly_dir) file_count = 0 for fname, md5 in checksums.items(): - if any(fname.endswith(suffix) for suffix in FILE_FILTERS): - key = f"{path_prefix}{rel}{fname}" + if any(fname.match(f"**{suffix}") for suffix in FILE_FILTERS): + key = path_prefix / rel / fname s3.put_object( - Bucket=test_bucket, - Key=key, + Bucket=str(test_bucket), + Key=str(key), Body=b"placeholder", Metadata={"md5": md5}, ) @@ -243,8 +239,8 @@ class TestScanStoreToSyntheticSummary: def test_builds_summary_from_ceph_store( self, - ceph_s3_client: object, - test_bucket: str, + ceph_s3_client: BaseClient, + test_bucket: PurePosixPath, ) -> None: """Verify synthetic summary captures assemblies from CEPH.""" s3 = ceph_s3_client @@ -258,10 +254,10 @@ def test_builds_summary_from_ceph_store( for assembly_dir, files in assemblies.items(): for fname in files: - key = f"{path_prefix}refseq/{assembly_dir}/{assembly_dir}{fname}" + key = path_prefix / "refseq" / assembly_dir / f"{assembly_dir}{fname}" s3.put_object( - Bucket=test_bucket, - Key=key, + Bucket=str(test_bucket), + Key=str(key), Body=b"placeholder", ) @@ -276,20 +272,20 @@ def test_builds_summary_from_ceph_store( rec1 = result["GCF_000001215.4"] assert rec1.accession == "GCF_000001215.4" assert rec1.status == "latest" - assert rec1.assembly_dir == "GCF_000001215.4_v1" + assert rec1.assembly_dir == PurePosixPath("GCF_000001215.4_v1") def test_synthetic_summary_diff_against_current( self, - ceph_s3_client: object, - test_bucket: str, + ceph_s3_client: BaseClient, + test_bucket: PurePosixPath, ) -> None: """Verify synthetic summary can be used as baseline for diffing.""" s3 = ceph_s3_client path_prefix = DEFAULT_LAKEHOUSE_KEY_PREFIX # Seed CEPH with one assembly - key1 = f"{path_prefix}refseq/GCF_000001215.4_old/GCF_000001215.4_old_genomic.fna.gz" - s3.put_object(Bucket=test_bucket, Key=key1, Body=b"data") + key1 = path_prefix / "refseq" / "GCF_000001215.4_old" / "GCF_000001215.4_old_genomic.fna.gz" + s3.put_object(Bucket=str(test_bucket), Key=str(key1), Body=b"data") # Build synthetic summary from store synthetic = scan_store_to_synthetic_summary(test_bucket, path_prefix, "2024/04/20") @@ -301,15 +297,15 @@ def test_synthetic_summary_diff_against_current( accession="GCF_000001215.4", status="latest", seq_rel_date=synthetic["GCF_000001215.4"].seq_rel_date, - ftp_path="", - assembly_dir="GCF_000001215.4_old", + ftp_url="", + assembly_dir=PurePosixPath("GCF_000001215.4_old"), ), "GCF_000005845.2": AssemblyRecord( accession="GCF_000005845.2", status="latest", seq_rel_date="2024/04/20", - ftp_path="", - assembly_dir="GCF_000005845.2_new", + ftp_url="", + assembly_dir=PurePosixPath("GCF_000005845.2_new"), ), } diff --git a/tests/integration/test_promote_e2e.py b/tests/integration/test_promote_e2e.py index 05d17f0a..26b21063 100644 --- a/tests/integration/test_promote_e2e.py +++ b/tests/integration/test_promote_e2e.py @@ -10,9 +10,11 @@ import hashlib import json -from pathlib import Path +from http import HTTPStatus +from pathlib import Path, PurePosixPath import pytest +from botocore.client import BaseClient from cdm_data_loaders.ncbi_ftp.assembly import build_accession_path from cdm_data_loaders.ncbi_ftp.metadata import ( @@ -20,20 +22,21 @@ build_descriptor_key, create_descriptor, ) -from cdm_data_loaders.ncbi_ftp.promote import DEFAULT_LAKEHOUSE_KEY_PREFIX, promote_from_s3 +from cdm_data_loaders.ncbi_ftp.promote import DEFAULT_LAKEHOUSE_KEY_PREFIX, _archive_assemblies, promote_from_s3 -from .conftest import get_object_metadata, list_all_keys, seed_lakehouse, staging_test_bucket # noqa: F401 +from .conftest import get_object_metadata, list_all_keys, seed_lakehouse # Fake assembly details used across tests ACCESSION_A = "GCF_900000001.1" -ASSEMBLY_DIR_A = "GCF_900000001.1_FakeAssemblyA" +ASSEMBLY_DIR_A: PurePosixPath = PurePosixPath("GCF_900000001.1_FakeAssemblyA") ACCESSION_B = "GCF_900000002.1" -ASSEMBLY_DIR_B = "GCF_900000002.1_FakeAssemblyB" +ASSEMBLY_DIR_B: PurePosixPath = PurePosixPath("GCF_900000002.1_FakeAssemblyB") ACCESSION_C = "GCF_900000003.1" -ASSEMBLY_DIR_C = "GCF_900000003.1_FakeAssemblyC" +ASSEMBLY_DIR_C: PurePosixPath = PurePosixPath("GCF_900000003.1_FakeAssemblyC") -STAGING_PREFIX = "staging/run1/" -PATH_PREFIX = DEFAULT_LAKEHOUSE_KEY_PREFIX +STAGING_PREFIX: PurePosixPath = PurePosixPath("staging") / "run1" +PATH_PREFIX: PurePosixPath = DEFAULT_LAKEHOUSE_KEY_PREFIX +ARCHIVE_PREFIX: PurePosixPath = PATH_PREFIX / "archive" / "2024-01" # Fake file contents for staging FAKE_GENOMIC = b">seq1\nATCGATCG\n" @@ -45,25 +48,25 @@ def _md5(data: bytes) -> str: def _stage_assembly( - s3: object, - bucket: str, - assembly_dir: str, + s3: BaseClient, + bucket: PurePosixPath, + assembly_dir: PurePosixPath, ) -> None: """Stage a fake assembly with data files and .md5 sidecars under the staging prefix.""" rel = build_accession_path(assembly_dir) - base = f"{STAGING_PREFIX}{rel}" + base = STAGING_PREFIX / rel files = { - f"{assembly_dir}_genomic.fna.gz": FAKE_GENOMIC, - f"{assembly_dir}_protein.faa.gz": FAKE_PROTEIN, + assembly_dir.with_name(f"{assembly_dir.name}_genomic.fna.gz"): FAKE_GENOMIC, + assembly_dir.with_name(f"{assembly_dir.name}_protein.faa.gz"): FAKE_PROTEIN, } for fname, content in files.items(): - key = f"{base}{fname}" - s3.put_object(Bucket=bucket, Key=key, Body=content) + key = base / fname + s3.put_object(Bucket=str(bucket), Key=str(key), Body=content) # Write .md5 sidecar - md5_key = f"{key}.md5" - s3.put_object(Bucket=bucket, Key=md5_key, Body=_md5(content).encode()) + md5_key = key.with_name(f"{key.name}.md5") + s3.put_object(Bucket=str(bucket), Key=str(md5_key), Body=_md5(content).encode()) def _write_manifest(tmp_path: Path, accessions: list[str], name: str) -> Path: @@ -81,7 +84,9 @@ def _write_manifest(tmp_path: Path, accessions: list[str], name: str) -> Path: class TestPromoteFromStaging: """Promote staged files to final Lakehouse paths.""" - def test_promote_from_staging(self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: + def test_promote_from_staging( + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath + ) -> None: """Staged files appear at the final Lakehouse path with MD5 metadata.""" s3 = ceph_s3_client _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) @@ -98,7 +103,7 @@ def test_promote_from_staging(self, ceph_s3_client: object, test_bucket: str, st assert report["dry_run"] is False # Verify files at final path - final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX / "raw_data") assert len(final_keys) >= 2 # noqa: PLR2004 # Verify MD5 metadata is set @@ -112,7 +117,9 @@ def test_promote_from_staging(self, ceph_s3_client: object, test_bucket: str, st class TestPromoteIdempotent: """Promoting the same staging data twice should succeed without errors.""" - def test_promote_idempotent(self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: + def test_promote_idempotent( + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath + ) -> None: """Second promote on empty staging succeeds and leaves the lakehouse unchanged. After the first promote, staged files are deleted. A second run therefore @@ -128,7 +135,7 @@ def test_promote_idempotent(self, ceph_s3_client: object, test_bucket: str, stag lakehouse_bucket=test_bucket, lakehouse_key_prefix=PATH_PREFIX, ) - keys_after_first = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + keys_after_first = list_all_keys(s3, test_bucket, PATH_PREFIX / "raw_data") report2 = promote_from_s3( staging_key_prefix=STAGING_PREFIX, @@ -136,7 +143,7 @@ def test_promote_idempotent(self, ceph_s3_client: object, test_bucket: str, stag lakehouse_bucket=test_bucket, lakehouse_key_prefix=PATH_PREFIX, ) - keys_after_second = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + keys_after_second = list_all_keys(s3, test_bucket, PATH_PREFIX / "raw_data") assert report1["failed"] == 0 assert report1["promoted"] >= 1 @@ -151,15 +158,15 @@ class TestPromoteArchiveUpdated: """Archive existing assemblies before overwriting with updated versions.""" def test_archive_updated( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath, tmp_path: Path ) -> None: """Updated assemblies are archived before being overwritten.""" s3 = ceph_s3_client # Seed "old" version at the final Lakehouse path - old_files = { - f"{ASSEMBLY_DIR_A}_genomic.fna.gz": "old genomic content", - f"{ASSEMBLY_DIR_A}_protein.faa.gz": "old protein content", + old_files: dict[PurePosixPath, str | bytes] = { + PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz"): "old genomic content", + PurePosixPath(f"{ASSEMBLY_DIR_A}_protein.faa.gz"): "old protein content", } seed_lakehouse(s3, test_bucket, ACCESSION_A, old_files, PATH_PREFIX, ASSEMBLY_DIR_A) @@ -172,7 +179,7 @@ def test_archive_updated( staging_key_prefix=STAGING_PREFIX, staging_bucket=staging_test_bucket, lakehouse_bucket=test_bucket, - updated_manifest_path=str(updated_manifest), + updated_manifest_path=updated_manifest, ncbi_release="2024-01", lakehouse_key_prefix=PATH_PREFIX, ) @@ -182,13 +189,13 @@ def test_archive_updated( assert report["failed"] == 0 # Verify archive exists - archive_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "archive/2024-01/") + archive_keys = list_all_keys(s3, test_bucket, ARCHIVE_PREFIX) assert len(archive_keys) >= 2 # noqa: PLR2004 # Verify archive metadata for key in archive_keys: - assert "/updated/" in key - assert "/2024-01/" in key + assert "updated" in key.parts + assert "2024-01" in key.parts @pytest.mark.requires_ceph @@ -197,14 +204,14 @@ class TestPromoteArchiveRemoved: """Archive and delete replaced/suppressed assemblies.""" def test_archive_removed( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath, tmp_path: Path ) -> None: """Removed assemblies are archived and source objects are deleted.""" s3 = ceph_s3_client # Seed assemblies at final path - files = { - f"{ASSEMBLY_DIR_A}_genomic.fna.gz": "content to archive", + files: dict[PurePosixPath, str | bytes] = { + PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz"): "content to archive", } seed_lakehouse(s3, test_bucket, ACCESSION_A, files, PATH_PREFIX, ASSEMBLY_DIR_A) @@ -215,7 +222,7 @@ def test_archive_removed( staging_key_prefix=STAGING_PREFIX, staging_bucket=staging_test_bucket, lakehouse_bucket=test_bucket, - removed_manifest_path=str(removed_manifest), + removed_manifest_path=removed_manifest, ncbi_release="2024-01", lakehouse_key_prefix=PATH_PREFIX, ) @@ -224,16 +231,16 @@ def test_archive_removed( assert report["failed"] == 0 # Verify archive exists - archive_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "archive/2024-01/") + archive_keys = list_all_keys(s3, test_bucket, ARCHIVE_PREFIX) assert len(archive_keys) >= 1 # Verify archive metadata for key in archive_keys: - assert "/replaced_or_suppressed/" in key + assert "replaced_or_suppressed" in key.parts # Verify source objects are deleted rel = build_accession_path(ASSEMBLY_DIR_A) - source_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + rel) + source_keys = list_all_keys(s3, test_bucket, PATH_PREFIX / rel) assert len(source_keys) == 0, f"Expected source objects deleted, found: {source_keys}" @@ -242,7 +249,9 @@ def test_archive_removed( class TestPromoteDryRun: """Dry-run mode should not create any objects.""" - def test_promote_dry_run(self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: + def test_promote_dry_run( + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath + ) -> None: """Dry-run logs actions but creates no objects at the final path.""" s3 = ceph_s3_client _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) @@ -259,7 +268,7 @@ def test_promote_dry_run(self, ceph_s3_client: object, test_bucket: str, staging assert report["promoted"] >= 1 # No objects should exist at the final path - final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX / "raw_data") assert len(final_keys) == 0, f"Dry-run should not create objects, found: {final_keys}" @@ -269,19 +278,19 @@ class TestPromoteTrimsManifest: """Manifest trimming removes promoted accessions.""" def test_trims_manifest( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath, tmp_path: Path ) -> None: """Transfer manifest in CEPH is trimmed to exclude promoted accessions.""" s3 = ceph_s3_client # Upload a transfer manifest with 3 entries to CEPH (manifest lives in staging) - manifest_key = "ncbi/transfer_manifest.txt" + manifest_key = PurePosixPath("ncbi") / "transfer_manifest.txt" manifest_lines = [ "/genomes/all/GCF/900/000/001/GCF_900000001.1_FakeAssemblyA/\n", "/genomes/all/GCF/900/000/002/GCF_900000002.1_FakeAssemblyB/\n", "/genomes/all/GCF/900/000/003/GCF_900000003.1_FakeAssemblyC/\n", ] - s3.put_object(Bucket=staging_test_bucket, Key=manifest_key, Body="".join(manifest_lines).encode()) + s3.put_object(Bucket=str(staging_test_bucket), Key=str(manifest_key), Body="".join(manifest_lines).encode()) # Stage only assemblies A and B (not C) _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) @@ -298,7 +307,7 @@ def test_trims_manifest( assert report["failed"] == 0 # Read back the manifest from CEPH (it lives in staging) - resp = s3.get_object(Bucket=staging_test_bucket, Key=manifest_key) + resp = s3.get_object(Bucket=str(staging_test_bucket), Key=str(manifest_key)) remaining = resp["Body"].read().decode() remaining_lines = [line.strip() for line in remaining.strip().splitlines() if line.strip()] @@ -312,16 +321,18 @@ def test_trims_manifest( class TestPromoteIncompleteStaging: """Incomplete staging (sidecar only, no data) should not promote anything.""" - def test_incomplete_staging(self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: + def test_incomplete_staging( + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath + ) -> None: """Only .md5 sidecars staged → nothing promoted.""" s3 = ceph_s3_client # Stage only .md5 sidecars (no data files) rel = build_accession_path(ASSEMBLY_DIR_A) - base = f"{STAGING_PREFIX}{rel}" - fname = f"{ASSEMBLY_DIR_A}_genomic.fna.gz" - md5_key = f"{base}{fname}.md5" - s3.put_object(Bucket=staging_test_bucket, Key=md5_key, Body=_md5(FAKE_GENOMIC).encode()) + base = STAGING_PREFIX / rel + fname = PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz") + md5_key = base / fname.with_name(f"{fname.name}.md5") + s3.put_object(Bucket=str(staging_test_bucket), Key=str(md5_key), Body=_md5(FAKE_GENOMIC).encode()) report = promote_from_s3( staging_key_prefix=STAGING_PREFIX, @@ -335,7 +346,7 @@ def test_incomplete_staging(self, ceph_s3_client: object, test_bucket: str, stag assert report["failed"] == 0 # No objects at final path - final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX / "raw_data") assert len(final_keys) == 0 @@ -344,7 +355,9 @@ def test_incomplete_staging(self, ceph_s3_client: object, test_bucket: str, stag class TestPromoteCreatesDescriptor: """Promote step writes a frictionless descriptor for each promoted assembly.""" - def test_descriptor_created(self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: + def test_descriptor_created( + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath + ) -> None: """After promote, a JSON descriptor exists under ``metadata/``.""" s3 = ceph_s3_client _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) @@ -357,14 +370,14 @@ def test_descriptor_created(self, ceph_s3_client: object, test_bucket: str, stag ) descriptor_key = build_descriptor_key(ASSEMBLY_DIR_A, PATH_PREFIX) - obj = s3.get_object(Bucket=test_bucket, Key=descriptor_key) + obj = s3.get_object(Bucket=str(test_bucket), Key=str(descriptor_key)) body = json.loads(obj["Body"].read()) assert body["identifier"] == f"NCBI:{ACCESSION_A}" assert body["resource_type"] == "dataset" def test_descriptor_resources_include_promoted_files( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath ) -> None: """Descriptor's ``resources`` list references the final Lakehouse key.""" s3 = ceph_s3_client @@ -378,14 +391,14 @@ def test_descriptor_resources_include_promoted_files( ) descriptor_key = build_descriptor_key(ASSEMBLY_DIR_A, PATH_PREFIX) - obj = s3.get_object(Bucket=test_bucket, Key=descriptor_key) + obj = s3.get_object(Bucket=str(test_bucket), Key=str(descriptor_key)) body = json.loads(obj["Body"].read()) resource_paths = [r["path"] for r in body["resources"]] - assert any(PATH_PREFIX + "raw_data/" in p for p in resource_paths) + assert any(f"{PATH_PREFIX / 'raw_data'}/" in p for p in resource_paths) def test_descriptor_resources_have_md5( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath ) -> None: """Resources with .md5 sidecars include the hash value.""" s3 = ceph_s3_client @@ -399,7 +412,7 @@ def test_descriptor_resources_have_md5( ) descriptor_key = build_descriptor_key(ASSEMBLY_DIR_A, PATH_PREFIX) - obj = s3.get_object(Bucket=test_bucket, Key=descriptor_key) + obj = s3.get_object(Bucket=str(test_bucket), Key=str(descriptor_key)) body = json.loads(obj["Body"].read()) # Both staged files have .md5 sidecars @@ -407,7 +420,7 @@ def test_descriptor_resources_have_md5( assert "hash" in resource, f"Expected hash in resource: {resource}" def test_multiple_assemblies_get_separate_descriptors( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath ) -> None: """Each assembly gets its own descriptor file.""" s3 = ceph_s3_client @@ -423,7 +436,7 @@ def test_multiple_assemblies_get_separate_descriptors( for assembly_dir, accession in [(ASSEMBLY_DIR_A, ACCESSION_A), (ASSEMBLY_DIR_B, ACCESSION_B)]: key = build_descriptor_key(assembly_dir, PATH_PREFIX) - obj = s3.get_object(Bucket=test_bucket, Key=key) + obj = s3.get_object(Bucket=str(test_bucket), Key=str(key)) body = json.loads(obj["Body"].read()) assert body["identifier"] == f"NCBI:{accession}" @@ -434,19 +447,19 @@ class TestPromoteArchiveUpdatedIncludesDescriptor: """Archiving updated assemblies also archives the descriptor.""" def test_archive_copies_descriptor( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath, tmp_path: Path ) -> None: """After archiving an updated assembly, the descriptor appears under archive/.""" s3 = ceph_s3_client # Seed old version at Lakehouse path *including* a live descriptor - old_files = {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": "old content"} + old_files: dict[PurePosixPath, str | bytes] = {PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz"): "old content"} seed_lakehouse(s3, test_bucket, ACCESSION_A, old_files, PATH_PREFIX, ASSEMBLY_DIR_A) # Pre-upload a descriptor so archive_descriptor can find it descriptor = create_descriptor(ASSEMBLY_DIR_A, ACCESSION_A, []) # Upload directly to CEPH (not via promote) descriptor_key = build_descriptor_key(ASSEMBLY_DIR_A, PATH_PREFIX) - s3.put_object(Bucket=test_bucket, Key=descriptor_key, Body=json.dumps(descriptor).encode()) + s3.put_object(Bucket=str(test_bucket), Key=str(descriptor_key), Body=json.dumps(descriptor).encode()) _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) updated_manifest = _write_manifest(tmp_path, [ACCESSION_A], "updated_manifest.txt") @@ -455,15 +468,15 @@ def test_archive_copies_descriptor( staging_key_prefix=STAGING_PREFIX, staging_bucket=staging_test_bucket, lakehouse_bucket=test_bucket, - updated_manifest_path=str(updated_manifest), + updated_manifest_path=updated_manifest, ncbi_release="2024-01", lakehouse_key_prefix=PATH_PREFIX, ) archive_key = build_archive_descriptor_key(ASSEMBLY_DIR_A, "2024-01", PATH_PREFIX, "updated") # Confirm the archive descriptor object exists - resp = s3.head_object(Bucket=test_bucket, Key=archive_key) - assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + resp = s3.head_object(Bucket=str(test_bucket), Key=str(archive_key)) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == HTTPStatus.OK @pytest.mark.requires_ceph @@ -472,18 +485,18 @@ class TestPromoteArchiveRemovedIncludesDescriptor: """Archiving removed assemblies also archives the descriptor.""" def test_archive_removed_copies_descriptor( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath, tmp_path: Path ) -> None: """After archiving a removed assembly, the descriptor is under archive/.""" s3 = ceph_s3_client # Seed the assembly at final Lakehouse path - files = {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": "content"} + files: dict[PurePosixPath, str | bytes] = {PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz"): "content"} seed_lakehouse(s3, test_bucket, ACCESSION_A, files, PATH_PREFIX, ASSEMBLY_DIR_A) # Pre-upload a descriptor descriptor = create_descriptor(ASSEMBLY_DIR_A, ACCESSION_A, []) descriptor_key = build_descriptor_key(ASSEMBLY_DIR_A, PATH_PREFIX) - s3.put_object(Bucket=test_bucket, Key=descriptor_key, Body=json.dumps(descriptor).encode()) + s3.put_object(Bucket=str(test_bucket), Key=str(descriptor_key), Body=json.dumps(descriptor).encode()) removed_manifest = _write_manifest(tmp_path, [ACCESSION_A], "removed_manifest.txt") @@ -491,14 +504,14 @@ def test_archive_removed_copies_descriptor( staging_key_prefix=STAGING_PREFIX, staging_bucket=staging_test_bucket, lakehouse_bucket=test_bucket, - removed_manifest_path=str(removed_manifest), + removed_manifest_path=removed_manifest, ncbi_release="2024-01", lakehouse_key_prefix=PATH_PREFIX, ) archive_key = build_archive_descriptor_key(ASSEMBLY_DIR_A, "2024-01", PATH_PREFIX, "replaced_or_suppressed") - resp = s3.head_object(Bucket=test_bucket, Key=archive_key) - assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + resp = s3.head_object(Bucket=str(test_bucket), Key=str(archive_key)) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == HTTPStatus.OK @pytest.mark.requires_ceph @@ -506,7 +519,9 @@ def test_archive_removed_copies_descriptor( class TestPromoteDryRunNoDescriptor: """Dry-run must not write any descriptor files.""" - def test_dry_run_no_descriptor(self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: + def test_dry_run_no_descriptor( + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath + ) -> None: """Dry-run does not upload a descriptor to the metadata/ prefix.""" s3 = ceph_s3_client _stage_assembly(s3, staging_test_bucket, ASSEMBLY_DIR_A) @@ -519,7 +534,7 @@ def test_dry_run_no_descriptor(self, ceph_s3_client: object, test_bucket: str, s dry_run=True, ) - metadata_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "metadata/") + metadata_keys = list_all_keys(s3, test_bucket, PATH_PREFIX / "metadata") assert len(metadata_keys) == 0, f"Dry-run should not create descriptor files, found: {metadata_keys}" @@ -532,28 +547,26 @@ class TestArchiveMultiFileConcurrent: """Verify parallel copy archives all files correctly with correct content.""" def test_all_files_archived_with_correct_content( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath, tmp_path: Path ) -> None: """Every file is archived with byte-identical content when copied concurrently.""" - from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies - s3 = ceph_s3_client # Seed many files for assembly A at final Lakehouse path - many_files = { - f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"GENOMIC_CONTENT", - f"{ASSEMBLY_DIR_A}_protein.faa.gz": b"PROTEIN_CONTENT", - f"{ASSEMBLY_DIR_A}_rna.fna.gz": b"RNA_CONTENT", - f"{ASSEMBLY_DIR_A}_assembly_report.txt": b"ASSEMBLY_REPORT", - f"{ASSEMBLY_DIR_A}_assembly_stats.txt": b"ASSEMBLY_STATS", - f"{ASSEMBLY_DIR_A}_cds_from_genomic.fna.gz": b"CDS_CONTENT", + many_files: dict[PurePosixPath, str | bytes] = { + PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz"): b"GENOMIC_CONTENT", + PurePosixPath(f"{ASSEMBLY_DIR_A}_protein.faa.gz"): b"PROTEIN_CONTENT", + PurePosixPath(f"{ASSEMBLY_DIR_A}_rna.fna.gz"): b"RNA_CONTENT", + PurePosixPath(f"{ASSEMBLY_DIR_A}_assembly_report.txt"): b"ASSEMBLY_REPORT", + PurePosixPath(f"{ASSEMBLY_DIR_A}_assembly_stats.txt"): b"ASSEMBLY_STATS", + PurePosixPath(f"{ASSEMBLY_DIR_A}_cds_from_genomic.fna.gz"): b"CDS_CONTENT", } seed_lakehouse(s3, test_bucket, ACCESSION_A, many_files, PATH_PREFIX, ASSEMBLY_DIR_A) updated_manifest = _write_manifest(tmp_path, [ACCESSION_A], "updated_manifest.txt") archived = _archive_assemblies( - str(updated_manifest), + updated_manifest, lakehouse_bucket=test_bucket, ncbi_release="2024-01", archive_reason="updated", @@ -566,24 +579,22 @@ def test_all_files_archived_with_correct_content( # Verify every archived file has correct content rel = build_accession_path(ASSEMBLY_DIR_A) for fname, expected_body in many_files.items(): - archive_key = f"{PATH_PREFIX}archive/2024-01/updated/{rel}{fname}" - obj = s3.get_object(Bucket=test_bucket, Key=archive_key) + archive_key = ARCHIVE_PREFIX / "updated" / rel / fname + obj = s3.get_object(Bucket=str(test_bucket), Key=str(archive_key)) actual_body = obj["Body"].read() assert actual_body == expected_body, f"Content mismatch for {fname}" def test_archive_key_paths_are_correct( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath, tmp_path: Path ) -> None: """Archived keys follow the exact ``archive/{release}/{reason}/{rel_path}`` pattern.""" - from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies - s3 = ceph_s3_client - files = {f"{ASSEMBLY_DIR_B}_genomic.fna.gz": b"content"} + files: dict[PurePosixPath, str | bytes] = {PurePosixPath(f"{ASSEMBLY_DIR_B}_genomic.fna.gz"): b"content"} seed_lakehouse(s3, test_bucket, ACCESSION_B, files, PATH_PREFIX, ASSEMBLY_DIR_B) removed_manifest = _write_manifest(tmp_path, [ACCESSION_B], "removed_manifest.txt") _archive_assemblies( - str(removed_manifest), + removed_manifest, lakehouse_bucket=test_bucket, ncbi_release="2024-02", archive_reason="replaced_or_suppressed", @@ -592,9 +603,11 @@ def test_archive_key_paths_are_correct( ) rel = build_accession_path(ASSEMBLY_DIR_B) - expected_key = f"{PATH_PREFIX}archive/2024-02/replaced_or_suppressed/{rel}{ASSEMBLY_DIR_B}_genomic.fna.gz" - resp = s3.head_object(Bucket=test_bucket, Key=expected_key) - assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + expected_key = ( + PATH_PREFIX / "archive" / "2024-02" / "replaced_or_suppressed" / rel / f"{ASSEMBLY_DIR_B}_genomic.fna.gz" + ) + resp = s3.head_object(Bucket=str(test_bucket), Key=str(expected_key)) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == HTTPStatus.OK @pytest.mark.requires_ceph @@ -603,23 +616,21 @@ class TestArchiveDeleteSourceBatch: """Verify batch delete removes all source objects after concurrent copy.""" def test_all_sources_deleted_after_archive( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath, tmp_path: Path ) -> None: """After archive with delete_source=True, no source objects remain.""" - from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies - s3 = ceph_s3_client - many_files = { - f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"genomic", - f"{ASSEMBLY_DIR_A}_protein.faa.gz": b"protein", - f"{ASSEMBLY_DIR_A}_rna.fna.gz": b"rna", - f"{ASSEMBLY_DIR_A}_assembly_report.txt": b"report", + many_files: dict[PurePosixPath, str | bytes] = { + PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz"): b"genomic", + PurePosixPath(f"{ASSEMBLY_DIR_A}_protein.faa.gz"): b"protein", + PurePosixPath(f"{ASSEMBLY_DIR_A}_rna.fna.gz"): b"rna", + PurePosixPath(f"{ASSEMBLY_DIR_A}_assembly_report.txt"): b"report", } source_keys = seed_lakehouse(s3, test_bucket, ACCESSION_A, many_files, PATH_PREFIX, ASSEMBLY_DIR_A) removed_manifest = _write_manifest(tmp_path, [ACCESSION_A], "removed_manifest.txt") archived = _archive_assemblies( - str(removed_manifest), + removed_manifest, lakehouse_bucket=test_bucket, ncbi_release="2024-01", archive_reason="replaced_or_suppressed", @@ -634,21 +645,19 @@ def test_all_sources_deleted_after_archive( assert len(remaining) == 0, f"Source not deleted: {key}" def test_archive_present_source_gone( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath, tmp_path: Path ) -> None: """Archive destinations exist AND sources are gone after replaced_or_suppressed archive.""" - from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies - s3 = ceph_s3_client - files = { - f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"genomic", - f"{ASSEMBLY_DIR_A}_protein.faa.gz": b"protein", + files: dict[PurePosixPath, str | bytes] = { + PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz"): b"genomic", + PurePosixPath(f"{ASSEMBLY_DIR_A}_protein.faa.gz"): b"protein", } seed_lakehouse(s3, test_bucket, ACCESSION_A, files, PATH_PREFIX, ASSEMBLY_DIR_A) removed_manifest = _write_manifest(tmp_path, [ACCESSION_A], "removed_manifest.txt") _archive_assemblies( - str(removed_manifest), + removed_manifest, lakehouse_bucket=test_bucket, ncbi_release="2024-01", archive_reason="replaced_or_suppressed", @@ -658,10 +667,10 @@ def test_archive_present_source_gone( rel = build_accession_path(ASSEMBLY_DIR_A) # Archive keys present - archive_keys = list_all_keys(s3, test_bucket, f"{PATH_PREFIX}archive/2024-01/replaced_or_suppressed/") + archive_keys = list_all_keys(s3, test_bucket, ARCHIVE_PREFIX / "replaced_or_suppressed") assert len(archive_keys) == len(files), f"Expected {len(files)} archive keys, got: {archive_keys}" # Source keys absent - source_keys = list_all_keys(s3, test_bucket, f"{PATH_PREFIX}{rel}") + source_keys = list_all_keys(s3, test_bucket, PATH_PREFIX / rel) assert len(source_keys) == 0, f"Source objects remain: {source_keys}" @@ -676,7 +685,7 @@ class TestPartialArchiveResume: """ def test_partial_updated_archive_resumes( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath, tmp_path: Path ) -> None: """Re-running after a partial updated archive overwrites stale copies and archives missing files. @@ -684,25 +693,27 @@ def test_partial_updated_archive_resumes( file_b and file_c were not. Re-run should overwrite file_a with current content and archive file_b, file_c. """ - from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies - s3 = ceph_s3_client rel = build_accession_path(ASSEMBLY_DIR_A) - file_a = f"{ASSEMBLY_DIR_A}_genomic.fna.gz" - file_b = f"{ASSEMBLY_DIR_A}_protein.faa.gz" - file_c = f"{ASSEMBLY_DIR_A}_rna.fna.gz" + file_a = PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz") + file_b = PurePosixPath(f"{ASSEMBLY_DIR_A}_protein.faa.gz") + file_c = PurePosixPath(f"{ASSEMBLY_DIR_A}_rna.fna.gz") - current_content = {file_a: b"current-genomic", file_b: b"current-protein", file_c: b"current-rna"} + current_content: dict[PurePosixPath, str | bytes] = { + file_a: b"current-genomic", + file_b: b"current-protein", + file_c: b"current-rna", + } seed_lakehouse(s3, test_bucket, ACCESSION_A, current_content, PATH_PREFIX, ASSEMBLY_DIR_A) # Pre-seed a stale archive copy for file_a (simulating prior partial run) - archive_prefix = f"{PATH_PREFIX}archive/2024-01/updated/{rel}" - s3.put_object(Bucket=test_bucket, Key=f"{archive_prefix}{file_a}", Body=b"stale-genomic") + archive_prefix = ARCHIVE_PREFIX / "updated" / rel + s3.put_object(Bucket=str(test_bucket), Key=str(archive_prefix / file_a), Body=b"stale-genomic") updated_manifest = _write_manifest(tmp_path, [ACCESSION_A], "updated_manifest.txt") archived = _archive_assemblies( - str(updated_manifest), + updated_manifest, lakehouse_bucket=test_bucket, ncbi_release="2024-01", archive_reason="updated", @@ -713,18 +724,18 @@ def test_partial_updated_archive_resumes( # All 3 files counted assert archived == 3 # noqa: PLR2004 # file_a overwritten with current content - obj_a = s3.get_object(Bucket=test_bucket, Key=f"{archive_prefix}{file_a}") + obj_a = s3.get_object(Bucket=str(test_bucket), Key=str(archive_prefix / file_a)) assert obj_a["Body"].read() == b"current-genomic", "file_a archive should be overwritten" # file_b and file_c now archived for fname in (file_b, file_c): - resp = s3.head_object(Bucket=test_bucket, Key=f"{archive_prefix}{fname}") - assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + resp = s3.head_object(Bucket=str(test_bucket), Key=str(archive_prefix / fname)) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == HTTPStatus.OK # Sources untouched (delete_source=False) - source_keys = list_all_keys(s3, test_bucket, f"{PATH_PREFIX}{rel}") + source_keys = list_all_keys(s3, test_bucket, PATH_PREFIX / rel) assert len(source_keys) == len(current_content) def test_partial_replaced_archive_resumes_and_deletes( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath, tmp_path: Path ) -> None: """Re-running replaced_or_suppressed archive after partial run completes and deletes all sources. @@ -732,35 +743,33 @@ def test_partial_replaced_archive_resumes_and_deletes( file_b was copied but NOT deleted (still at source), file_c was untouched. Re-run processes file_b and file_c, deletes both. Result: no sources remain. """ - from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies - s3 = ceph_s3_client rel = build_accession_path(ASSEMBLY_DIR_A) - file_a = f"{ASSEMBLY_DIR_A}_genomic.fna.gz" - file_b = f"{ASSEMBLY_DIR_A}_protein.faa.gz" - file_c = f"{ASSEMBLY_DIR_A}_rna.fna.gz" - archive_prefix = f"{PATH_PREFIX}archive/2024-01/replaced_or_suppressed/{rel}" + file_a = PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz") + file_b = PurePosixPath(f"{ASSEMBLY_DIR_A}_protein.faa.gz") + file_c = PurePosixPath(f"{ASSEMBLY_DIR_A}_rna.fna.gz") + archive_prefix = ARCHIVE_PREFIX / "replaced_or_suppressed" / rel # Only file_b and file_c remain at source (file_a already gone) s3.put_object( - Bucket=test_bucket, - Key=f"{PATH_PREFIX}{rel}{file_b}", + Bucket=str(test_bucket), + Key=str(PATH_PREFIX / rel / file_b), Body=b"protein", Metadata={"md5": hashlib.md5(b"protein").hexdigest()}, # noqa: S324 ) s3.put_object( - Bucket=test_bucket, - Key=f"{PATH_PREFIX}{rel}{file_c}", + Bucket=str(test_bucket), + Key=str(PATH_PREFIX / rel / file_c), Body=b"rna", Metadata={"md5": hashlib.md5(b"rna").hexdigest()}, # noqa: S324 ) # file_a already at archive destination - s3.put_object(Bucket=test_bucket, Key=f"{archive_prefix}{file_a}", Body=b"genomic") + s3.put_object(Bucket=str(test_bucket), Key=str(archive_prefix / file_a), Body=b"genomic") removed_manifest = _write_manifest(tmp_path, [ACCESSION_A], "removed_manifest.txt") archived = _archive_assemblies( - str(removed_manifest), + removed_manifest, lakehouse_bucket=test_bucket, ncbi_release="2024-01", archive_reason="replaced_or_suppressed", @@ -772,25 +781,23 @@ def test_partial_replaced_archive_resumes_and_deletes( assert archived == 2 # noqa: PLR2004 # file_b and file_c archive keys exist for fname in (file_b, file_c): - resp = s3.head_object(Bucket=test_bucket, Key=f"{archive_prefix}{fname}") - assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + resp = s3.head_object(Bucket=str(test_bucket), Key=str(archive_prefix / fname)) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == HTTPStatus.OK # No source keys remain (file_b and file_c were deleted) - source_keys = list_all_keys(s3, test_bucket, f"{PATH_PREFIX}{rel}") + source_keys = list_all_keys(s3, test_bucket, PATH_PREFIX / rel) assert len(source_keys) == 0, f"Source objects remain: {source_keys}" # file_a archive key is still intact - resp_a = s3.head_object(Bucket=test_bucket, Key=f"{archive_prefix}{file_a}") - assert resp_a["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + resp_a = s3.head_object(Bucket=str(test_bucket), Key=str(archive_prefix / file_a)) + assert resp_a["ResponseMetadata"]["HTTPStatusCode"] == HTTPStatus.OK def test_full_rerun_after_complete_archive_is_idempotent( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath, tmp_path: Path ) -> None: """Running archive again when all files already exist at archive paths is safe (no errors).""" - from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies - s3 = ceph_s3_client - files = { - f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"genomic", - f"{ASSEMBLY_DIR_A}_protein.faa.gz": b"protein", + files: dict[PurePosixPath, str | bytes] = { + PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz"): b"genomic", + PurePosixPath(f"{ASSEMBLY_DIR_A}_protein.faa.gz"): b"protein", } seed_lakehouse(s3, test_bucket, ACCESSION_A, files, PATH_PREFIX, ASSEMBLY_DIR_A) @@ -798,7 +805,7 @@ def test_full_rerun_after_complete_archive_is_idempotent( # First run — archives all files archived_1 = _archive_assemblies( - str(updated_manifest), + updated_manifest, lakehouse_bucket=test_bucket, ncbi_release="2024-01", archive_reason="updated", @@ -808,7 +815,7 @@ def test_full_rerun_after_complete_archive_is_idempotent( # Second run — same manifest, same source files still present archived_2 = _archive_assemblies( - str(updated_manifest), + updated_manifest, lakehouse_bucket=test_bucket, ncbi_release="2024-01", archive_reason="updated", @@ -821,8 +828,8 @@ def test_full_rerun_after_complete_archive_is_idempotent( # Archive keys still present with correct content rel = build_accession_path(ASSEMBLY_DIR_A) for fname, expected_body in files.items(): - key = f"{PATH_PREFIX}archive/2024-01/updated/{rel}{fname}" - obj = s3.get_object(Bucket=test_bucket, Key=key) + key = ARCHIVE_PREFIX / "updated" / rel / fname + obj = s3.get_object(Bucket=str(test_bucket), Key=str(key)) assert obj["Body"].read() == expected_body @@ -832,22 +839,20 @@ class TestArchiveMultiAccessionManifest: """Multiple accessions in a single manifest are all archived.""" def test_two_accessions_both_archived( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath, tmp_path: Path ) -> None: """Both accessions are archived with correct keys when listed in one manifest.""" - from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies - s3 = ceph_s3_client - files_a = {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"genomic-A"} - files_b = {f"{ASSEMBLY_DIR_B}_genomic.fna.gz": b"genomic-B"} + files_a: dict[PurePosixPath, str | bytes] = {PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz"): b"genomic-A"} + files_b: dict[PurePosixPath, str | bytes] = {PurePosixPath(f"{ASSEMBLY_DIR_B}_genomic.fna.gz"): b"genomic-B"} seed_lakehouse(s3, test_bucket, ACCESSION_A, files_a, PATH_PREFIX, ASSEMBLY_DIR_A) seed_lakehouse(s3, test_bucket, ACCESSION_B, files_b, PATH_PREFIX, ASSEMBLY_DIR_B) manifest = _write_manifest(tmp_path, [ACCESSION_A, ACCESSION_B], "removed_manifest.txt") archived = _archive_assemblies( - str(manifest), + manifest, lakehouse_bucket=test_bucket, ncbi_release="2024-01", archive_reason="replaced_or_suppressed", @@ -858,20 +863,24 @@ def test_two_accessions_both_archived( assert archived == 2 # noqa: PLR2004 rel_a = build_accession_path(ASSEMBLY_DIR_A) rel_b = build_accession_path(ASSEMBLY_DIR_B) - key_a = f"{PATH_PREFIX}archive/2024-01/replaced_or_suppressed/{rel_a}{ASSEMBLY_DIR_A}_genomic.fna.gz" - key_b = f"{PATH_PREFIX}archive/2024-01/replaced_or_suppressed/{rel_b}{ASSEMBLY_DIR_B}_genomic.fna.gz" - assert s3.head_object(Bucket=test_bucket, Key=key_a)["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 - assert s3.head_object(Bucket=test_bucket, Key=key_b)["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + key_a = ARCHIVE_PREFIX / "replaced_or_suppressed" / rel_a / f"{ASSEMBLY_DIR_A}_genomic.fna.gz" + key_b = ARCHIVE_PREFIX / "replaced_or_suppressed" / rel_b / f"{ASSEMBLY_DIR_B}_genomic.fna.gz" + assert ( + s3.head_object(Bucket=str(test_bucket), Key=str(key_a))["ResponseMetadata"]["HTTPStatusCode"] + == HTTPStatus.OK + ) + assert ( + s3.head_object(Bucket=str(test_bucket), Key=str(key_b))["ResponseMetadata"]["HTTPStatusCode"] + == HTTPStatus.OK + ) # Sources deleted - assert len(list_all_keys(s3, test_bucket, f"{PATH_PREFIX}{rel_a}")) == 0 - assert len(list_all_keys(s3, test_bucket, f"{PATH_PREFIX}{rel_b}")) == 0 + assert len(list_all_keys(s3, test_bucket, PATH_PREFIX / rel_a)) == 0 + assert len(list_all_keys(s3, test_bucket, PATH_PREFIX / rel_b)) == 0 def test_three_accessions_correct_archive_reason_segment( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath, tmp_path: Path ) -> None: """Archive keys for all three accessions include the archive_reason segment.""" - from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies - s3 = ceph_s3_client accessions_and_dirs = [ (ACCESSION_A, ASSEMBLY_DIR_A), @@ -883,14 +892,14 @@ def test_three_accessions_correct_archive_reason_segment( s3, test_bucket, accession, - {f"{assembly_dir}_genomic.fna.gz": b"data"}, + {PurePosixPath(f"{assembly_dir}_genomic.fna.gz"): b"data"}, PATH_PREFIX, assembly_dir, ) manifest = _write_manifest(tmp_path, [acc for acc, _ in accessions_and_dirs], "removed_manifest.txt") _archive_assemblies( - str(manifest), + manifest, lakehouse_bucket=test_bucket, ncbi_release="2024-03", archive_reason="replaced_or_suppressed", @@ -898,11 +907,11 @@ def test_three_accessions_correct_archive_reason_segment( delete_source=False, ) - all_archive_keys = list_all_keys(s3, test_bucket, f"{PATH_PREFIX}archive/2024-03/") + all_archive_keys = list_all_keys(s3, test_bucket, PATH_PREFIX / "archive" / "2024-03") assert len(all_archive_keys) == 3 # noqa: PLR2004 for key in all_archive_keys: - assert "/replaced_or_suppressed/" in key, f"Archive key missing reason segment: {key}" - assert "/2024-03/" in key, f"Archive key missing release segment: {key}" + assert "replaced_or_suppressed" in key.parts, f"Archive key missing reason segment: {key}" + assert "2024-03" in key.parts, f"Archive key missing release segment: {key}" @pytest.mark.requires_ceph @@ -911,22 +920,20 @@ class TestArchiveDryRunParallel: """Dry-run with many files leaves everything unchanged.""" def test_dry_run_no_copies_no_deletes( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str, tmp_path: Path + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath, tmp_path: Path ) -> None: """Dry-run with multiple files per accession creates no archive keys and keeps sources.""" - from cdm_data_loaders.ncbi_ftp.promote import _archive_assemblies - s3 = ceph_s3_client - many_files = { - f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"genomic", - f"{ASSEMBLY_DIR_A}_protein.faa.gz": b"protein", - f"{ASSEMBLY_DIR_A}_rna.fna.gz": b"rna", + many_files: dict[PurePosixPath, str | bytes] = { + PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz"): b"genomic", + PurePosixPath(f"{ASSEMBLY_DIR_A}_protein.faa.gz"): b"protein", + PurePosixPath(f"{ASSEMBLY_DIR_A}_rna.fna.gz"): b"rna", } source_keys = seed_lakehouse(s3, test_bucket, ACCESSION_A, many_files, PATH_PREFIX, ASSEMBLY_DIR_A) removed_manifest = _write_manifest(tmp_path, [ACCESSION_A], "removed_manifest.txt") archived = _archive_assemblies( - str(removed_manifest), + removed_manifest, lakehouse_bucket=test_bucket, ncbi_release="2024-01", archive_reason="replaced_or_suppressed", @@ -937,7 +944,7 @@ def test_dry_run_no_copies_no_deletes( assert archived == len(many_files) # No archive keys - archive_keys = list_all_keys(s3, test_bucket, f"{PATH_PREFIX}archive/") + archive_keys = list_all_keys(s3, test_bucket, PATH_PREFIX / "archive") assert len(archive_keys) == 0, f"Dry-run created archive keys: {archive_keys}" # All sources still present for key in source_keys: @@ -949,21 +956,21 @@ def test_dry_run_no_copies_no_deletes( def _stage_many( - s3: object, - bucket: str, - assembly_dir: str, - files: dict[str, bytes], + s3: BaseClient, + bucket: PurePosixPath, + assembly_dir: PurePosixPath, + files: dict[PurePosixPath, bytes], *, with_md5: bool = True, ) -> None: """Stage *files* with optional .md5 sidecars under the standard staging prefix.""" rel = build_accession_path(assembly_dir) - base = f"{STAGING_PREFIX}{rel}" + base = STAGING_PREFIX / rel for fname, content in files.items(): - key = f"{base}{fname}" - s3.put_object(Bucket=bucket, Key=key, Body=content) + key = base / fname + s3.put_object(Bucket=str(bucket), Key=str(key), Body=content) if with_md5: - s3.put_object(Bucket=bucket, Key=f"{key}.md5", Body=_md5(content).encode()) + s3.put_object(Bucket=str(bucket), Key=f"{key}.md5", Body=_md5(content).encode()) @pytest.mark.requires_ceph @@ -972,17 +979,17 @@ class TestPromoteMultiFileConcurrent: """Verify concurrent promotion lands all files with correct content and MD5.""" def test_six_files_all_promoted_with_correct_content( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath ) -> None: """Every staged file arrives at the correct final key with byte-identical content.""" s3 = ceph_s3_client - many_files = { - f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"GENOMIC", - f"{ASSEMBLY_DIR_A}_protein.faa.gz": b"PROTEIN", - f"{ASSEMBLY_DIR_A}_rna.fna.gz": b"RNA", - f"{ASSEMBLY_DIR_A}_assembly_report.txt": b"REPORT", - f"{ASSEMBLY_DIR_A}_assembly_stats.txt": b"STATS", - f"{ASSEMBLY_DIR_A}_cds_from_genomic.fna.gz": b"CDS", + many_files: dict[PurePosixPath, bytes] = { + PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz"): b"GENOMIC", + PurePosixPath(f"{ASSEMBLY_DIR_A}_protein.faa.gz"): b"PROTEIN", + PurePosixPath(f"{ASSEMBLY_DIR_A}_rna.fna.gz"): b"RNA", + PurePosixPath(f"{ASSEMBLY_DIR_A}_assembly_report.txt"): b"REPORT", + PurePosixPath(f"{ASSEMBLY_DIR_A}_assembly_stats.txt"): b"STATS", + PurePosixPath(f"{ASSEMBLY_DIR_A}_cds_from_genomic.fna.gz"): b"CDS", } _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, many_files) @@ -998,19 +1005,19 @@ def test_six_files_all_promoted_with_correct_content( rel = build_accession_path(ASSEMBLY_DIR_A) for fname, expected_body in many_files.items(): - key = f"{PATH_PREFIX}{rel}{fname}" - obj = s3.get_object(Bucket=test_bucket, Key=key) + key = PATH_PREFIX / rel / fname + obj = s3.get_object(Bucket=str(test_bucket), Key=str(key)) assert obj["Body"].read() == expected_body, f"Content mismatch: {fname}" def test_md5_metadata_correct_per_file( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath ) -> None: """Each promoted file carries MD5 metadata matching its own content, not another file's.""" s3 = ceph_s3_client - files = { - f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"GENOMIC_UNIQUE", - f"{ASSEMBLY_DIR_A}_protein.faa.gz": b"PROTEIN_UNIQUE", - f"{ASSEMBLY_DIR_A}_rna.fna.gz": b"RNA_UNIQUE", + files: dict[PurePosixPath, bytes] = { + PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz"): b"GENOMIC_UNIQUE", + PurePosixPath(f"{ASSEMBLY_DIR_A}_protein.faa.gz"): b"PROTEIN_UNIQUE", + PurePosixPath(f"{ASSEMBLY_DIR_A}_rna.fna.gz"): b"RNA_UNIQUE", } _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, files, with_md5=True) @@ -1023,16 +1030,16 @@ def test_md5_metadata_correct_per_file( rel = build_accession_path(ASSEMBLY_DIR_A) for fname, content in files.items(): - key = f"{PATH_PREFIX}{rel}{fname}" + key = PATH_PREFIX / rel / fname meta = get_object_metadata(s3, test_bucket, key) assert meta.get("md5") == _md5(content), f"Wrong MD5 metadata on {fname}" def test_file_without_sidecar_has_no_md5_metadata( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath ) -> None: """A file staged without a .md5 sidecar is promoted but has no md5 metadata key.""" s3 = ceph_s3_client - fname = f"{ASSEMBLY_DIR_A}_genomic.fna.gz" + fname = PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz") _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, {fname: FAKE_GENOMIC}, with_md5=False) promote_from_s3( @@ -1043,7 +1050,7 @@ def test_file_without_sidecar_has_no_md5_metadata( ) rel = build_accession_path(ASSEMBLY_DIR_A) - meta = get_object_metadata(s3, test_bucket, f"{PATH_PREFIX}{rel}{fname}") + meta = get_object_metadata(s3, test_bucket, PATH_PREFIX / rel / fname) assert "md5" not in meta, f"Expected no md5 metadata, got: {meta}" @@ -1053,13 +1060,13 @@ class TestPromoteStagingCleanup: """After a fully successful promote, all staged files and sidecars are deleted.""" def test_staged_data_files_deleted( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath ) -> None: """Data files are removed from staging after a successful assembly promote.""" s3 = ceph_s3_client - files = { - f"{ASSEMBLY_DIR_A}_genomic.fna.gz": FAKE_GENOMIC, - f"{ASSEMBLY_DIR_A}_protein.faa.gz": FAKE_PROTEIN, + files: dict[PurePosixPath, bytes] = { + PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz"): FAKE_GENOMIC, + PurePosixPath(f"{ASSEMBLY_DIR_A}_protein.faa.gz"): FAKE_PROTEIN, } _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, files, with_md5=False) @@ -1073,18 +1080,20 @@ def test_staged_data_files_deleted( remaining_staging = list_all_keys(s3, staging_test_bucket, STAGING_PREFIX) assert len(remaining_staging) == 0, f"Staging not cleaned: {remaining_staging}" - def test_md5_sidecars_deleted(self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: + def test_md5_sidecars_deleted( + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath + ) -> None: """Both data files and .md5 sidecars are removed from staging after promote.""" s3 = ceph_s3_client - files = { - f"{ASSEMBLY_DIR_A}_genomic.fna.gz": FAKE_GENOMIC, - f"{ASSEMBLY_DIR_A}_protein.faa.gz": FAKE_PROTEIN, + files: dict[PurePosixPath, bytes] = { + PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz"): FAKE_GENOMIC, + PurePosixPath(f"{ASSEMBLY_DIR_A}_protein.faa.gz"): FAKE_PROTEIN, } _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, files, with_md5=True) # Verify sidecars exist before promote before_keys = list_all_keys(s3, staging_test_bucket, STAGING_PREFIX) - assert any(k.endswith(".md5") for k in before_keys), "Test setup: expected .md5 sidecars" + assert any(k.match("**.md5") for k in before_keys), "Test setup: expected .md5 sidecars" promote_from_s3( staging_key_prefix=STAGING_PREFIX, @@ -1097,7 +1106,7 @@ def test_md5_sidecars_deleted(self, ceph_s3_client: object, test_bucket: str, st assert len(after_keys) == 0, f"Staging not fully cleaned (including sidecars): {after_keys}" def test_two_assemblies_staging_both_cleaned( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath ) -> None: """Staging for both assemblies is fully cleaned when both assemblies succeed.""" s3 = ceph_s3_client @@ -1105,14 +1114,14 @@ def test_two_assemblies_staging_both_cleaned( s3, staging_test_bucket, ASSEMBLY_DIR_A, - {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": FAKE_GENOMIC}, + {PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz"): FAKE_GENOMIC}, with_md5=True, ) _stage_many( s3, staging_test_bucket, ASSEMBLY_DIR_B, - {f"{ASSEMBLY_DIR_B}_genomic.fna.gz": FAKE_GENOMIC}, + {PurePosixPath(f"{ASSEMBLY_DIR_B}_genomic.fna.gz"): FAKE_GENOMIC}, with_md5=True, ) @@ -1135,12 +1144,12 @@ class TestPromoteTwoAssembliesBothLand: """Both assemblies staged together are both promoted to correct Lakehouse paths.""" def test_both_assemblies_at_correct_final_paths( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath ) -> None: """Each assembly's files appear at distinct, correctly-routed final Lakehouse paths.""" s3 = ceph_s3_client - files_a = {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"genomic-A"} - files_b = {f"{ASSEMBLY_DIR_B}_genomic.fna.gz": b"genomic-B"} + files_a = {PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz"): b"genomic-A"} + files_b = {PurePosixPath(f"{ASSEMBLY_DIR_B}_genomic.fna.gz"): b"genomic-B"} _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, files_a) _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_B, files_b) @@ -1156,18 +1165,22 @@ def test_both_assemblies_at_correct_final_paths( rel_a = build_accession_path(ASSEMBLY_DIR_A) rel_b = build_accession_path(ASSEMBLY_DIR_B) - obj_a = s3.get_object(Bucket=test_bucket, Key=f"{PATH_PREFIX}{rel_a}{ASSEMBLY_DIR_A}_genomic.fna.gz") - obj_b = s3.get_object(Bucket=test_bucket, Key=f"{PATH_PREFIX}{rel_b}{ASSEMBLY_DIR_B}_genomic.fna.gz") + obj_a = s3.get_object( + Bucket=str(test_bucket), Key=str(PATH_PREFIX / rel_a / f"{ASSEMBLY_DIR_A}_genomic.fna.gz") + ) + obj_b = s3.get_object( + Bucket=str(test_bucket), Key=str(PATH_PREFIX / rel_b / f"{ASSEMBLY_DIR_B}_genomic.fna.gz") + ) assert obj_a["Body"].read() == b"genomic-A" assert obj_b["Body"].read() == b"genomic-B" def test_final_path_keys_do_not_overlap( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath ) -> None: """Files for assembly A and assembly B land at distinct paths — no key collision.""" s3 = ceph_s3_client - _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": b"a"}) - _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_B, {f"{ASSEMBLY_DIR_B}_genomic.fna.gz": b"b"}) + _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, {PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz"): b"a"}) + _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_B, {PurePosixPath(f"{ASSEMBLY_DIR_B}_genomic.fna.gz"): b"b"}) promote_from_s3( staging_key_prefix=STAGING_PREFIX, @@ -1178,8 +1191,8 @@ def test_final_path_keys_do_not_overlap( rel_a = build_accession_path(ASSEMBLY_DIR_A) rel_b = build_accession_path(ASSEMBLY_DIR_B) - keys_a = list_all_keys(s3, test_bucket, f"{PATH_PREFIX}{rel_a}") - keys_b = list_all_keys(s3, test_bucket, f"{PATH_PREFIX}{rel_b}") + keys_a = list_all_keys(s3, test_bucket, PATH_PREFIX / rel_a) + keys_b = list_all_keys(s3, test_bucket, PATH_PREFIX / rel_b) assert len(keys_a) == 1 assert len(keys_b) == 1 assert keys_a[0] != keys_b[0] @@ -1191,14 +1204,14 @@ class TestPromoteDryRunMultiFile: """dry_run leaves staging untouched and writes nothing to the Lakehouse.""" def test_dry_run_many_files_staging_untouched( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath ) -> None: """All staged files (data + .md5) survive a dry-run promote unchanged.""" s3 = ceph_s3_client - many_files = { - f"{ASSEMBLY_DIR_A}_genomic.fna.gz": FAKE_GENOMIC, - f"{ASSEMBLY_DIR_A}_protein.faa.gz": FAKE_PROTEIN, - f"{ASSEMBLY_DIR_A}_rna.fna.gz": b"RNA", + many_files: dict[PurePosixPath, bytes] = { + PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz"): FAKE_GENOMIC, + PurePosixPath(f"{ASSEMBLY_DIR_A}_protein.faa.gz"): FAKE_PROTEIN, + PurePosixPath(f"{ASSEMBLY_DIR_A}_rna.fna.gz"): b"RNA", } _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, many_files, with_md5=True) staging_before = list_all_keys(s3, staging_test_bucket, STAGING_PREFIX) @@ -1219,16 +1232,20 @@ def test_dry_run_many_files_staging_untouched( assert staging_after == staging_before, "Dry-run should not alter staging" # Nothing at final path - final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX / "raw_data") assert len(final_keys) == 0, f"Dry-run created Lakehouse objects: {final_keys}" def test_dry_run_two_assemblies_nothing_written( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath ) -> None: """Dry-run with two staged assemblies creates no Lakehouse objects.""" s3 = ceph_s3_client - _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_A, {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": FAKE_GENOMIC}) - _stage_many(s3, staging_test_bucket, ASSEMBLY_DIR_B, {f"{ASSEMBLY_DIR_B}_genomic.fna.gz": FAKE_GENOMIC}) + _stage_many( + s3, staging_test_bucket, ASSEMBLY_DIR_A, {PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz"): FAKE_GENOMIC} + ) + _stage_many( + s3, staging_test_bucket, ASSEMBLY_DIR_B, {PurePosixPath(f"{ASSEMBLY_DIR_B}_genomic.fna.gz"): FAKE_GENOMIC} + ) promote_from_s3( staging_key_prefix=STAGING_PREFIX, @@ -1238,7 +1255,7 @@ def test_dry_run_two_assemblies_nothing_written( dry_run=True, ) - final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX / "raw_data") assert len(final_keys) == 0, f"Dry-run created objects: {final_keys}" @@ -1247,14 +1264,16 @@ def test_dry_run_two_assemblies_nothing_written( class TestPromoteSecondRunOnEmptyStaging: """After staging is cleaned, a second promote run promotes 0 files without error.""" - def test_second_run_promoted_zero(self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str) -> None: + def test_second_run_promoted_zero( + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath + ) -> None: """Re-running promote on already-cleaned staging succeeds with promoted=0.""" s3 = ceph_s3_client _stage_many( s3, staging_test_bucket, ASSEMBLY_DIR_A, - {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": FAKE_GENOMIC}, + {PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz"): FAKE_GENOMIC}, with_md5=True, ) @@ -1277,11 +1296,11 @@ def test_second_run_promoted_zero(self, ceph_s3_client: object, test_bucket: str # Final key still present after second run rel = build_accession_path(ASSEMBLY_DIR_A) - final_keys = list_all_keys(s3, test_bucket, f"{PATH_PREFIX}{rel}") + final_keys = list_all_keys(s3, test_bucket, PATH_PREFIX / rel) assert len(final_keys) == 1 def test_lakehouse_unchanged_on_second_run( - self, ceph_s3_client: object, test_bucket: str, staging_test_bucket: str + self, ceph_s3_client: BaseClient, test_bucket: PurePosixPath, staging_test_bucket: PurePosixPath ) -> None: """Lakehouse contents are identical before and after a second (no-op) promote run.""" s3 = ceph_s3_client @@ -1289,7 +1308,10 @@ def test_lakehouse_unchanged_on_second_run( s3, staging_test_bucket, ASSEMBLY_DIR_A, - {f"{ASSEMBLY_DIR_A}_genomic.fna.gz": FAKE_GENOMIC, f"{ASSEMBLY_DIR_A}_protein.faa.gz": FAKE_PROTEIN}, + { + PurePosixPath(f"{ASSEMBLY_DIR_A}_genomic.fna.gz"): FAKE_GENOMIC, + PurePosixPath(f"{ASSEMBLY_DIR_A}_protein.faa.gz"): FAKE_PROTEIN, + }, with_md5=True, ) @@ -1299,7 +1321,7 @@ def test_lakehouse_unchanged_on_second_run( lakehouse_bucket=test_bucket, lakehouse_key_prefix=PATH_PREFIX, ) - keys_after_first = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + keys_after_first = list_all_keys(s3, test_bucket, PATH_PREFIX / "raw_data") promote_from_s3( staging_key_prefix=STAGING_PREFIX, @@ -1307,6 +1329,6 @@ def test_lakehouse_unchanged_on_second_run( lakehouse_bucket=test_bucket, lakehouse_key_prefix=PATH_PREFIX, ) - keys_after_second = list_all_keys(s3, test_bucket, PATH_PREFIX + "raw_data/") + keys_after_second = list_all_keys(s3, test_bucket, PATH_PREFIX / "raw_data") assert keys_after_first == keys_after_second diff --git a/tests/ncbi_ftp/conftest.py b/tests/ncbi_ftp/conftest.py index 97f5e035..3279f70c 100644 --- a/tests/ncbi_ftp/conftest.py +++ b/tests/ncbi_ftp/conftest.py @@ -1,20 +1,23 @@ """Shared fixtures for ncbi_ftp tests.""" from collections.abc import Generator -from unittest.mock import patch +from pathlib import PurePosixPath +from types import SimpleNamespace +from unittest.mock import MagicMock, patch import boto3 import botocore.client import pytest from moto import mock_aws +import cdm_data_loaders.ncbi_ftp.manifest as manifest_mod import cdm_data_loaders.ncbi_ftp.promote as promote_mod import cdm_data_loaders.utils.s3 as s3_utils from cdm_data_loaders.utils.s3 import CDM_LAKE_BUCKET, reset_s3_client from tests.s3_helpers import strip_checksum_algorithm AWS_REGION = "us-east-1" -TEST_BUCKET = CDM_LAKE_BUCKET +TEST_BUCKET: PurePosixPath = PurePosixPath(CDM_LAKE_BUCKET) # Minimal assembly_summary_refseq.txt content (tab-separated, 20+ columns) @@ -44,6 +47,19 @@ "ASM9999v1\t\t\t\tna\n" ) +# Relative paths for test accessions +ACC_PATH_215 = PurePosixPath("GCF") / "000" / "001" / "215" / "GCF_000001215.4_Release_6_plus_ISO1_MT" +ACC_PATH_405 = PurePosixPath("GCF") / "000" / "001" / "405" / "GCF_000001405.40_GRCh38.p14" +ACC_PATH_845 = PurePosixPath("GCF") / "000" / "005" / "845" / "GCF_000005845.2_ASM584v2" +ACC_PATH_999 = PurePosixPath("GCF") / "000" / "009" / "999" / "GCF_000009999.1_ASM999v1" + +_MD5_CHECKSUMS_TXT = ( + "d41d8cd98f00b204e9800998ecf8427e ./GCF_000001215.4_Release_6_plus_ISO1_MT_genomic.fna.gz\n" + "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4 ./GCF_000001215.4_Release_6_plus_ISO1_MT_protein.faa.gz\n" + "ffffffffffffffffffffffffffffffff ./GCF_000001215.4_Release_6_plus_ISO1_MT_assembly_report.txt\n" + "0000000000000000000000000000dead ./GCF_000001215.4_Release_6_plus_ISO1_MT_README.txt\n" +) + @pytest.fixture def mock_s3_client(monkeypatch: pytest.MonkeyPatch) -> Generator[botocore.client.BaseClient]: @@ -58,7 +74,7 @@ def mock_s3_client(monkeypatch: pytest.MonkeyPatch) -> Generator[botocore.client with mock_aws(): client = boto3.client("s3", region_name=AWS_REGION) - client.create_bucket(Bucket=TEST_BUCKET) + client.create_bucket(Bucket=str(TEST_BUCKET)) reset_s3_client() with ( @@ -75,3 +91,15 @@ def mock_s3_client_no_checksum(mock_s3_client: botocore.client.BaseClient) -> bo mock_s3_client.copy_object = strip_checksum_algorithm(mock_s3_client.copy_object) # type: ignore[method-assign] mock_s3_client.upload_file = strip_checksum_algorithm(mock_s3_client.upload_file) # type: ignore[method-assign] return mock_s3_client + + +@pytest.fixture +def manifest_transfer_mocks(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace: + """Mocked manifest transfer functions for retrieving MD5 checksum text and listing matching S3 objects.""" + mock_retrieve = MagicMock(return_value=_MD5_CHECKSUMS_TXT) + mock_list = MagicMock(return_value=["one key"]) + + monkeypatch.setattr(manifest_mod, "ftp_retrieve_text", mock_retrieve) + monkeypatch.setattr(manifest_mod, "list_matching_objects", mock_list) + + return SimpleNamespace(retrieve=mock_retrieve, list=mock_list) diff --git a/tests/ncbi_ftp/test_assembly.py b/tests/ncbi_ftp/test_assembly.py index d15df95d..67534165 100644 --- a/tests/ncbi_ftp/test_assembly.py +++ b/tests/ncbi_ftp/test_assembly.py @@ -1,5 +1,7 @@ """Tests for ncbi_ftp.assembly module — path helpers, file filtering, checksum parsing.""" +from pathlib import PurePosixPath + import pytest from cdm_data_loaders.ncbi_ftp.assembly import ( @@ -8,6 +10,8 @@ parse_md5_checksums_file, ) +from .conftest import ACC_PATH_215 + # Path helpers @@ -16,17 +20,17 @@ [ pytest.param( "GCF_000001215.4_Release_6_plus_ISO1_MT", - "raw_data/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/", + PurePosixPath("raw_data") / ACC_PATH_215, id="gcf", ), pytest.param( "GCA_012345678.1_ASM1234v1", - "raw_data/GCA/012/345/678/GCA_012345678.1_ASM1234v1/", + PurePosixPath("raw_data") / "GCA" / "012" / "345" / "678" / "GCA_012345678.1_ASM1234v1", id="gca", ), ], ) -def test_build_accession_path(assembly_dir: str, expected: str) -> None: +def test_build_accession_path(assembly_dir: PurePosixPath, expected: str) -> None: """Verify accession path construction for various inputs.""" assert build_accession_path(assembly_dir) == expected @@ -34,25 +38,25 @@ def test_build_accession_path(assembly_dir: str, expected: str) -> None: def test_build_accession_path_invalid() -> None: """Verify ValueError on invalid assembly name.""" with pytest.raises(ValueError, match="Cannot parse"): - build_accession_path("invalid_name") + build_accession_path(PurePosixPath("invalid_name")) @pytest.mark.parametrize( ("path", "expected"), [ pytest.param( - "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/", - ("GCF", "GCF_000001215.4_Release_6_plus_ISO1_MT", "GCF_000001215.4"), + PurePosixPath("/") / "genomes" / "all" / ACC_PATH_215, + ("GCF", PurePosixPath("GCF_000001215.4_Release_6_plus_ISO1_MT"), "GCF_000001215.4"), id="with_trailing_slash", ), pytest.param( - "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT", - ("GCF", "GCF_000001215.4_Release_6_plus_ISO1_MT", "GCF_000001215.4"), + PurePosixPath("/") / "genomes" / "all" / ACC_PATH_215, + ("GCF", PurePosixPath("GCF_000001215.4_Release_6_plus_ISO1_MT"), "GCF_000001215.4"), id="without_trailing_slash", ), ], ) -def test_parse_assembly_path(path: str, expected: tuple[str, str, str]) -> None: +def test_parse_assembly_path(path: PurePosixPath, expected: tuple[str, PurePosixPath, str]) -> None: """Verify db, assembly_dir, and accession are parsed correctly.""" assert parse_assembly_path(path) == expected @@ -60,7 +64,7 @@ def test_parse_assembly_path(path: str, expected: tuple[str, str, str]) -> None: def test_parse_assembly_path_invalid() -> None: """Verify ValueError on invalid path.""" with pytest.raises(ValueError, match="Cannot parse"): - parse_assembly_path("/random/path/") + parse_assembly_path(PurePosixPath("/") / "random" / "path") # parse_md5_checksums_file @@ -71,23 +75,26 @@ def test_parse_assembly_path_invalid() -> None: [ pytest.param( "abc123 ./GCF_000001215.4_genomic.fna.gz\ndef456 ./GCF_000001215.4_genomic.gff.gz\n", - {"GCF_000001215.4_genomic.fna.gz": "abc123", "GCF_000001215.4_genomic.gff.gz": "def456"}, + { + PurePosixPath("GCF_000001215.4_genomic.fna.gz"): "abc123", + PurePosixPath("GCF_000001215.4_genomic.gff.gz"): "def456", + }, id="dot_slash_prefix", ), pytest.param( "abc123 GCF_000001215.4_genomic.fna.gz\n", - {"GCF_000001215.4_genomic.fna.gz": "abc123"}, + {PurePosixPath("GCF_000001215.4_genomic.fna.gz"): "abc123"}, id="no_dot_slash_prefix", ), pytest.param("", {}, id="empty_string"), pytest.param(" \n \n", {}, id="whitespace_only"), pytest.param( "abc123 file1.txt\n\n\ndef456 file2.txt\n", - {"file1.txt": "abc123", "file2.txt": "def456"}, + {PurePosixPath("file1.txt"): "abc123", PurePosixPath("file2.txt"): "def456"}, id="blank_lines_ignored", ), ], ) -def test_parse_md5_checksums_file(text: str, expected: dict[str, str]) -> None: +def test_parse_md5_checksums_file(text: str, expected: dict[PurePosixPath, str]) -> None: """Verify parse_md5_checksums_file handles various input formats.""" assert parse_md5_checksums_file(text) == expected diff --git a/tests/ncbi_ftp/test_manifest.py b/tests/ncbi_ftp/test_manifest.py index b558f162..3d5f38fc 100644 --- a/tests/ncbi_ftp/test_manifest.py +++ b/tests/ncbi_ftp/test_manifest.py @@ -2,7 +2,8 @@ import json from datetime import UTC, datetime -from pathlib import Path +from pathlib import Path, PurePosixPath +from types import SimpleNamespace from typing import Any from unittest.mock import MagicMock, patch @@ -27,7 +28,7 @@ write_updated_manifest, ) -from .conftest import SAMPLE_SUMMARY +from .conftest import ACC_PATH_215, ACC_PATH_405, SAMPLE_SUMMARY _EXPECTED_TWO = 2 @@ -40,29 +41,29 @@ accession="GCF_000001215.4", status="latest", seq_rel_date="2014/10/21", - ftp_path="https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT", - assembly_dir="GCF_000001215.4_Release_6_plus_ISO1_MT", + ftp_url="https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT", + assembly_dir=PurePosixPath("GCF_000001215.4_Release_6_plus_ISO1_MT"), ), "GCF_000001405.40": AssemblyRecord( accession="GCF_000001405.40", status="latest", seq_rel_date="2022/02/03", - ftp_path="https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/405/GCF_000001405.40_GRCh38.p14", - assembly_dir="GCF_000001405.40_GRCh38.p14", + ftp_url="https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/405/GCF_000001405.40_GRCh38.p14", + assembly_dir=PurePosixPath("GCF_000001405.40_GRCh38.p14"), ), "GCF_000005845.2": AssemblyRecord( accession="GCF_000005845.2", status="replaced", seq_rel_date="2013/09/26", - ftp_path="https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/005/845/GCF_000005845.2_ASM584v2", - assembly_dir="GCF_000005845.2_ASM584v2", + ftp_url="https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/005/845/GCF_000005845.2_ASM584v2", + assembly_dir=PurePosixPath("GCF_000005845.2_ASM584v2"), ), "GCF_000009999.1": AssemblyRecord( accession="GCF_000009999.1", status="suppressed", seq_rel_date="2010/01/01", - ftp_path="https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/009/999/GCF_000009999.1_ASM999v1", - assembly_dir="GCF_000009999.1_ASM999v1", + ftp_url="https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/009/999/GCF_000009999.1_ASM999v1", + assembly_dir=PurePosixPath("GCF_000009999.1_ASM999v1"), ), # GCF_000099999.1 is excluded because ftp_path == "na" } @@ -82,7 +83,7 @@ def test_parse_assembly_summary_empty() -> None: def test_parse_assembly_summary_input_types(source: str, tmp_path: Path) -> None: """Parsing works from a file path, string path, and list of lines.""" if source == "list_of_lines": - arg = SAMPLE_SUMMARY.splitlines(keepends=True) + arg: str | Path | list[str] = list(SAMPLE_SUMMARY.splitlines(keepends=True)) else: f = tmp_path / "summary.tsv" f.write_text(SAMPLE_SUMMARY) @@ -96,8 +97,8 @@ def test_parse_assembly_summary_input_types(source: str, tmp_path: Path) -> None def test_get_latest_assembly_paths() -> None: """Only 'latest' assemblies appear; paths are FTP directories with trailing slash.""" assert dict(get_latest_assembly_paths(parse_assembly_summary(SAMPLE_SUMMARY))) == { - "GCF_000001215.4": "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/", - "GCF_000001405.40": "/genomes/all/GCF/000/001/405/GCF_000001405.40_GRCh38.p14/", + "GCF_000001215.4": PurePosixPath("/") / "genomes" / "all" / ACC_PATH_215, + "GCF_000001405.40": PurePosixPath("/") / "genomes" / "all" / ACC_PATH_405, } @@ -163,8 +164,12 @@ def test_compute_diff_scan_store_fallback() -> None: ], ) def test_accession_prefix(accession: str, expected: str | None) -> None: - """3-digit prefix is extracted from the accession; invalid input returns None.""" - assert accession_prefix(accession) == expected + """3-digit prefix is extracted from the accession; invalid input raises ValueError.""" + if expected is None: + with pytest.raises(ValueError, match="Could not parse accession"): + accession_prefix(accession) + else: + assert accession_prefix(accession) == expected def test_filter_by_prefix_range() -> None: @@ -185,11 +190,10 @@ def test_write_transfer_manifest(tmp_path: Path) -> None: manifest_file = tmp_path / "transfer.txt" paths = write_transfer_manifest(diff, current, manifest_file) assert len(paths) > 0 - lines = [line.strip() for line in manifest_file.read_text().splitlines() if line.strip()] - assert len(lines) == len(paths) - for line in lines: - assert line.startswith("/genomes/") - assert line.endswith("/") + for path in paths: + assert path.is_relative_to("/genomes") + parts = path.parts + assert len(parts) == 8 # noqa: PLR2004 /genomes/all/GCF/012/345/678/GCF_012345678_blah_blah (initial '/' counts as a part) def test_write_removed_manifest(tmp_path: Path) -> None: @@ -241,63 +245,57 @@ def test_write_diff_summary(tmp_path: Path) -> None: ("url", "expected", "kwargs"), [ pytest.param( - "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6", - "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6", + "https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT", + PurePosixPath("/") / "genomes" / "all" / ACC_PATH_215, {}, id="https_url", ), pytest.param( - "ftp://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6", - "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6", + "ftp://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT", + PurePosixPath("/") / "genomes" / "all" / ACC_PATH_215, {}, id="ftp_url", ), pytest.param( - "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6", - "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6", + "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT", + None, {}, id="bare_path", ), pytest.param( "ftp://custom.host.example.com/genomes/all/GCF/000/001/215", - "/genomes/all/GCF/000/001/215", + PurePosixPath("/") / "genomes" / "all" / "GCF" / "000" / "001" / "215", {"ftp_host": "custom.host.example.com"}, id="custom_ftp_host", ), ], ) -def test_ftp_dir_from_url(url: str, expected: str, kwargs: dict) -> None: - assert _ftp_dir_from_url(url, **kwargs) == expected +def test_ftp_dir_from_url(url: str, expected: str | None, kwargs: dict) -> None: + """Wrapper for _ftp_dir_from_url.""" + if expected is None: + with pytest.raises(ValueError, match="Could not parse FTP URL"): + _ftp_dir_from_url(url, **kwargs) + else: + assert _ftp_dir_from_url(url, **kwargs) == expected # verify_transfer_candidates -_MD5_CHECKSUMS_TXT = ( - "d41d8cd98f00b204e9800998ecf8427e ./GCF_000001215.4_Release_6_plus_ISO1_MT_genomic.fna.gz\n" - "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4 ./GCF_000001215.4_Release_6_plus_ISO1_MT_protein.faa.gz\n" - "ffffffffffffffffffffffffffffffff ./GCF_000001215.4_Release_6_plus_ISO1_MT_assembly_report.txt\n" - "0000000000000000000000000000dead ./GCF_000001215.4_Release_6_plus_ISO1_MT_README.txt\n" -) - - -_BUCKET = "cdm-lake" -_KEY_PREFIX = "tenant-general-warehouse/kbase/datasets/ncbi/" +_BUCKET: PurePosixPath = PurePosixPath("cdm-lake") +_KEY_PREFIX: PurePosixPath = PurePosixPath("tenant-general-warehouse") / "kbase" / "datasets " / "ncbi" def _assemblies() -> dict: return parse_assembly_summary(SAMPLE_SUMMARY) -@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects", return_value=["one key"]) +@pytest.mark.usefixtures("manifest_transfer_mocks") @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") -@patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") def test_verify_transfer_candidates_prunes_when_all_match( mock_connect: MagicMock, - mock_retrieve: MagicMock, mock_head: MagicMock, - mock_list: MagicMock, ) -> None: """Assemblies where every file matches S3 are pruned from the list.""" mock_connect.return_value = MagicMock() @@ -327,15 +325,12 @@ def head_side_effect(s3_path: str) -> dict | None: assert verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) == [] -@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects", return_value=["one key"]) +@pytest.mark.usefixtures("manifest_transfer_mocks") @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") -@patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") def test_verify_transfer_candidates_keeps_when_md5_differs( mock_connect: MagicMock, - mock_retrieve: MagicMock, mock_head: MagicMock, - mock_list: MagicMock, ) -> None: """Assembly is kept when at least one file has a different MD5.""" mock_connect.return_value = MagicMock() @@ -343,15 +338,12 @@ def test_verify_transfer_candidates_keeps_when_md5_differs( assert verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) == ["GCF_000001215.4"] -@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects", return_value=["one key"]) +@pytest.mark.usefixtures("manifest_transfer_mocks") @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") -@patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") def test_verify_transfer_candidates_keeps_when_s3_object_missing( mock_connect: MagicMock, - mock_retrieve: MagicMock, mock_head: MagicMock, - mock_list: MagicMock, ) -> None: """Assembly is kept when at least one file doesn't exist in S3.""" mock_connect.return_value = MagicMock() @@ -365,15 +357,12 @@ def head_side_effect(s3_path: str) -> dict[str, Any]: assert verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) == ["GCF_000001215.4"] -@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects", return_value=["one key"]) +@pytest.mark.usefixtures("manifest_transfer_mocks") @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") -@patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") def test_verify_transfer_candidates_keeps_when_no_md5_metadata( mock_connect: MagicMock, - mock_retrieve: MagicMock, mock_head: MagicMock, - mock_list: MagicMock, ) -> None: """Assembly is kept when S3 object exists but has no md5 metadata.""" mock_connect.return_value = MagicMock() @@ -381,40 +370,38 @@ def test_verify_transfer_candidates_keeps_when_no_md5_metadata( assert verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) == ["GCF_000001215.4"] -@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects", return_value=["one key"]) -@patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", side_effect=Exception("FTP error")) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") def test_verify_transfer_candidates_keeps_when_ftp_fails( - mock_connect: MagicMock, mock_retrieve: MagicMock, mock_list: MagicMock + mock_connect: MagicMock, manifest_transfer_mocks: SimpleNamespace ) -> None: """Assembly is kept (conservative) when md5checksums.txt cannot be fetched.""" mock_connect.return_value = MagicMock() + manifest_transfer_mocks.retrieve.side_effect = Exception("FTP error") assert verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) == ["GCF_000001215.4"] -@patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") -def test_verify_transfer_candidates_empty_input(mock_connect: MagicMock) -> None: +def test_verify_transfer_candidates_empty_input() -> None: """Empty accession list returns empty result without connecting.""" - assert verify_transfer_candidates([], {}, _BUCKET, "prefix/") == [] + with patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp", return_value=MagicMock()): + assert verify_transfer_candidates([], {}, _BUCKET, PurePosixPath("prefix")) == [] +@pytest.mark.usefixtures("manifest_transfer_mocks") @patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects", return_value=["one key"]) -@patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") -def test_verify_transfer_candidates_unknown_accession_kept(mock_connect: MagicMock, mock_list: MagicMock) -> None: +def test_verify_transfer_candidates_unknown_accession_kept( + mock_connect: MagicMock, +) -> None: """Accessions not in assemblies dict are kept (conservative).""" mock_connect.return_value = MagicMock() - assert verify_transfer_candidates(["GCF_999999999.1"], {}, _BUCKET, "prefix/") == ["GCF_999999999.1"] + assert verify_transfer_candidates(["GCF_999999999.1"], {}, _BUCKET, PurePosixPath("prefix")) == ["GCF_999999999.1"] -@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects", return_value=["one key"]) +@pytest.mark.usefixtures("manifest_transfer_mocks") @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") -@patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") def test_verify_transfer_candidates_short_circuits_on_first_mismatch( mock_connect: MagicMock, - mock_retrieve: MagicMock, mock_head: MagicMock, - mock_list: MagicMock, ) -> None: """Verification stops checking after the first missing/mismatched file.""" mock_connect.return_value = MagicMock() @@ -429,15 +416,12 @@ def head_side_effects(s3_path: str) -> dict[str, Any]: assert mock_head.call_count == 1 -@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects", return_value=["one key"]) +@pytest.mark.usefixtures("manifest_transfer_mocks") @patch("cdm_data_loaders.ncbi_ftp.manifest.head_object") -@patch("cdm_data_loaders.ncbi_ftp.manifest.ftp_retrieve_text", return_value=_MD5_CHECKSUMS_TXT) @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") def test_verify_transfer_candidates_mixed( mock_connect: MagicMock, - mock_retrieve: MagicMock, mock_head: MagicMock, - mock_list: MagicMock, ) -> None: """A mix of matching and non-matching assemblies: matched pruned, unmatched kept.""" mock_connect.return_value = MagicMock() @@ -469,13 +453,14 @@ def head_side_effect(s3_path: str) -> dict | None: assert result == ["GCF_000001405.40"] -@patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects", return_value=[]) +@pytest.mark.usefixtures("manifest_transfer_mocks") @patch("cdm_data_loaders.ncbi_ftp.manifest.connect_ftp") def test_verify_transfer_candidates_skips_ftp_when_folder_missing( mock_connect: MagicMock, - mock_list: MagicMock, + manifest_transfer_mocks: SimpleNamespace, ) -> None: """Accessions with no objects in S3 are confirmed without FTP round-trip.""" + manifest_transfer_mocks.list.return_value = [] result = verify_transfer_candidates(["GCF_000001215.4"], _assemblies(), _BUCKET, _KEY_PREFIX) assert result == ["GCF_000001215.4"] mock_connect.assert_not_called() @@ -488,42 +473,68 @@ def test_verify_transfer_candidates_skips_ftp_when_folder_missing( ("key", "expected"), [ pytest.param( - "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6_plus_ISO1_MT/file.gz", + PurePosixPath("tenant-general-warehouse") + / "kbase" + / "datasets" + / "ncbi" + / "refseq" + / "GCF_000001215.4_Release_6_plus_ISO1_MT" + / "file.gz", ("GCF_000001215.4_Release_6_plus_ISO1_MT", "GCF_000001215.4"), id="long_path", ), pytest.param( - "some/path/GCA_999999999.1_whatever/data.txt", + PurePosixPath("some") / "path" / "GCA_999999999.1_whatever" / "data.txt", ("GCA_999999999.1_whatever", "GCA_999999999.1"), id="short_path", ), pytest.param( - "prefix/GCA_999999999.1_assembly_name/subdir/data.txt", + PurePosixPath("prefix") / "GCA_999999999.1_assembly_name" / "subdir" / "data.txt", ("GCA_999999999.1_assembly_name", "GCA_999999999.1"), id="subdir", ), - pytest.param("some/random/path", (None, None), id="no_accession"), - pytest.param("", (None, None), id="empty"), + pytest.param(PurePosixPath("some") / "random" / "path", None, id="no_accession"), + pytest.param(PurePosixPath(""), None, id="empty"), ], ) -def test_extract_accession_dir_and_id_from_s3_key(key: str, expected: str | None) -> None: +def test_extract_accession_dir_and_id_from_s3_key(key: PurePosixPath, expected: tuple[str, str] | None) -> None: """Accession is extracted from S3 key paths; invalid/empty paths return None.""" - assert _extract_accession_dir_and_id_from_s3_key(key) == expected + if expected is None: + with pytest.raises(ValueError, match="Could not parse S3 key for accession info"): + _extract_accession_dir_and_id_from_s3_key(key) + else: + assert _extract_accession_dir_and_id_from_s3_key(key) == expected def _make_mock_list_object_return_value() -> list[dict[str, Any]]: """Return a response for list objects with two assemblies (GCF_000001215.4, GCF_000005845.2).""" return [ { - "Key": "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6/file1.gz", + "Key": PurePosixPath("tenant-general-warehouse") + / "kbase" + / "datasets" + / "ncbi" + / "refseq" + / "GCF_000001215.4_Release_6/file1.gz", "LastModified": datetime(2024, 1, 15, tzinfo=UTC), }, { - "Key": "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000001215.4_Release_6/file2.gz", + "Key": PurePosixPath("tenant-general-warehouse") + / "kbase" + / "datasets" + / "ncbi" + / "refseq" + / "GCF_000001215.4_Release_6/file2.gz", "LastModified": datetime(2024, 1, 16, tzinfo=UTC), }, { - "Key": "tenant-general-warehouse/kbase/datasets/ncbi/refseq/GCF_000005845.2_Assembly/file.gz", + "Key": PurePosixPath("tenant-general-warehouse") + / "kbase" + / "datasets" + / "ncbi" + / "refseq" + / "GCF_000005845.2_Assembly" + / "file.gz", "LastModified": datetime(2024, 2, 20, tzinfo=UTC), }, ] @@ -533,20 +544,20 @@ def _make_mock_list_object_return_value() -> list[dict[str, Any]]: def test_scan_store_builds_summary(mock_list_matching_objects: MagicMock) -> None: """Synthetic summary is built correctly with provided release_date for all assemblies.""" mock_list_matching_objects.return_value = _make_mock_list_object_return_value() - assert scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/01/31") == { + assert scan_store_to_synthetic_summary(PurePosixPath("test-bucket"), PurePosixPath("prefix"), "2024/01/31") == { "GCF_000001215.4": AssemblyRecord( accession="GCF_000001215.4", status="latest", seq_rel_date="2024/01/31", - ftp_path="https://ftp.ncbi.nlm.nih.gov/synthetic/GCF_000001215.4_Release_6", - assembly_dir="GCF_000001215.4_Release_6", + ftp_url="https://ftp.ncbi.nlm.nih.gov/synthetic/GCF_000001215.4_Release_6", + assembly_dir=PurePosixPath("GCF_000001215.4_Release_6"), ), "GCF_000005845.2": AssemblyRecord( accession="GCF_000005845.2", status="latest", seq_rel_date="2024/01/31", - ftp_path="https://ftp.ncbi.nlm.nih.gov/synthetic/GCF_000005845.2_Assembly", - assembly_dir="GCF_000005845.2_Assembly", + ftp_url="https://ftp.ncbi.nlm.nih.gov/synthetic/GCF_000005845.2_Assembly", + assembly_dir=PurePosixPath("GCF_000005845.2_Assembly"), ), } @@ -565,7 +576,9 @@ def test_scan_store_applies_release_date_to_all(mock_list_matching_objects: Magi }, ] assert ( - scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/03/31")["GCF_000001215.4"].seq_rel_date + scan_store_to_synthetic_summary(PurePosixPath("test-bucket"), PurePosixPath("prefix"), "2024/03/31")[ + "GCF_000001215.4" + ].seq_rel_date == "2024/03/31" ) @@ -575,7 +588,7 @@ def test_scan_store_raises_for_invalid_release_date(mock_list_matching_objects: """Invalid release_date format is rejected.""" mock_list_matching_objects.return_value = _make_mock_list_object_return_value() with pytest.raises(ValueError, match="Invalid release_date"): - scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024-03-31") + scan_store_to_synthetic_summary(PurePosixPath("test-bucket"), PurePosixPath("prefix"), "2024-03-31") @patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects") @@ -584,31 +597,37 @@ def test_scan_store_invokes_progress_callback(mock_list_matching_objects: MagicM mock_list_matching_objects.return_value = _make_mock_list_object_return_value() calls: list[tuple[int, str]] = [] scan_store_to_synthetic_summary( - "test-bucket", "prefix/", "2024/01/31", progress_callback=lambda n, a: calls.append((n, a)) + PurePosixPath("test-bucket"), + PurePosixPath("prefix"), + "2024/01/31", + progress_callback=lambda n, a: calls.append((n, a)), ) - assert len(calls) == 2 + assert len(calls) == 2 # noqa: PLR2004 assert calls[0][0] == 1 - assert calls[1][0] == 2 + assert calls[1][0] == 2 # noqa: PLR2004 @patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects") def test_scan_store_handles_empty_store(mock_list_matching_objects: MagicMock) -> None: """Empty store returns empty dict.""" mock_list_matching_objects.return_value = [] - assert scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/01/31") == {} + assert scan_store_to_synthetic_summary(PurePosixPath("test-bucket"), PurePosixPath("prefix"), "2024/01/31") == {} @patch("cdm_data_loaders.ncbi_ftp.manifest.list_matching_objects") def test_scan_store_skips_objects_without_accession(mock_list_matching_objects: MagicMock) -> None: """Objects without valid accessions in the key are skipped.""" mock_list_matching_objects.return_value = [ - {"Key": "prefix/some/random/file.txt", "LastModified": datetime(2024, 1, 1, tzinfo=UTC)}, { - "Key": "prefix/GCF_000001215.4_Assembly/valid_file.gz", + "Key": PurePosixPath("prefix") / "some" / "random" / "file.txt", + "LastModified": datetime(2024, 1, 1, tzinfo=UTC), + }, + { + "Key": PurePosixPath("prefix") / "GCF_000001215.4_Assembly" / "valid_file.gz", "LastModified": datetime(2024, 2, 1, tzinfo=UTC), }, ] - result = scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/01/31") + result = scan_store_to_synthetic_summary(PurePosixPath("test-bucket"), PurePosixPath("prefix"), "2024/01/31") assert len(result) == 1 assert "GCF_000001215.4" in result @@ -627,16 +646,18 @@ def test_scan_store_assembly_dir_survives_round_trip(mock_list_matching_objects: } ] - synthetic = scan_store_to_synthetic_summary("test-bucket", "prefix/", "2024/03/10") + synthetic = scan_store_to_synthetic_summary(PurePosixPath("test-bucket"), PurePosixPath("prefix"), "2024/03/10") + # save scanned records to synthetic summary file out_file = tmp_path / "synthetic_summary.txt" with out_file.open("w") as f: for acc in sorted(synthetic.keys()): rec = synthetic[acc] f.write( - f"{rec.accession}\t.\t.\t.\t.\t.\t.\t.\t.\t.\t{rec.status}\t.\t.\t.\t{rec.seq_rel_date}\t.\t.\t.\t.\t{rec.ftp_path}\t.\n" + f"{rec.accession}\t.\t.\t.\t.\t.\t.\t.\t.\t.\t{rec.status}\t.\t.\t.\t{rec.seq_rel_date}\t.\t.\t.\t.\t{rec.ftp_url}\t.\n" ) + # ensure generated summary file can be parsed to regenerate original sythentic summary records reparsed = parse_assembly_summary(out_file) assert "GCF_000001215.4" in reparsed reparsed_rec = reparsed["GCF_000001215.4"] diff --git a/tests/ncbi_ftp/test_metadata.py b/tests/ncbi_ftp/test_metadata.py index cbb58f13..93d3f306 100644 --- a/tests/ncbi_ftp/test_metadata.py +++ b/tests/ncbi_ftp/test_metadata.py @@ -1,22 +1,17 @@ """Unit tests for cdm_data_loaders.ncbi_ftp.metadata.""" -from __future__ import annotations - import json import time -from typing import TYPE_CHECKING +from collections.abc import Generator +from pathlib import PurePosixPath from unittest.mock import MagicMock, patch from urllib.parse import urlparse import boto3 +import botocore.client import pytest from moto import mock_aws -if TYPE_CHECKING: - from collections.abc import Generator - - import botocore.client - import cdm_data_loaders.ncbi_ftp.metadata as metadata_mod import cdm_data_loaders.utils.s3 as s3_utils from cdm_data_loaders.ncbi_ftp.metadata import ( @@ -34,22 +29,36 @@ AWS_REGION = "us-east-1" _ACCESSION = "GCF_000001215.4" -_ASSEMBLY_DIR = "GCF_000001215.4_Release_6_plus_ISO1_MT" +_ASSEMBLY_DIR = PurePosixPath("GCF_000001215.4_Release_6_plus_ISO1_MT") _RELEASE_TAG = "2024-01" -_KEY_PREFIX = "tenant-general-warehouse/kbase/datasets/ncbi/" +_KEY_PREFIX = PurePosixPath("tenant-general-warehouse") / "kbase" / "datasets" / "ncbi" _TIMESTAMP = 1_700_000_000 _SAMPLE_RESOURCES: list[DescriptorResource] = [ { "name": "GCF_000001215.4_genomic.fna.gz", - "path": f"{_KEY_PREFIX}raw_data/GCF/000/001/215/{_ASSEMBLY_DIR}/GCF_000001215.4_genomic.fna.gz", + "path": _KEY_PREFIX + / "raw_data" + / "GCF" + / "000" + / "001" + / "215" + / _ASSEMBLY_DIR + / "GCF_000001215.4_genomic.fna.gz", "format": "gz", "bytes": 1024, "hash": "abc123", }, { "name": "GCF_000001215.4_assembly_report.txt", - "path": f"{_KEY_PREFIX}raw_data/GCF/000/001/215/{_ASSEMBLY_DIR}/GCF_000001215.4_assembly_report.txt", + "path": _KEY_PREFIX + / "raw_data" + / "GCF" + / "000" + / "001" + / "215" + / _ASSEMBLY_DIR + / "GCF_000001215.4_assembly_report.txt", "format": "txt", "bytes": 512, "hash": None, # no md5 sidecar for this one @@ -60,27 +69,32 @@ # build_descriptor_key / build_archive_descriptor_key -@pytest.mark.parametrize("prefix", [_KEY_PREFIX, _KEY_PREFIX.rstrip("/")]) -def test_build_descriptor_key(prefix: str) -> None: +@pytest.mark.parametrize("prefix", [_KEY_PREFIX]) +def test_build_descriptor_key(prefix: PurePosixPath) -> None: """Key is under metadata/, ends with _datapackage.json, trailing slash on prefix is normalized.""" key = build_descriptor_key(_ASSEMBLY_DIR, prefix) - assert key == f"{_KEY_PREFIX}metadata/{_ASSEMBLY_DIR}_datapackage.json" - assert "//" not in key + assert key == _KEY_PREFIX / "metadata" / _ASSEMBLY_DIR.with_name(f"{_ASSEMBLY_DIR.name}_datapackage.json") @pytest.mark.parametrize( ("prefix", "tag"), [ - pytest.param(_KEY_PREFIX, _RELEASE_TAG, id="trailing_slash"), - pytest.param(_KEY_PREFIX.rstrip("/"), _RELEASE_TAG, id="no_trailing_slash"), + pytest.param(_KEY_PREFIX, _RELEASE_TAG, id="tag"), pytest.param(_KEY_PREFIX, "2025-06", id="different_tag"), ], ) -def test_build_archive_descriptor_key(prefix: str, tag: str) -> None: - """Archive key includes tag and reason segment; no double slash; prefix trailing slash normalized.""" +def test_build_archive_descriptor_key(prefix: PurePosixPath, tag: str) -> None: + """Archive key includes tag and reason segment.""" key = build_archive_descriptor_key(_ASSEMBLY_DIR, tag, prefix, "updated") - assert key == f"{_KEY_PREFIX}archive/{tag}/updated/metadata/{_ASSEMBLY_DIR}_datapackage.json" - assert "//" not in key + expected = ( + _KEY_PREFIX + / "archive" + / tag + / "updated" + / "metadata" + / _ASSEMBLY_DIR.with_name(f"{_ASSEMBLY_DIR.name}_datapackage.json") + ) + assert key == expected # create_descriptor @@ -92,7 +106,8 @@ def test_create_descriptor() -> None: # URL hostname is computed; can't express as equality host = urlparse(d["url"]).hostname - assert host is not None and (host == "ncbi.nlm.nih.gov" or host.endswith(".ncbi.nlm.nih.gov")) + assert host is not None + assert host == "ncbi.nlm.nih.gov" or host.endswith(".ncbi.nlm.nih.gov") # resource[1]: hash=None → key absent; bytes=512 → key present r1 = d["resources"][1] @@ -127,13 +142,13 @@ def test_create_descriptor() -> None: } ], } - assert _ASSEMBLY_DIR in d["titles"][0]["title"] + assert f"{_ASSEMBLY_DIR}" in d["titles"][0]["title"] assert _ACCESSION in d["descriptions"][0]["description_text"] r0 = d["resources"][0] assert {k: r0[k] for k in ("hash", "bytes", "path")} == { "hash": "abc123", "bytes": 1024, - "path": _SAMPLE_RESOURCES[0]["path"], + "path": str(_SAMPLE_RESOURCES[0].get("path")), } @@ -148,7 +163,13 @@ def test_create_descriptor_default_timestamp_is_recent() -> None: def test_create_descriptor_resource_name_lowercased() -> None: """Resource names are converted to lowercase.""" resources: list[DescriptorResource] = [ - {"name": "FILE_UPPER.FNA.GZ", "path": "s3://bucket/a", "format": "gz", "bytes": 100, "hash": "x"}, + { + "name": "FILE_UPPER.FNA.GZ", + "path": PurePosixPath("s3://") / "bucket" / "a", + "format": "gz", + "bytes": 100, + "hash": "x", + }, ] d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, resources, timestamp=_TIMESTAMP) assert d["resources"][0]["name"] == "file_upper.fna.gz" @@ -157,7 +178,7 @@ def test_create_descriptor_resource_name_lowercased() -> None: def test_create_descriptor_null_bytes_omitted() -> None: """Resources with bytes=None have the 'bytes' key removed from the output.""" resources: list[DescriptorResource] = [ - {"name": "f.txt", "path": "s3://b/f.txt", "format": "txt", "bytes": None, "hash": "x"}, + {"name": "f.txt", "path": PurePosixPath("s3://") / "b" / "f.txt", "format": "txt", "bytes": None, "hash": "x"}, ] d = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, resources, timestamp=_TIMESTAMP) assert "bytes" not in d["resources"][0] @@ -201,7 +222,7 @@ def mock_s3(monkeypatch: pytest.MonkeyPatch) -> Generator[botocore.client.BaseCl boto3.DEFAULT_SESSION = None with mock_aws(): client = boto3.client("s3", region_name=AWS_REGION) - client.create_bucket(Bucket=TEST_BUCKET) + client.create_bucket(Bucket=str(TEST_BUCKET)) reset_s3_client() with ( patch.object(s3_utils, "get_s3_client", return_value=client), @@ -212,11 +233,13 @@ def mock_s3(monkeypatch: pytest.MonkeyPatch) -> Generator[botocore.client.BaseCl @pytest.fixture -def mock_s3_with_descriptor(mock_s3: botocore.client.BaseClient) -> tuple[botocore.client.BaseClient, MagicMock]: +def mock_s3_with_descriptor( + mock_s3: botocore.client.BaseClient, +) -> Generator[tuple[botocore.client.BaseClient, MagicMock]]: """mock_s3 with a live descriptor pre-uploaded and copy_object patched.""" descriptor = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) live_key = build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX) - mock_s3.put_object(Bucket=TEST_BUCKET, Key=live_key, Body=json.dumps(descriptor).encode()) + mock_s3.put_object(Bucket=str(TEST_BUCKET), Key=f"{live_key}", Body=json.dumps(descriptor).encode()) with patch.object(metadata_mod, "copy_object") as mock_copy: yield mock_s3, mock_copy @@ -228,9 +251,9 @@ def test_upload_descriptor(mock_s3: botocore.client.BaseClient) -> None: key = upload_descriptor(descriptor, _ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX) expected_key = build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX) assert key == expected_key - assert key.startswith(_KEY_PREFIX) - assert key.endswith("_datapackage.json") - body = json.loads(mock_s3.get_object(Bucket=TEST_BUCKET, Key=key)["Body"].read()) + assert key.is_relative_to(_KEY_PREFIX) + assert key.match("**_datapackage.json") + body = json.loads(mock_s3.get_object(Bucket=str(TEST_BUCKET), Key=str(key))["Body"].read()) assert body["identifier"] == f"NCBI:{_ACCESSION}" @@ -240,7 +263,7 @@ def test_upload_descriptor_dry_run(mock_s3: botocore.client.BaseClient) -> None: descriptor = create_descriptor(_ASSEMBLY_DIR, _ACCESSION, _SAMPLE_RESOURCES, timestamp=_TIMESTAMP) key = upload_descriptor(descriptor, _ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX, dry_run=True) assert key == build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX) - objs = mock_s3.list_objects_v2(Bucket=TEST_BUCKET).get("Contents", []) + objs = mock_s3.list_objects_v2(Bucket=str(TEST_BUCKET)).get("Contents", []) assert not any(o["Key"] == key for o in objs) @@ -252,8 +275,8 @@ def test_archive_descriptor(mock_s3_with_descriptor: tuple[botocore.client.BaseC assert result is True mock_copy.assert_called_once() args = mock_copy.call_args[0] - assert f"{TEST_BUCKET}/{build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX)}" in args - assert f"{TEST_BUCKET}/{build_archive_descriptor_key(_ASSEMBLY_DIR, _RELEASE_TAG, _KEY_PREFIX)}" in args + assert f"{TEST_BUCKET / build_descriptor_key(_ASSEMBLY_DIR, _KEY_PREFIX)}" in args + assert f"{TEST_BUCKET / build_archive_descriptor_key(_ASSEMBLY_DIR, _RELEASE_TAG, _KEY_PREFIX)}" in args @pytest.mark.s3 @@ -264,7 +287,7 @@ def test_archive_descriptor_dry_run(mock_s3_with_descriptor: tuple[botocore.clie mock_copy.assert_not_called() -@pytest.mark.s3 -def test_archive_descriptor_missing_returns_false(mock_s3: botocore.client.BaseClient) -> None: +@pytest.mark.usefixtures("mock_s3") +def test_archive_descriptor_missing_returns_false() -> None: """Returns False when no descriptor exists at the live key.""" assert archive_descriptor(_ASSEMBLY_DIR, TEST_BUCKET, _KEY_PREFIX, _RELEASE_TAG) is False diff --git a/tests/ncbi_ftp/test_notebooks.py b/tests/ncbi_ftp/test_notebooks.py index d905089b..652feeaf 100644 --- a/tests/ncbi_ftp/test_notebooks.py +++ b/tests/ncbi_ftp/test_notebooks.py @@ -2,26 +2,25 @@ import ast import json -from pathlib import Path +from pathlib import Path, PurePosixPath import pytest from cdm_data_loaders.ncbi_ftp.assembly import FTP_HOST -from cdm_data_loaders.ncbi_ftp.manifest import ( # noqa: F401 +from cdm_data_loaders.ncbi_ftp.manifest import ( AssemblyRecord, compute_diff, download_assembly_summary, - filter_by_prefix_range, - parse_assembly_summary, - write_diff_summary, - write_removed_manifest, - write_transfer_manifest, write_updated_manifest, ) from cdm_data_loaders.ncbi_ftp.promote import ( DEFAULT_LAKEHOUSE_KEY_PREFIX, promote_from_s3, ) +from cdm_data_loaders.pipelines.ncbi_ftp_download import ( + DEFAULT_STAGING_KEY_PREFIX, + download_and_stage, +) from cdm_data_loaders.utils.s3 import split_s3_path NOTEBOOKS_DIR = Path(__file__).resolve().parents[2] / "notebooks" @@ -61,7 +60,8 @@ def test_notebook_syntax(notebook: str) -> None: def test_manifest_notebook_imports() -> None: """All manifest notebook imports are verified at module load time above.""" - assert isinstance(FTP_HOST, str) and FTP_HOST + assert isinstance(FTP_HOST, str) + assert FTP_HOST assert AssemblyRecord is not None assert callable(download_assembly_summary) assert callable(compute_diff) @@ -71,16 +71,11 @@ def test_manifest_notebook_imports() -> None: def test_promote_notebook_imports() -> None: """All promote notebook imports are verified at module load time above.""" assert callable(promote_from_s3) - assert isinstance(DEFAULT_LAKEHOUSE_KEY_PREFIX, str) + assert isinstance(DEFAULT_LAKEHOUSE_KEY_PREFIX, PurePosixPath) assert callable(split_s3_path) def test_download_notebook_imports() -> None: """All download notebook imports resolve without error.""" - from cdm_data_loaders.pipelines.ncbi_ftp_download import ( - DEFAULT_STAGING_KEY_PREFIX, - download_and_stage, - ) - assert callable(download_and_stage) - assert isinstance(DEFAULT_STAGING_KEY_PREFIX, str) + assert isinstance(DEFAULT_STAGING_KEY_PREFIX, PurePosixPath) diff --git a/tests/ncbi_ftp/test_promote.py b/tests/ncbi_ftp/test_promote.py index 34bc82e7..09c5ddd0 100644 --- a/tests/ncbi_ftp/test_promote.py +++ b/tests/ncbi_ftp/test_promote.py @@ -2,8 +2,9 @@ import hashlib import logging -from pathlib import Path -from typing import Any +from http import HTTPStatus +from pathlib import Path, PurePosixPath +from typing import Protocol, cast from unittest.mock import patch import botocore.client @@ -19,68 +20,73 @@ _trim_manifest, promote_from_s3, ) -from tests.ncbi_ftp.conftest import TEST_BUCKET +from tests.ncbi_ftp.conftest import ACC_PATH_215, ACC_PATH_845, TEST_BUCKET # Promotion test constants -_STAGE_PREFIX = "staging/run1/" +_STAGE_PREFIX: PurePosixPath = PurePosixPath("staging") / "run1" # Assembly 1 -_ACC1 = "GCF_000001215.4" -_DIR1 = "GCF_000001215.4_Release_6" -_STG1 = f"{_STAGE_PREFIX}raw_data/GCF/000/001/215/{_DIR1}/" -_LKH1 = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{_DIR1}/" +_ACC1: str = "GCF_000001215.4" +_STG1: PurePosixPath = _STAGE_PREFIX / "raw_data" / ACC_PATH_215 +_LKH1: PurePosixPath = DEFAULT_LAKEHOUSE_KEY_PREFIX / "raw_data" / ACC_PATH_215 # Assembly 2 -_ACC2 = "GCF_000005845.2" -_DIR2 = "GCF_000005845.2_ASM584v2" -_STG2 = f"{_STAGE_PREFIX}raw_data/GCF/000/005/845/{_DIR2}/" -_LKH2 = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/005/845/{_DIR2}/" +_ACC2: str = "GCF_000005845.2" +_STG2: PurePosixPath = _STAGE_PREFIX / "raw_data" / ACC_PATH_845 +_LKH2: PurePosixPath = DEFAULT_LAKEHOUSE_KEY_PREFIX / "raw_data" / ACC_PATH_845 + + +class DownloadFileClient(Protocol): + """Protocol for dynamic BaseClient with download_file function.""" + + def download_file(self, Bucket: str, Key: str, Filename: str, **kw: object) -> None: # noqa: N803 + """Download a file from an S3 store.""" def _stage( s3: botocore.client.BaseClient, - staging_base: str, - files: dict[str, bytes], + staging_base: PurePosixPath, + files: dict[PurePosixPath, bytes], *, with_md5: bool = True, with_crc64: bool = False, -) -> list[str]: +) -> list[PurePosixPath]: """Stage files at *staging_base*, optionally adding .md5 / .crc64nvme sidecars. Returns list of all staged keys (data files only, not sidecars). """ - keys = [] + keys: list[PurePosixPath] = [] for fname, content in files.items(): - key = f"{staging_base}{fname}" - s3.put_object(Bucket=TEST_BUCKET, Key=key, Body=content) + key: PurePosixPath = staging_base / fname + s3.put_object(Bucket=str(TEST_BUCKET), Key=str(key), Body=content) keys.append(key) if with_md5: s3.put_object( - Bucket=TEST_BUCKET, - Key=f"{key}.md5", + Bucket=str(TEST_BUCKET), + Key=str(key.with_name(f"{key.name}.md5")), Body=hashlib.md5(content).hexdigest().encode(), # noqa: S324 ) if with_crc64: - s3.put_object(Bucket=TEST_BUCKET, Key=f"{key}.crc64nvme", Body=b"fake-crc") + s3.put_object(Bucket=str(TEST_BUCKET), Key=str(key.with_name(f"{key.name}.crc64nvme")), Body=b"fake-crc") return keys -def _stage_files(s3_client: botocore.client.BaseClient, prefix: str) -> None: +def _stage_files(s3_client: botocore.client.BaseClient, prefix: PurePosixPath) -> None: """Upload sample staged files to mock S3.""" for key in [ - f"{prefix}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz", - f"{prefix}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz.md5", - f"{prefix}download_report.json", + prefix / "raw_data" / ACC_PATH_215 / "GCF_000001215.4_genomic.fna.gz", + prefix / "raw_data" / ACC_PATH_215 / "GCF_000001215.4_genomic.fna.gz.md5", + prefix / "download_report.json", ]: - body = b"md5hash123" if key.endswith(".md5") else b"data" - s3_client.put_object(Bucket=TEST_BUCKET, Key=key, Body=body) + body = b"md5hash123" if key.match("**.md5") else b"data" + s3_client.put_object(Bucket=str(TEST_BUCKET), Key=str(key), Body=body) @pytest.mark.s3 def test_promote_dry_run_no_writes(mock_s3_client_no_checksum: botocore.client.BaseClient) -> None: """Verify dry_run does not write any objects.""" - prefix = "staging/run1/" + prefix = PurePosixPath("staging/run1") _stage_files(mock_s3_client_no_checksum, prefix) report = promote_from_s3( @@ -89,22 +95,25 @@ def test_promote_dry_run_no_writes(mock_s3_client_no_checksum: botocore.client.B assert report["promoted"] == 1 assert report["dry_run"] is True - final_key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz" - assert mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=final_key).get("KeyCount", 0) == 0 + final_key = DEFAULT_LAKEHOUSE_KEY_PREFIX / "raw_data" / ACC_PATH_215 / "GCF_000001215.4_genomic.fna.gz" + assert ( + mock_s3_client_no_checksum.list_objects_v2(Bucket=str(TEST_BUCKET), Prefix=str(final_key)).get("KeyCount", 0) + == 0 + ) @pytest.mark.s3 def test_promote_with_metadata(mock_s3_client_no_checksum: botocore.client.BaseClient) -> None: """Objects are promoted with MD5 metadata; download_report.json is skipped.""" - prefix = "staging/run1/" + prefix = PurePosixPath("staging/run1") _stage_files(mock_s3_client_no_checksum, prefix) report = promote_from_s3(staging_key_prefix=prefix, staging_bucket=TEST_BUCKET, lakehouse_bucket=TEST_BUCKET) assert report["promoted"] == 1 # only .fna.gz, not download_report.json assert report["failed"] == 0 - final_key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz" - resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=final_key) + final_key = DEFAULT_LAKEHOUSE_KEY_PREFIX / "raw_data" / ACC_PATH_215 / "GCF_000001215.4_genomic.fna.gz" + resp = mock_s3_client_no_checksum.head_object(Bucket=str(TEST_BUCKET), Key=str(final_key)) assert resp["Metadata"].get("md5") == "md5hash123" @@ -137,10 +146,12 @@ def test_trim_manifest( expected_absent: list[str], ) -> None: """Promoted accessions are removed; others remain (partial) or the manifest empties (all).""" - manifest_key = "manifests/transfer_manifest.txt" - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=manifest_key, Body=manifest_body.encode()) + manifest_key = PurePosixPath("manifests/transfer_manifest.txt") + mock_s3_client_no_checksum.put_object(Bucket=str(TEST_BUCKET), Key=str(manifest_key), Body=manifest_body.encode()) _trim_manifest(manifest_key, TEST_BUCKET, promoted_set) - remaining = mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=manifest_key)["Body"].read().decode() + remaining = ( + mock_s3_client_no_checksum.get_object(Bucket=str(TEST_BUCKET), Key=str(manifest_key))["Body"].read().decode() + ) for acc in expected_present: assert acc in remaining for acc in expected_absent: @@ -150,17 +161,18 @@ def test_trim_manifest( # Helpers for _archive_assemblies tests -def _mock_list_matching_objects(path: str) -> list[dict[str, Any]]: - bucket = "some-bucket" - prefix = "some/prefix/" - if path == f"{bucket}/{prefix}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6": +def _mock_list_matching_objects(path: str) -> list[dict[str, str]]: + bucket = PurePosixPath("some-bucket") + prefix = PurePosixPath("some/prefix") + p = PurePosixPath(path) + if p == bucket / prefix / "raw_data" / ACC_PATH_215: return [ - {"Key": f"{prefix}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz"}, - {"Key": f"{prefix}raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_protein.faa.gz"}, + {"Key": f"{prefix / 'raw_data' / ACC_PATH_215 / 'GCF_000001215.4_genomic.fna.gz'}"}, + {"Key": f"{prefix / 'raw_data' / ACC_PATH_215 / 'GCF_000001215.4_protein.faa.gz'}"}, ] - if path == f"{bucket}/{prefix}raw_data/GCF/000/005/845/GCF_000005845.2_ASM584v2": + if p == bucket / prefix / "raw_data" / ACC_PATH_845: return [ - {"Key": f"{prefix}raw_data/GCF/000/005/845/GCF_000005845.2_ASM584v2/GCF_000005845.2_genomic.fna.gz"}, + {"Key": f"{prefix / 'raw_data' / ACC_PATH_845 / 'GCF_000005845.2_genomic.fna.gz'}"}, ] return [] @@ -170,23 +182,30 @@ def _mock_list_matching_objects(path: str) -> list[dict[str, Any]]: [ pytest.param( "GCF_012345678.90_Some_description", - "some/prefix/", - "some/prefix/raw_data/GCF/012/345/678/GCF_012345678.90_Some_description", + PurePosixPath("some") / "prefix", + PurePosixPath("some") + / "prefix" + / "raw_data" + / "GCF" + / "012" + / "345" + / "678" + / "GCF_012345678.90_Some_description", id="standard", ), pytest.param( - "GCF_000001215.4_Release_6", - "another/prefix/", - "another/prefix/raw_data/GCF/000/001/215/GCF_000001215.4_Release_6", + "GCF_000001215.4_Release_6_plus_ISO1_MT", + PurePosixPath("another") / "prefix", + PurePosixPath("another") / "prefix" / "raw_data" / ACC_PATH_215, id="standard-2", ), - pytest.param("INVALID_ACCESSION", "prefix/", None, id="invalid-format"), + pytest.param("INVALID_ACCESSION", PurePosixPath("prefix"), None, id="invalid-format"), ], ) def test_get_accession_path_prefix( accession: str, - prefix: str, - expected: str | None, + prefix: PurePosixPath, + expected: PurePosixPath | None, ) -> None: """get_accession_path_prefix returns correct path for valid accessions, None for invalid.""" result = _get_accession_path_prefix(accession, prefix) @@ -197,19 +216,33 @@ def test_get_accession_path_prefix( ("accession", "bucket", "prefix", "release_tag", "archive_reason", "expected"), [ pytest.param( - "GCF_000001215.4_Release_6", + "GCF_000001215.4_Release_6_plus_ISO1_MT", "some-bucket", - "some/prefix/", + PurePosixPath("some/prefix"), "2024-01", "test-reason", [ ( - "some/prefix/raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz", - "some/prefix/archive/2024-01/test-reason/raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_genomic.fna.gz", + PurePosixPath("some") / "prefix" / "raw_data" / ACC_PATH_215 / "GCF_000001215.4_genomic.fna.gz", + PurePosixPath("some") + / "prefix" + / "archive" + / "2024-01" + / "test-reason" + / "raw_data" + / ACC_PATH_215 + / "GCF_000001215.4_genomic.fna.gz", ), ( - "some/prefix/raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_protein.faa.gz", - "some/prefix/archive/2024-01/test-reason/raw_data/GCF/000/001/215/GCF_000001215.4_Release_6/GCF_000001215.4_protein.faa.gz", + PurePosixPath("some") / "prefix" / "raw_data" / ACC_PATH_215 / "GCF_000001215.4_protein.faa.gz", + PurePosixPath("some") + / "prefix" + / "archive" + / "2024-01" + / "test-reason" + / "raw_data" + / ACC_PATH_215 + / "GCF_000001215.4_protein.faa.gz", ), ], id="standard", @@ -217,13 +250,20 @@ def test_get_accession_path_prefix( pytest.param( "GCF_000005845.2_ASM584v2", "some-bucket", - "some/prefix/", + PurePosixPath("some/prefix"), "2024-01", "test-reason", [ ( - "some/prefix/raw_data/GCF/000/005/845/GCF_000005845.2_ASM584v2/GCF_000005845.2_genomic.fna.gz", - "some/prefix/archive/2024-01/test-reason/raw_data/GCF/000/005/845/GCF_000005845.2_ASM584v2/GCF_000005845.2_genomic.fna.gz", + PurePosixPath("some") / "prefix" / "raw_data" / ACC_PATH_845 / "GCF_000005845.2_genomic.fna.gz", + PurePosixPath("some") + / "prefix" + / "archive" + / "2024-01" + / "test-reason" + / "raw_data" + / ACC_PATH_845 + / "GCF_000005845.2_genomic.fna.gz", ), ], id="standard-2", @@ -231,7 +271,7 @@ def test_get_accession_path_prefix( pytest.param( "GCF_000001405.39_GRCh38.p14", "some-bucket", - "some/prefix/", + PurePosixPath("some") / "prefix", "2024-01", "test-reason", [], @@ -240,7 +280,7 @@ def test_get_accession_path_prefix( pytest.param( "INVALID_ACCESSION", "some-bucket", - "some/prefix/", + PurePosixPath("some") / "prefix", "2024-01", "test-reason", [], @@ -250,8 +290,8 @@ def test_get_accession_path_prefix( ) def test_get_source_dest_pairs_for_accession( # noqa: PLR0913 accession: str, - bucket: str, - prefix: str, + bucket: PurePosixPath, + prefix: PurePosixPath, release_tag: str, archive_reason: str, expected: list[tuple[str, str]], @@ -272,14 +312,18 @@ def test_get_source_dest_pairs_for_accession( # noqa: PLR0913 ("key_pairs", "log_count", "expected", "info_log_strings"), [ pytest.param( - [("source1", "dest1"), ("source2", "dest2")], + [(PurePosixPath("source1"), PurePosixPath("dest1")), (PurePosixPath("source2"), PurePosixPath("dest2"))], 0, 2, ["[dry-run] would archive: source1 -> dest1", "[dry-run] would archive: source2 -> dest2"], id="standard", ), pytest.param( - [("source1", "dest1"), ("source2", "dest2"), ("source3", "dest3")], + [ + (PurePosixPath("source1"), PurePosixPath("dest1")), + (PurePosixPath("source2"), PurePosixPath("dest2")), + (PurePosixPath("source3"), PurePosixPath("dest3")), + ], 9, 12, ["[dry-run] would archive: source1 -> dest1"], @@ -295,7 +339,7 @@ def test_get_source_dest_pairs_for_accession( # noqa: PLR0913 ], ) def test_dry_run_output( - key_pairs: list[tuple[str, str]], + key_pairs: list[tuple[PurePosixPath, PurePosixPath]], log_count: int, expected: int, info_log_strings: list[str], @@ -313,7 +357,7 @@ def test_dry_run_output( ("key_pairs", "bucket", "delete_source", "expected", "existing_objects"), [ pytest.param( - [("source1", "dest1"), ("source2", "dest2")], + [(PurePosixPath("source1"), PurePosixPath("dest1")), (PurePosixPath("source2"), PurePosixPath("dest2"))], TEST_BUCKET, False, 2, @@ -321,7 +365,7 @@ def test_dry_run_output( id="keep-source", ), pytest.param( - [("source1", "dest1"), ("source2", "dest2")], + [(PurePosixPath("source1"), PurePosixPath("dest1")), (PurePosixPath("source2"), PurePosixPath("dest2"))], TEST_BUCKET, True, 2, @@ -341,18 +385,18 @@ def test_dry_run_output( @pytest.mark.s3 def test_archive_objects( # noqa: PLR0913 mock_s3_client_no_checksum: botocore.client.BaseClient, - key_pairs: list[tuple[str, str]], - bucket: str, + key_pairs: list[tuple[PurePosixPath, PurePosixPath]], + bucket: PurePosixPath, delete_source: bool, expected: int, existing_objects: list[str], ) -> None: """_archive_assemblies copies source to dest for each pair, and deletes source if delete_source=True.""" for source, _ in key_pairs: - mock_s3_client_no_checksum.put_object(Bucket=bucket, Key=source, Body=b"data") + mock_s3_client_no_checksum.put_object(Bucket=str(bucket), Key=str(source), Body=b"data") assert _archive_objects(key_pairs, bucket, delete_source=delete_source) == expected - assert mock_s3_client_no_checksum.list_objects_v2(Bucket=bucket).get("KeyCount", 0) == len(existing_objects) - for obj in mock_s3_client_no_checksum.list_objects_v2(Bucket=bucket).get("Contents", []): + assert mock_s3_client_no_checksum.list_objects_v2(Bucket=str(bucket)).get("KeyCount", 0) == len(existing_objects) + for obj in mock_s3_client_no_checksum.list_objects_v2(Bucket=str(bucket)).get("Contents", []): assert obj["Key"] in existing_objects, f"Unexpected object in bucket: {obj['Key']}" @@ -360,15 +404,15 @@ def test_archive_objects( # noqa: PLR0913 def test_archive_assemblies_removed(mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path) -> None: """Removed accessions are archived and originals deleted.""" accession = "GCF_000005845.2" - key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/005/845/{accession}_ASM584v2/{accession}_genomic.fna.gz" - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"data") + key = DEFAULT_LAKEHOUSE_KEY_PREFIX / "raw_data" / ACC_PATH_845 / f"{accession}_genomic.fna.gz" + mock_s3_client_no_checksum.put_object(Bucket=str(TEST_BUCKET), Key=str(key), Body=b"data") manifest = tmp_path / "removed.txt" manifest.write_text(f"{accession}\n") assert ( _archive_assemblies( - str(manifest), + manifest, lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="replaced_or_suppressed", @@ -376,13 +420,21 @@ def test_archive_assemblies_removed(mock_s3_client_no_checksum: botocore.client. ) == 1 ) - assert mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=key).get("KeyCount", 0) == 0 + assert mock_s3_client_no_checksum.list_objects_v2(Bucket=str(TEST_BUCKET), Prefix=str(key)).get("KeyCount", 0) == 0 archive_key = ( - f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/replaced_or_suppressed/" - f"raw_data/GCF/000/005/845/{accession}_ASM584v2/{accession}_genomic.fna.gz" + DEFAULT_LAKEHOUSE_KEY_PREFIX + / "archive" + / "2024-01" + / "replaced_or_suppressed" + / "raw_data" + / ACC_PATH_845 + / f"{accession}_genomic.fna.gz" + ) + assert ( + mock_s3_client_no_checksum.list_objects_v2(Bucket=str(TEST_BUCKET), Prefix=str(archive_key)).get("KeyCount", 0) + == 1 ) - assert mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_key).get("KeyCount", 0) == 1 @pytest.mark.s3 @@ -391,16 +443,15 @@ def test_archive_assemblies_updated_no_delete( ) -> None: """Updated accessions are archived but originals remain.""" accession = "GCF_000001215.4" - asm_dir = f"{accession}_Release_6" - key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"original-data") + key = DEFAULT_LAKEHOUSE_KEY_PREFIX / "raw_data" / ACC_PATH_215 / f"{accession}_genomic.fna.gz" + mock_s3_client_no_checksum.put_object(Bucket=str(TEST_BUCKET), Key=str(key), Body=b"original-data") manifest = tmp_path / "updated.txt" manifest.write_text(f"{accession}\n") assert ( _archive_assemblies( - str(manifest), + manifest, lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-06", archive_reason="updated", @@ -408,11 +459,19 @@ def test_archive_assemblies_updated_no_delete( ) == 1 ) - assert mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=key).get("KeyCount", 0) == 1 + assert mock_s3_client_no_checksum.list_objects_v2(Bucket=str(TEST_BUCKET), Prefix=str(key)).get("KeyCount", 0) == 1 - archive_key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-06/updated/raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" - resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=archive_key) - assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + archive_key = ( + DEFAULT_LAKEHOUSE_KEY_PREFIX + / "archive" + / "2024-06" + / "updated" + / "raw_data" + / ACC_PATH_215 + / f"{accession}_genomic.fna.gz" + ) + resp = mock_s3_client_no_checksum.head_object(Bucket=str(TEST_BUCKET), Key=str(archive_key)) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == HTTPStatus.OK @pytest.mark.s3 @@ -421,36 +480,57 @@ def test_archive_assemblies_multiple_releases_no_collision( ) -> None: """Archiving the same accession in different releases creates distinct folders.""" accession = "GCF_000001215.4" - asm_dir = f"{accession}_Release_6" - key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"v1-data") + key = DEFAULT_LAKEHOUSE_KEY_PREFIX / "raw_data" / ACC_PATH_215 / f"{accession}_genomic.fna.gz" + mock_s3_client_no_checksum.put_object(Bucket=str(TEST_BUCKET), Key=str(key), Body=b"v1-data") manifest = tmp_path / "updated.txt" manifest.write_text(f"{accession}\n") - _archive_assemblies(str(manifest), lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated") - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"v2-data") - _archive_assemblies(str(manifest), lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-06", archive_reason="updated") - - archive_key_1 = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/updated/raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" - archive_key_2 = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-06/updated/raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" - assert mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=archive_key_1)["Body"].read() == b"v1-data" - assert mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=archive_key_2)["Body"].read() == b"v2-data" + _archive_assemblies(manifest, lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated") + mock_s3_client_no_checksum.put_object(Bucket=str(TEST_BUCKET), Key=str(key), Body=b"v2-data") + _archive_assemblies(manifest, lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-06", archive_reason="updated") + + archive_key_1 = ( + DEFAULT_LAKEHOUSE_KEY_PREFIX + / "archive" + / "2024-01" + / "updated" + / "raw_data" + / ACC_PATH_215 + / f"{accession}_genomic.fna.gz" + ) + archive_key_2 = ( + DEFAULT_LAKEHOUSE_KEY_PREFIX + / "archive" + / "2024-06" + / "updated" + / "raw_data" + / ACC_PATH_215 + / f"{accession}_genomic.fna.gz" + ) + assert ( + mock_s3_client_no_checksum.get_object(Bucket=str(TEST_BUCKET), Key=str(archive_key_1))["Body"].read() + == b"v1-data" + ) + assert ( + mock_s3_client_no_checksum.get_object(Bucket=str(TEST_BUCKET), Key=str(archive_key_2))["Body"].read() + == b"v2-data" + ) @pytest.mark.s3 def test_archive_assemblies_dry_run(mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path) -> None: """dry_run does not copy or delete anything.""" accession = "GCF_000005845.2" - key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/005/845/{accession}_ASM584v2/{accession}_genomic.fna.gz" - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"data") + key = DEFAULT_LAKEHOUSE_KEY_PREFIX / "raw_data" / ACC_PATH_845 / f"{accession}_genomic.fna.gz" + mock_s3_client_no_checksum.put_object(Bucket=str(TEST_BUCKET), Key=str(key), Body=b"data") manifest = tmp_path / "removed.txt" manifest.write_text(f"{accession}\n") assert ( _archive_assemblies( - str(manifest), + manifest, lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="replaced_or_suppressed", @@ -459,20 +539,26 @@ def test_archive_assemblies_dry_run(mock_s3_client_no_checksum: botocore.client. ) == 1 ) - assert mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=key).get("KeyCount", 0) == 1 + assert mock_s3_client_no_checksum.list_objects_v2(Bucket=str(TEST_BUCKET), Prefix=str(key)).get("KeyCount", 0) == 1 - archive_prefix = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/" - assert mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_prefix).get("KeyCount", 0) == 0 + archive_prefix = DEFAULT_LAKEHOUSE_KEY_PREFIX / "archive" / "2024-01" + assert ( + mock_s3_client_no_checksum.list_objects_v2(Bucket=str(TEST_BUCKET), Prefix=str(archive_prefix)).get( + "KeyCount", 0 + ) + == 0 + ) @pytest.mark.s3 def test_archive_assemblies_no_objects_skips( - mock_s3_client_no_checksum: botocore.client.BaseClient, tmp_path: Path + mock_s3_client_no_checksum: botocore.client.BaseClient, # noqa: ARG001 + tmp_path: Path, ) -> None: """Accessions with no existing S3 objects are silently skipped.""" manifest = tmp_path / "updated.txt" manifest.write_text("GCF_000001215.4\n") - assert _archive_assemblies(str(manifest), lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01") == 0 + assert _archive_assemblies(manifest, lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01") == 0 @pytest.mark.s3 @@ -481,17 +567,27 @@ def test_archive_assemblies_unknown_release_fallback( ) -> None: """ncbi_release=None falls back to 'unknown' in the archive path.""" accession = "GCF_000001215.4" - asm_dir = f"{accession}_Release_6" - key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"data") + key = DEFAULT_LAKEHOUSE_KEY_PREFIX / "raw_data" / ACC_PATH_215 / f"{accession}_genomic.fna.gz" + mock_s3_client_no_checksum.put_object(Bucket=str(TEST_BUCKET), Key=str(key), Body=b"data") manifest = tmp_path / "updated.txt" manifest.write_text(f"{accession}\n") - assert _archive_assemblies(str(manifest), lakehouse_bucket=TEST_BUCKET, ncbi_release=None) == 1 + assert _archive_assemblies(manifest, lakehouse_bucket=TEST_BUCKET, ncbi_release=None) == 1 - archive_key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/unknown/unknown/raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" - assert mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_key).get("KeyCount", 0) == 1 + archive_key = ( + DEFAULT_LAKEHOUSE_KEY_PREFIX + / "archive" + / "unknown" + / "unknown" + / "raw_data" + / ACC_PATH_215 + / f"{accession}_genomic.fna.gz" + ) + assert ( + mock_s3_client_no_checksum.list_objects_v2(Bucket=str(TEST_BUCKET), Prefix=str(archive_key)).get("KeyCount", 0) + == 1 + ) # Concurrent / multi-file archive (new behaviour) @@ -503,8 +599,7 @@ def test_archive_assemblies_multi_file_all_copied( ) -> None: """All files for an accession are copied concurrently — none missed.""" accession = "GCF_000001215.4" - asm_dir = f"{accession}_Release_6" - base = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/" + base = DEFAULT_LAKEHOUSE_KEY_PREFIX / "raw_data" / ACC_PATH_215 file_names = [ f"{accession}_genomic.fna.gz", f"{accession}_protein.faa.gz", @@ -513,13 +608,13 @@ def test_archive_assemblies_multi_file_all_copied( f"{accession}_assembly_stats.txt", ] for fname in file_names: - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{base}{fname}", Body=fname.encode()) + mock_s3_client_no_checksum.put_object(Bucket=str(TEST_BUCKET), Key=str(base / fname), Body=fname.encode()) manifest = tmp_path / "updated.txt" manifest.write_text(f"{accession}\n") archived = _archive_assemblies( - str(manifest), + manifest, lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated", @@ -527,10 +622,10 @@ def test_archive_assemblies_multi_file_all_copied( ) assert archived == len(file_names) - archive_base = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/updated/raw_data/GCF/000/001/215/{asm_dir}/" + archive_base = DEFAULT_LAKEHOUSE_KEY_PREFIX / "archive" / "2024-01" / "updated" / "raw_data" / ACC_PATH_215 for fname in file_names: - resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{archive_base}{fname}") - assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + resp = mock_s3_client_no_checksum.head_object(Bucket=str(TEST_BUCKET), Key=str(archive_base / fname)) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == HTTPStatus.OK @pytest.mark.s3 @@ -539,29 +634,28 @@ def test_archive_assemblies_multi_file_content_preserved( ) -> None: """Archive copies preserve byte-for-byte content of each file.""" accession = "GCF_000001215.4" - asm_dir = f"{accession}_Release_6" - base = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/" + base = DEFAULT_LAKEHOUSE_KEY_PREFIX / "raw_data" / ACC_PATH_215 files = { f"{accession}_genomic.fna.gz": b"\x1f\x8bGENOMIC", f"{accession}_protein.faa.gz": b"\x1f\x8bPROTEIN", } for fname, body in files.items(): - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{base}{fname}", Body=body) + mock_s3_client_no_checksum.put_object(Bucket=str(TEST_BUCKET), Key=str(base / fname), Body=body) manifest = tmp_path / "updated.txt" manifest.write_text(f"{accession}\n") _archive_assemblies( - str(manifest), + manifest, lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated", delete_source=False, ) - archive_base = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/updated/raw_data/GCF/000/001/215/{asm_dir}/" + archive_base = DEFAULT_LAKEHOUSE_KEY_PREFIX / "archive" / "2024-01" / "updated" / "raw_data" / ACC_PATH_215 for fname, original_body in files.items(): - obj = mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=f"{archive_base}{fname}") + obj = mock_s3_client_no_checksum.get_object(Bucket=str(TEST_BUCKET), Key=str(archive_base / fname)) assert obj["Body"].read() == original_body, f"Content mismatch for {fname}" @@ -571,21 +665,20 @@ def test_archive_assemblies_multi_file_delete_all( ) -> None: """Batch delete removes ALL source files when delete_source=True.""" accession = "GCF_000005845.2" - asm_dir = f"{accession}_ASM584v2" - base = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/005/845/{asm_dir}/" + base = DEFAULT_LAKEHOUSE_KEY_PREFIX / "raw_data" / ACC_PATH_845 file_names = [ f"{accession}_genomic.fna.gz", f"{accession}_protein.faa.gz", f"{accession}_assembly_report.txt", ] for fname in file_names: - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{base}{fname}", Body=b"data") + mock_s3_client_no_checksum.put_object(Bucket=str(TEST_BUCKET), Key=str(base / fname), Body=b"data") manifest = tmp_path / "removed.txt" manifest.write_text(f"{accession}\n") archived = _archive_assemblies( - str(manifest), + manifest, lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-03", archive_reason="replaced_or_suppressed", @@ -595,15 +688,15 @@ def test_archive_assemblies_multi_file_delete_all( assert archived == len(file_names) # All sources deleted for fname in file_names: - result = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=f"{base}{fname}") + result = mock_s3_client_no_checksum.list_objects_v2(Bucket=str(TEST_BUCKET), Prefix=str(base / fname)) assert result.get("KeyCount", 0) == 0, f"Source not deleted: {fname}" # All archives present archive_base = ( - f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-03/replaced_or_suppressed/raw_data/GCF/000/005/845/{asm_dir}/" + DEFAULT_LAKEHOUSE_KEY_PREFIX / "archive" / "2024-03" / "replaced_or_suppressed" / "raw_data" / ACC_PATH_845 ) for fname in file_names: - resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{archive_base}{fname}") - assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + resp = mock_s3_client_no_checksum.head_object(Bucket=str(TEST_BUCKET), Key=str(archive_base / fname)) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == HTTPStatus.OK # Partial-archive idempotency @@ -619,23 +712,24 @@ def test_archive_assemblies_partial_already_archived_overwritten( The second run should archive both file_a (overwrite) and file_b. """ accession = "GCF_000001215.4" - asm_dir = f"{accession}_Release_6" - base = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/" - archive_base = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/updated/raw_data/GCF/000/001/215/{asm_dir}/" + base = DEFAULT_LAKEHOUSE_KEY_PREFIX / "raw_data" / ACC_PATH_215 + archive_base = DEFAULT_LAKEHOUSE_KEY_PREFIX / "archive" / "2024-01" / "updated" / "raw_data" / ACC_PATH_215 file_a = f"{accession}_genomic.fna.gz" file_b = f"{accession}_protein.faa.gz" - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{base}{file_a}", Body=b"new-genomic") - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{base}{file_b}", Body=b"new-protein") + mock_s3_client_no_checksum.put_object(Bucket=str(TEST_BUCKET), Key=str(base / file_a), Body=b"new-genomic") + mock_s3_client_no_checksum.put_object(Bucket=str(TEST_BUCKET), Key=str(base / file_b), Body=b"new-protein") # Simulate partial prior run: file_a already archived with stale content - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{archive_base}{file_a}", Body=b"stale-genomic") + mock_s3_client_no_checksum.put_object( + Bucket=str(TEST_BUCKET), Key=f"{archive_base / file_a}", Body=b"stale-genomic" + ) manifest = tmp_path / "updated.txt" manifest.write_text(f"{accession}\n") archived = _archive_assemblies( - str(manifest), + manifest, lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated", @@ -644,10 +738,10 @@ def test_archive_assemblies_partial_already_archived_overwritten( assert archived == 2 # noqa: PLR2004 # file_a should now have the current content (overwritten) - obj_a = mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=f"{archive_base}{file_a}") + obj_a = mock_s3_client_no_checksum.get_object(Bucket=str(TEST_BUCKET), Key=str(archive_base / file_a)) assert obj_a["Body"].read() == b"new-genomic", "Re-run should overwrite stale archive" # file_b should now be archived - obj_b = mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=f"{archive_base}{file_b}") + obj_b = mock_s3_client_no_checksum.get_object(Bucket=str(TEST_BUCKET), Key=str(archive_base / file_b)) assert obj_b["Body"].read() == b"new-protein" @@ -662,10 +756,9 @@ def test_archive_assemblies_partial_delete_resumes( (file_a is gone), archives both, and deletes both. """ accession = "GCF_000005845.2" - asm_dir = f"{accession}_ASM584v2" - base = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/005/845/{asm_dir}/" + base = DEFAULT_LAKEHOUSE_KEY_PREFIX / "raw_data" / ACC_PATH_845 archive_base = ( - f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-03/replaced_or_suppressed/raw_data/GCF/000/005/845/{asm_dir}/" + DEFAULT_LAKEHOUSE_KEY_PREFIX / "archive" / "2024-03" / "replaced_or_suppressed" / "raw_data" / ACC_PATH_845 ) file_b = f"{accession}_protein.faa.gz" @@ -673,19 +766,19 @@ def test_archive_assemblies_partial_delete_resumes( # file_a already gone (deleted in first partial run) # file_b present at source (not yet deleted from first partial run) - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{base}{file_b}", Body=b"protein") + mock_s3_client_no_checksum.put_object(Bucket=str(TEST_BUCKET), Key=str(base / file_b), Body=b"protein") # file_c present at source (not touched at all) - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{base}{file_c}", Body=b"report") + mock_s3_client_no_checksum.put_object(Bucket=str(TEST_BUCKET), Key=str(base / file_c), Body=b"report") # file_a already at archive destination mock_s3_client_no_checksum.put_object( - Bucket=TEST_BUCKET, Key=f"{archive_base}{accession}_genomic.fna.gz", Body=b"genomic" + Bucket=str(TEST_BUCKET), Key=f"{archive_base / accession}_genomic.fna.gz", Body=b"genomic" ) manifest = tmp_path / "removed.txt" manifest.write_text(f"{accession}\n") archived = _archive_assemblies( - str(manifest), + manifest, lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-03", archive_reason="replaced_or_suppressed", @@ -696,11 +789,13 @@ def test_archive_assemblies_partial_delete_resumes( assert archived == 2 # noqa: PLR2004 # Both now gone from source for fname in (file_b, file_c): - result = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=f"{base}{fname}") + result = mock_s3_client_no_checksum.list_objects_v2(Bucket=str(TEST_BUCKET), Prefix=str(base / fname)) assert result.get("KeyCount", 0) == 0, f"Expected {fname} deleted" # file_a archive still intact (not touched by re-run) - resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{archive_base}{accession}_genomic.fna.gz") - assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + resp = mock_s3_client_no_checksum.head_object( + Bucket=str(TEST_BUCKET), Key=f"{archive_base / accession}_genomic.fna.gz" + ) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == HTTPStatus.OK @pytest.mark.s3 @@ -709,28 +804,27 @@ def test_archive_assemblies_idempotent_updated_reruns_cleanly( ) -> None: """Running updated archive twice on the same data produces the same result.""" accession = "GCF_000001215.4" - asm_dir = f"{accession}_Release_6" - base = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/" + base = DEFAULT_LAKEHOUSE_KEY_PREFIX / "raw_data" / ACC_PATH_215 file_names = [f"{accession}_genomic.fna.gz", f"{accession}_protein.faa.gz"] for fname in file_names: - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{base}{fname}", Body=b"content") + mock_s3_client_no_checksum.put_object(Bucket=str(TEST_BUCKET), Key=str(base / fname), Body=b"content") manifest = tmp_path / "updated.txt" manifest.write_text(f"{accession}\n") archived_1 = _archive_assemblies( - str(manifest), lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated" + manifest, lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated" ) archived_2 = _archive_assemblies( - str(manifest), lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated" + manifest, lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated" ) assert archived_1 == len(file_names) assert archived_2 == len(file_names) # Sources still present after both runs (delete_source=False) for fname in file_names: - resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{base}{fname}") - assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + resp = mock_s3_client_no_checksum.head_object(Bucket=str(TEST_BUCKET), Key=str(base / fname)) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == HTTPStatus.OK @pytest.mark.s3 @@ -743,20 +837,29 @@ def test_archive_assemblies_multi_accession_manifest( ("GCF_000005845.2", "GCF_000005845.2_ASM584v2", "GCF/000/005/845"), ] for accession, asm_dir, path in accessions: - key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/{path}/{asm_dir}/{accession}_genomic.fna.gz" - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"data") + key = DEFAULT_LAKEHOUSE_KEY_PREFIX / "raw_data" / path / asm_dir / f"{accession}_genomic.fna.gz" + mock_s3_client_no_checksum.put_object(Bucket=str(TEST_BUCKET), Key=str(key), Body=b"data") manifest = tmp_path / "updated.txt" manifest.write_text("\n".join(acc for acc, _, _ in accessions) + "\n") archived = _archive_assemblies( - str(manifest), lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated" + manifest, lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated" ) assert archived == len(accessions) for accession, asm_dir, path in accessions: - archive_key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/2024-01/updated/raw_data/{path}/{asm_dir}/{accession}_genomic.fna.gz" - result = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_key) + archive_key = ( + DEFAULT_LAKEHOUSE_KEY_PREFIX + / "archive" + / "2024-01" + / "updated" + / "raw_data" + / path + / asm_dir + / f"{accession}_genomic.fna.gz" + ) + result = mock_s3_client_no_checksum.list_objects_v2(Bucket=str(TEST_BUCKET), Prefix=str(archive_key)) assert result.get("KeyCount", 0) == 1, f"Archive missing for {accession}" @@ -766,17 +869,16 @@ def test_archive_assemblies_dry_run_multi_file( ) -> None: """dry_run with multiple files per accession makes no copies and no deletes.""" accession = "GCF_000005845.2" - asm_dir = f"{accession}_ASM584v2" - base = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/005/845/{asm_dir}/" + base = DEFAULT_LAKEHOUSE_KEY_PREFIX / "raw_data" / ACC_PATH_845 file_names = [f"{accession}_genomic.fna.gz", f"{accession}_protein.faa.gz", f"{accession}_rna.fna.gz"] for fname in file_names: - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{base}{fname}", Body=b"data") + mock_s3_client_no_checksum.put_object(Bucket=str(TEST_BUCKET), Key=str(base / fname), Body=b"data") manifest = tmp_path / "removed.txt" manifest.write_text(f"{accession}\n") archived = _archive_assemblies( - str(manifest), + manifest, lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="replaced_or_suppressed", @@ -787,13 +889,13 @@ def test_archive_assemblies_dry_run_multi_file( # Reported count matches assert archived == len(file_names) # No actual archive keys created - archive_prefix = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}archive/" - result = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=archive_prefix) + archive_prefix = DEFAULT_LAKEHOUSE_KEY_PREFIX / "archive" + result = mock_s3_client_no_checksum.list_objects_v2(Bucket=str(TEST_BUCKET), Prefix=str(archive_prefix)) assert result.get("KeyCount", 0) == 0 # Sources untouched for fname in file_names: - resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{base}{fname}") - assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + resp = mock_s3_client_no_checksum.head_object(Bucket=str(TEST_BUCKET), Key=str(base / fname)) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == HTTPStatus.OK @pytest.mark.s3 @@ -802,15 +904,14 @@ def test_archive_assemblies_invalid_accession_skipped( ) -> None: """Malformed accession lines are skipped; valid ones still archived.""" accession = "GCF_000001215.4" - asm_dir = f"{accession}_Release_6" - key = f"{DEFAULT_LAKEHOUSE_KEY_PREFIX}raw_data/GCF/000/001/215/{asm_dir}/{accession}_genomic.fna.gz" - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=key, Body=b"data") + key = DEFAULT_LAKEHOUSE_KEY_PREFIX / "raw_data" / ACC_PATH_215 / f"{accession}_genomic.fna.gz" + mock_s3_client_no_checksum.put_object(Bucket=str(TEST_BUCKET), Key=str(key), Body=b"data") manifest = tmp_path / "mixed.txt" manifest.write_text("NOT_AN_ACCESSION\n\n \n" + f"{accession}\n") archived = _archive_assemblies( - str(manifest), lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated" + manifest, lakehouse_bucket=TEST_BUCKET, ncbi_release="2024-01", archive_reason="updated" ) assert archived == 1 @@ -830,7 +931,7 @@ def test_promote_multi_file_all_land_at_final_path( f"{_ACC1}_assembly_report.txt", f"{_ACC1}_assembly_stats.txt", ] - _stage(mock_s3_client_no_checksum, _STG1, {f: f.encode() for f in file_names}) + _stage(mock_s3_client_no_checksum, _STG1, {PurePosixPath(f): f.encode() for f in file_names}) report = promote_from_s3( staging_key_prefix=_STAGE_PREFIX, @@ -841,8 +942,8 @@ def test_promote_multi_file_all_land_at_final_path( assert report["promoted"] == len(file_names) assert report["failed"] == 0 for fname in file_names: - resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{_LKH1}{fname}") - assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + resp = mock_s3_client_no_checksum.head_object(Bucket=str(TEST_BUCKET), Key=f"{_LKH1 / fname}") + assert resp["ResponseMetadata"]["HTTPStatusCode"] == HTTPStatus.OK @pytest.mark.s3 @@ -851,9 +952,9 @@ def test_promote_multi_file_content_preserved( ) -> None: """Content at the final key is byte-identical to the staged content.""" files = { - f"{_ACC1}_genomic.fna.gz": b"\x1f\x8bGENOMIC", - f"{_ACC1}_protein.faa.gz": b"\x1f\x8bPROTEIN", - f"{_ACC1}_rna.fna.gz": b"\x1f\x8bRNA", + PurePosixPath(f"{_ACC1}_genomic.fna.gz"): b"\x1f\x8bGENOMIC", + PurePosixPath(f"{_ACC1}_protein.faa.gz"): b"\x1f\x8bPROTEIN", + PurePosixPath(f"{_ACC1}_rna.fna.gz"): b"\x1f\x8bRNA", } _stage(mock_s3_client_no_checksum, _STG1, files) @@ -864,7 +965,7 @@ def test_promote_multi_file_content_preserved( ) for fname, expected in files.items(): - obj = mock_s3_client_no_checksum.get_object(Bucket=TEST_BUCKET, Key=f"{_LKH1}{fname}") + obj = mock_s3_client_no_checksum.get_object(Bucket=str(TEST_BUCKET), Key=f"{_LKH1 / fname}") assert obj["Body"].read() == expected, f"Content mismatch for {fname}" @@ -874,7 +975,7 @@ def test_promote_md5_metadata_set_from_sidecar( ) -> None: """MD5 metadata on the promoted object matches the .md5 sidecar value.""" content = b"\x1f\x8bGENOMIC" - fname = f"{_ACC1}_genomic.fna.gz" + fname = PurePosixPath(f"{_ACC1}_genomic.fna.gz") _stage(mock_s3_client_no_checksum, _STG1, {fname: content}, with_md5=True) expected_md5 = hashlib.md5(content).hexdigest() # noqa: S324 @@ -884,7 +985,7 @@ def test_promote_md5_metadata_set_from_sidecar( lakehouse_bucket=TEST_BUCKET, ) - resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{_LKH1}{fname}") + resp = mock_s3_client_no_checksum.head_object(Bucket=str(TEST_BUCKET), Key=str(_LKH1 / fname)) assert resp["Metadata"].get("md5") == expected_md5 @@ -893,7 +994,7 @@ def test_promote_no_sidecar_no_md5_metadata( mock_s3_client_no_checksum: botocore.client.BaseClient, ) -> None: """A file staged without a .md5 sidecar is promoted but carries no md5 metadata.""" - fname = f"{_ACC1}_genomic.fna.gz" + fname = PurePosixPath(f"{_ACC1}_genomic.fna.gz") _stage(mock_s3_client_no_checksum, _STG1, {fname: b"data"}, with_md5=False) promote_from_s3( @@ -902,7 +1003,7 @@ def test_promote_no_sidecar_no_md5_metadata( lakehouse_bucket=TEST_BUCKET, ) - resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{_LKH1}{fname}") + resp = mock_s3_client_no_checksum.head_object(Bucket=str(TEST_BUCKET), Key=str(_LKH1 / fname)) assert resp["Metadata"].get("md5") is None @@ -912,8 +1013,8 @@ def test_promote_staging_data_files_deleted_after_promote( ) -> None: """Staged data files are deleted from staging after a fully successful assembly promote.""" files = { - f"{_ACC1}_genomic.fna.gz": b"genomic", - f"{_ACC1}_protein.faa.gz": b"protein", + PurePosixPath(f"{_ACC1}_genomic.fna.gz"): b"genomic", + PurePosixPath(f"{_ACC1}_protein.faa.gz"): b"protein", } staged_keys = _stage(mock_s3_client_no_checksum, _STG1, files) @@ -924,7 +1025,7 @@ def test_promote_staging_data_files_deleted_after_promote( ) for key in staged_keys: - result = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=key) + result = mock_s3_client_no_checksum.list_objects_v2(Bucket=str(TEST_BUCKET), Prefix=str(key)) assert result.get("KeyCount", 0) == 0, f"Staged data file not deleted: {key}" @@ -934,8 +1035,8 @@ def test_promote_md5_sidecars_deleted_after_promote( ) -> None: """Staged .md5 sidecar files are deleted from staging after a successful promote.""" files = { - f"{_ACC1}_genomic.fna.gz": b"genomic", - f"{_ACC1}_protein.faa.gz": b"protein", + PurePosixPath(f"{_ACC1}_genomic.fna.gz"): b"genomic", + PurePosixPath(f"{_ACC1}_protein.faa.gz"): b"protein", } staged_keys = _stage(mock_s3_client_no_checksum, _STG1, files, with_md5=True) @@ -947,7 +1048,7 @@ def test_promote_md5_sidecars_deleted_after_promote( for key in staged_keys: for sidecar_key in (f"{key}.md5", f"{key}.crc64nvme"): - result = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=sidecar_key) + result = mock_s3_client_no_checksum.list_objects_v2(Bucket=str(TEST_BUCKET), Prefix=str(sidecar_key)) assert result.get("KeyCount", 0) == 0, f"Sidecar not deleted: {sidecar_key}" @@ -956,7 +1057,7 @@ def test_promote_crc64nvme_sidecars_deleted_after_promote( mock_s3_client_no_checksum: botocore.client.BaseClient, ) -> None: """Staged .crc64nvme sidecar files are also batch-deleted after a successful promote.""" - fname = f"{_ACC1}_genomic.fna.gz" + fname = PurePosixPath(f"{_ACC1}_genomic.fna.gz") _stage(mock_s3_client_no_checksum, _STG1, {fname: b"data"}, with_md5=True, with_crc64=True) promote_from_s3( @@ -967,7 +1068,7 @@ def test_promote_crc64nvme_sidecars_deleted_after_promote( staged_key = f"{_STG1}{fname}" for sidecar_key in (f"{staged_key}.md5", f"{staged_key}.crc64nvme"): - result = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=sidecar_key) + result = mock_s3_client_no_checksum.list_objects_v2(Bucket=str(TEST_BUCKET), Prefix=str(sidecar_key)) assert result.get("KeyCount", 0) == 0, f"Sidecar not deleted: {sidecar_key}" @@ -980,22 +1081,23 @@ def test_promote_partial_failure_staging_not_cleaned( Preserving staging on partial failure lets an operator re-run without re-staging and without losing the partially-promoted state. """ - file_ok = f"{_ACC1}_genomic.fna.gz" - file_fail = f"{_ACC1}_protein.faa.gz" + file_ok = PurePosixPath(f"{_ACC1}_genomic.fna.gz") + file_fail = PurePosixPath(f"{_ACC1}_protein.faa.gz") _stage(mock_s3_client_no_checksum, _STG1, {file_ok: b"ok", file_fail: b"fail"}) - staged_ok = f"{_STG1}{file_ok}" - staged_fail = f"{_STG1}{file_fail}" + staged_ok = _STG1 / file_ok + staged_fail = _STG1 / file_fail # Make download_file raise for exactly the failing key - original_download = mock_s3_client_no_checksum.download_file + dynamic_client = cast("DownloadFileClient", mock_s3_client_no_checksum) + original_download = dynamic_client.download_file def _download_one_fail(Bucket: str, Key: str, Filename: str, **kw: object) -> None: # noqa: N803 - if Key == staged_fail: + if Key == f"{staged_fail}": msg = "simulated download failure" raise RuntimeError(msg) return original_download(Bucket=Bucket, Key=Key, Filename=Filename, **kw) - mock_s3_client_no_checksum.download_file = _download_one_fail + dynamic_client.download_file = _download_one_fail report = promote_from_s3( staging_key_prefix=_STAGE_PREFIX, @@ -1006,8 +1108,8 @@ def _download_one_fail(Bucket: str, Key: str, Filename: str, **kw: object) -> No assert report["failed"] == 1 # Staging files must still be present (cleanup skipped due to failure) for key in (staged_ok, staged_fail): - resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=key) - assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200, ( + resp = mock_s3_client_no_checksum.head_object(Bucket=str(TEST_BUCKET), Key=str(key)) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == HTTPStatus.OK, ( f"Expected staged file to survive partial failure: {key}" ) @@ -1016,20 +1118,25 @@ def _download_one_fail(Bucket: str, Key: str, Filename: str, **kw: object) -> No def test_promote_partial_failure_failed_count( mock_s3_client_no_checksum: botocore.client.BaseClient, ) -> None: - """report[\"failed\"] reflects the number of files that could not be promoted.""" - file_names = [f"{_ACC1}_genomic.fna.gz", f"{_ACC1}_protein.faa.gz", f"{_ACC1}_rna.fna.gz"] + r"""report[\"failed\"] reflects the number of files that could not be promoted.""" + file_names = [ + PurePosixPath(f"{_ACC1}_genomic.fna.gz"), + PurePosixPath(f"{_ACC1}_protein.faa.gz"), + PurePosixPath(f"{_ACC1}_rna.fna.gz"), + ] _stage(mock_s3_client_no_checksum, _STG1, dict.fromkeys(file_names, b"data")) - failing_key = f"{_STG1}{file_names[1]}" - original_download = mock_s3_client_no_checksum.download_file + failing_key = f"{_STG1 / file_names[1]}" + dynamic_client = cast("DownloadFileClient", mock_s3_client_no_checksum) + original_download = dynamic_client.download_file def _download_middle_fail(Bucket: str, Key: str, Filename: str, **kw: object) -> None: # noqa: N803 - if Key == failing_key: + if Key == str(failing_key): msg = "simulated failure" raise RuntimeError(msg) return original_download(Bucket=Bucket, Key=Key, Filename=Filename, **kw) - mock_s3_client_no_checksum.download_file = _download_middle_fail + dynamic_client.download_file = _download_middle_fail report = promote_from_s3( staging_key_prefix=_STAGE_PREFIX, @@ -1054,24 +1161,25 @@ def test_promote_two_assemblies_independent_cleanup( _stage( mock_s3_client_no_checksum, _STG1, - {f"{_ACC1}_genomic.fna.gz": b"g1", f"{_ACC1}_protein.faa.gz": b"p1"}, + {PurePosixPath(f"{_ACC1}_genomic.fna.gz"): b"g1", PurePosixPath(f"{_ACC1}_protein.faa.gz"): b"p1"}, ) # Assembly 2: two files, one will fail _stage( mock_s3_client_no_checksum, _STG2, - {f"{_ACC2}_genomic.fna.gz": b"g2", f"{_ACC2}_protein.faa.gz": b"p2"}, + {PurePosixPath(f"{_ACC2}_genomic.fna.gz"): b"g2", PurePosixPath(f"{_ACC2}_protein.faa.gz"): b"p2"}, ) - failing_key = f"{_STG2}{_ACC2}_protein.faa.gz" - original_download = mock_s3_client_no_checksum.download_file + failing_key = f"{_STG2 / _ACC2}_protein.faa.gz" + dynamic_client = cast("DownloadFileClient", mock_s3_client_no_checksum) + original_download = dynamic_client.download_file def _patched(Bucket: str, Key: str, Filename: str, **kw: object) -> None: # noqa: N803 - if Key == failing_key: + if Key == f"{failing_key}": msg = "simulated failure" raise RuntimeError(msg) return original_download(Bucket=Bucket, Key=Key, Filename=Filename, **kw) - mock_s3_client_no_checksum.download_file = _patched + dynamic_client.download_file = _patched report = promote_from_s3( staging_key_prefix=_STAGE_PREFIX, @@ -1083,13 +1191,13 @@ def _patched(Bucket: str, Key: str, Filename: str, **kw: object) -> None: # noq # Assembly 1 staging must be gone for fname in (f"{_ACC1}_genomic.fna.gz", f"{_ACC1}_protein.faa.gz"): - result = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=f"{_STG1}{fname}") + result = mock_s3_client_no_checksum.list_objects_v2(Bucket=str(TEST_BUCKET), Prefix=str(_STG1 / fname)) assert result.get("KeyCount", 0) == 0, f"Assembly 1 staging should be cleaned: {fname}" # Assembly 2 staging must remain (partial failure) for fname in (f"{_ACC2}_genomic.fna.gz", f"{_ACC2}_protein.faa.gz"): - resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{_STG2}{fname}") - assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200, ( + resp = mock_s3_client_no_checksum.head_object(Bucket=str(TEST_BUCKET), Key=str(_STG2 / fname)) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == HTTPStatus.OK, ( f"Assembly 2 staging must survive partial failure: {fname}" ) @@ -1099,8 +1207,8 @@ def test_promote_multi_assembly_all_succeed_all_cleaned( mock_s3_client_no_checksum: botocore.client.BaseClient, ) -> None: """Two assemblies both fully succeed → all staged files removed for both.""" - _stage(mock_s3_client_no_checksum, _STG1, {f"{_ACC1}_genomic.fna.gz": b"g1"}) - _stage(mock_s3_client_no_checksum, _STG2, {f"{_ACC2}_genomic.fna.gz": b"g2"}) + _stage(mock_s3_client_no_checksum, _STG1, {PurePosixPath(f"{_ACC1}_genomic.fna.gz"): b"g1"}) + _stage(mock_s3_client_no_checksum, _STG2, {PurePosixPath(f"{_ACC2}_genomic.fna.gz"): b"g2"}) report = promote_from_s3( staging_key_prefix=_STAGE_PREFIX, @@ -1115,10 +1223,10 @@ def test_promote_multi_assembly_all_succeed_all_cleaned( (_STG1, f"{_ACC1}_genomic.fna.gz", _LKH1), (_STG2, f"{_ACC2}_genomic.fna.gz", _LKH2), ): - result = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=f"{stg}{fname}") + result = mock_s3_client_no_checksum.list_objects_v2(Bucket=str(TEST_BUCKET), Prefix=str(stg / fname)) assert result.get("KeyCount", 0) == 0, f"Staging not cleaned: {fname}" - resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{lkh}{fname}") - assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + resp = mock_s3_client_no_checksum.head_object(Bucket=str(TEST_BUCKET), Key=str(lkh / fname)) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == HTTPStatus.OK @pytest.mark.s3 @@ -1127,7 +1235,7 @@ def test_promote_dry_run_multi_file_no_writes_no_cleanup( ) -> None: """dry_run with multiple files writes nothing to final path and does not delete staging.""" file_names = [f"{_ACC1}_genomic.fna.gz", f"{_ACC1}_protein.faa.gz", f"{_ACC1}_rna.fna.gz"] - staged_keys = _stage(mock_s3_client_no_checksum, _STG1, {f: f.encode() for f in file_names}) + staged_keys = _stage(mock_s3_client_no_checksum, _STG1, {PurePosixPath(f): f.encode() for f in file_names}) report = promote_from_s3( staging_key_prefix=_STAGE_PREFIX, @@ -1140,13 +1248,13 @@ def test_promote_dry_run_multi_file_no_writes_no_cleanup( assert report["dry_run"] is True # Final path must be empty - result = mock_s3_client_no_checksum.list_objects_v2(Bucket=TEST_BUCKET, Prefix=_LKH1) + result = mock_s3_client_no_checksum.list_objects_v2(Bucket=str(TEST_BUCKET), Prefix=str(_LKH1)) assert result.get("KeyCount", 0) == 0 # Staging keys must survive for key in staged_keys: - resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=key) - assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200, f"Staging deleted during dry-run: {key}" # noqa: PLR2004 + resp = mock_s3_client_no_checksum.head_object(Bucket=str(TEST_BUCKET), Key=str(key)) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == HTTPStatus.OK, f"Staging deleted during dry-run: {key}" @pytest.mark.s3 @@ -1155,9 +1263,13 @@ def test_promote_skips_non_raw_data_paths( ) -> None: """Files outside raw_data/ (e.g. download_report.json) are silently skipped.""" # Stage a real data file alongside non-promotable files - _stage(mock_s3_client_no_checksum, _STG1, {f"{_ACC1}_genomic.fna.gz": b"data"}) - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{_STAGE_PREFIX}download_report.json", Body=b"{}") - mock_s3_client_no_checksum.put_object(Bucket=TEST_BUCKET, Key=f"{_STAGE_PREFIX}logs/run.log", Body=b"logs") + _stage(mock_s3_client_no_checksum, _STG1, {PurePosixPath(f"{_ACC1}_genomic.fna.gz"): b"data"}) + mock_s3_client_no_checksum.put_object( + Bucket=str(TEST_BUCKET), Key=str(_STAGE_PREFIX / "download_report.json"), Body=b"{}" + ) + mock_s3_client_no_checksum.put_object( + Bucket=str(TEST_BUCKET), Key=str(_STAGE_PREFIX / "logs/run.log"), Body=b"logs" + ) report = promote_from_s3( staging_key_prefix=_STAGE_PREFIX, @@ -1174,7 +1286,7 @@ def test_promote_idempotent_second_run_on_empty_staging( mock_s3_client_no_checksum: botocore.client.BaseClient, ) -> None: """Second promote run after staging has been cleaned promotes 0 files without error.""" - _stage(mock_s3_client_no_checksum, _STG1, {f"{_ACC1}_genomic.fna.gz": b"data"}) + _stage(mock_s3_client_no_checksum, _STG1, {PurePosixPath(f"{_ACC1}_genomic.fna.gz"): b"data"}) report1 = promote_from_s3( staging_key_prefix=_STAGE_PREFIX, @@ -1192,8 +1304,8 @@ def test_promote_idempotent_second_run_on_empty_staging( assert report2["failed"] == 0 # Final key still present after second run - resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{_LKH1}{_ACC1}_genomic.fna.gz") - assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # noqa: PLR2004 + resp = mock_s3_client_no_checksum.head_object(Bucket=str(TEST_BUCKET), Key=f"{_LKH1 / _ACC1}_genomic.fna.gz") + assert resp["ResponseMetadata"]["HTTPStatusCode"] == HTTPStatus.OK @pytest.mark.s3 @@ -1202,9 +1314,9 @@ def test_promote_multi_file_md5_per_file( ) -> None: """Each promoted file carries the MD5 matching its own content, not another file's.""" files = { - f"{_ACC1}_genomic.fna.gz": b"GENOMIC_UNIQUE", - f"{_ACC1}_protein.faa.gz": b"PROTEIN_UNIQUE", - f"{_ACC1}_rna.fna.gz": b"RNA_UNIQUE", + PurePosixPath(f"{_ACC1}_genomic.fna.gz"): b"GENOMIC_UNIQUE", + PurePosixPath(f"{_ACC1}_protein.faa.gz"): b"PROTEIN_UNIQUE", + PurePosixPath(f"{_ACC1}_rna.fna.gz"): b"RNA_UNIQUE", } _stage(mock_s3_client_no_checksum, _STG1, files, with_md5=True) @@ -1216,5 +1328,5 @@ def test_promote_multi_file_md5_per_file( for fname, content in files.items(): expected_md5 = hashlib.md5(content).hexdigest() # noqa: S324 - resp = mock_s3_client_no_checksum.head_object(Bucket=TEST_BUCKET, Key=f"{_LKH1}{fname}") + resp = mock_s3_client_no_checksum.head_object(Bucket=str(TEST_BUCKET), Key=str(_LKH1 / fname)) assert resp["Metadata"].get("md5") == expected_md5, f"Wrong MD5 on {fname}" diff --git a/tests/pipelines/test_ncbi_ftp_download.py b/tests/pipelines/test_ncbi_ftp_download.py index d51aa0ef..675ac53a 100644 --- a/tests/pipelines/test_ncbi_ftp_download.py +++ b/tests/pipelines/test_ncbi_ftp_download.py @@ -1,7 +1,9 @@ """Tests for pipelines.ncbi_ftp_download — settings, batch orchestration, CLI.""" import json -from pathlib import Path +from collections.abc import Generator +from pathlib import Path, PurePosixPath +from typing import Any, cast from unittest.mock import MagicMock, patch import boto3 @@ -9,6 +11,7 @@ from moto import mock_aws from pydantic import ValidationError +import cdm_data_loaders.utils.s3 as s3_mod from cdm_data_loaders.ncbi_ftp.assembly import FTP_HOST from cdm_data_loaders.pipelines.cts_defaults import INPUT_MOUNT, OUTPUT_MOUNT from cdm_data_loaders.pipelines.ncbi_ftp_download import ( @@ -37,9 +40,10 @@ _EXPECTED_ATTEMPTED = 2 -def make_settings(**kwargs: str | int) -> DownloadSettings: +def make_settings(**kwargs: str | int | bool | Path | PurePosixPath) -> DownloadSettings: """Generate a validated DownloadSettings object.""" - return DownloadSettings(_cli_parse_args=[], **kwargs) + settings_ctor = cast("Any", DownloadSettings) + return settings_ctor(_cli_parse_args=[], **kwargs) # Settings defaults @@ -51,12 +55,12 @@ class TestDownloadSettingsDefaults: def test_manifest_default(self) -> None: """Verify default manifest path uses INPUT_MOUNT.""" s = make_settings() - assert s.manifest == f"{INPUT_MOUNT}/transfer_manifest.txt" + assert s.manifest == Path(INPUT_MOUNT) / "transfer_manifest.txt" def test_output_dir_default(self) -> None: """Verify default output_dir uses OUTPUT_MOUNT.""" s = make_settings() - assert s.output_dir == OUTPUT_MOUNT + assert s.output_dir == Path(OUTPUT_MOUNT) def test_threads_default(self) -> None: """Verify default threads is 4.""" @@ -83,14 +87,14 @@ class TestDownloadSettingsAllParams: def test_all_params(self) -> None: """Verify all parameters are correctly set when provided.""" s = make_settings( - manifest="/data/my_manifest.txt", - output_dir="/data/output", + manifest=Path("/") / "data" / "my_manifest.txt", + output_dir=Path("/") / "data" / "output", threads=_CUSTOM_THREADS, ftp_host="ftp.example.com", limit=_CUSTOM_LIMIT, ) - assert s.manifest == "/data/my_manifest.txt" - assert s.output_dir == "/data/output" + assert s.manifest == Path("/") / "data" / "my_manifest.txt" + assert s.output_dir == Path("/") / "data" / "output" assert s.threads == _CUSTOM_THREADS assert s.ftp_host == "ftp.example.com" assert s.limit == _CUSTOM_LIMIT @@ -104,13 +108,13 @@ class TestDownloadSettingsAliases: def test_manifest_alias_m(self) -> None: """Verify 'm' alias resolves to manifest.""" - s = make_settings(m="/data/m.txt") - assert s.manifest == "/data/m.txt" + s = make_settings(m=Path("/") / "data" / "m.txt") + assert s.manifest == Path("/") / "data" / "m.txt" def test_output_dir_alias(self) -> None: """Verify 'output_dir' / 'output-dir' alias resolves to output_dir.""" - s = make_settings(output_dir="/data/o") - assert s.output_dir == "/data/o" + s = make_settings(output_dir=Path("/") / "data" / "o") + assert s.output_dir == Path("/") / "data" / "o" def test_threads_alias_t(self) -> None: """Verify 't' alias resolves to threads.""" @@ -162,7 +166,7 @@ class TestDownloadBatch: """Test download_batch with mocked internals.""" @pytest.fixture(autouse=True) - def _mock_ftp_pool(self) -> None: + def _mock_ftp_pool(self) -> Generator[None]: """Prevent real FTP connections from the ThreadLocalFTP pool.""" mock_pool = MagicMock() with patch("cdm_data_loaders.pipelines.ncbi_ftp_download.ThreadLocalFTP", return_value=mock_pool): @@ -181,8 +185,8 @@ def test_reads_manifest_and_calls_download(self, tmp_path: Path) -> None: mock_stats = {"accession": "test", "files_downloaded": 3} with patch("cdm_data_loaders.pipelines.ncbi_ftp_download.download_assembly_to_local", return_value=mock_stats): report = download_batch( - manifest_path=str(manifest), - output_dir=str(output), + manifest_path=manifest, + output_dir=output, threads=1, ftp_host="ftp.example.com", ) @@ -204,8 +208,8 @@ def test_limit_truncates(self, tmp_path: Path) -> None: mock_stats = {"accession": "test", "files_downloaded": 1} with patch("cdm_data_loaders.pipelines.ncbi_ftp_download.download_assembly_to_local", return_value=mock_stats): report = download_batch( - manifest_path=str(manifest), - output_dir=str(output), + manifest_path=manifest, + output_dir=output, threads=1, limit=1, ) @@ -220,7 +224,7 @@ def test_writes_report_json(self, tmp_path: Path) -> None: mock_stats = {"accession": "GCF_000001215.4", "files_downloaded": 5} with patch("cdm_data_loaders.pipelines.ncbi_ftp_download.download_assembly_to_local", return_value=mock_stats): - download_batch(manifest_path=str(manifest), output_dir=str(output), threads=1) + download_batch(manifest_path=manifest, output_dir=output, threads=1) report_file = output / "download_report.json" assert report_file.exists() @@ -239,7 +243,7 @@ def test_handles_download_failure(self, tmp_path: Path) -> None: "cdm_data_loaders.pipelines.ncbi_ftp_download.download_assembly_to_local", side_effect=RuntimeError("connection lost"), ): - report = download_batch(manifest_path=str(manifest), output_dir=str(output), threads=1) + report = download_batch(manifest_path=manifest, output_dir=output, threads=1) assert report["failed"] == 1 assert report["succeeded"] == 0 @@ -251,11 +255,11 @@ def test_handles_download_failure(self, tmp_path: Path) -> None: "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/\n" "/genomes/all/GCF/000/001/405/GCF_000001405.40_GRCh38.p14/\n" ) -_TEST_BUCKET = "test-bucket" -_STAGING_PREFIX = "staging/run1/" +_TEST_BUCKET = PurePosixPath("test-bucket") +_STAGING_PREFIX = PurePosixPath("staging") / "run1" -def _make_moto_s3(monkeypatch: pytest.MonkeyPatch): +def _make_moto_s3(monkeypatch: pytest.MonkeyPatch): # noqa: ANN202 """Return a moto-backed S3 client with the test bucket created.""" # Remove any real endpoint/credential env vars so moto intercepts all HTTP calls. monkeypatch.setenv("AWS_ACCESS_KEY_ID", "testing") @@ -265,7 +269,7 @@ def _make_moto_s3(monkeypatch: pytest.MonkeyPatch): monkeypatch.delenv("AWS_ENDPOINT_URL_S3", raising=False) boto3.DEFAULT_SESSION = None client = boto3.client("s3", region_name="us-east-1") - client.create_bucket(Bucket=_TEST_BUCKET) + client.create_bucket(Bucket=f"{_TEST_BUCKET}") return client @@ -275,15 +279,15 @@ def _make_moto_s3(monkeypatch: pytest.MonkeyPatch): @pytest.mark.parametrize( ("manifest_s3_key", "use_local"), [ - pytest.param("staging/input/transfer_manifest.txt", False, id="s3_source"), + pytest.param(Path("staging") / "input" / "transfer_manifest.txt", False, id="s3_source"), pytest.param(None, True, id="local_source"), ], ) @mock_aws def test_download_and_stage_manifest_source( tmp_path: Path, - manifest_s3_key: str | None, - use_local: bool, + manifest_s3_key: PurePosixPath | None, + use_local: bool, # noqa: ARG001 monkeypatch: pytest.MonkeyPatch, ) -> None: """Assembly paths from the manifest are processed regardless of source (S3 or local).""" @@ -292,19 +296,17 @@ def test_download_and_stage_manifest_source( manifest_local: Path | None = None if manifest_s3_key is not None: - s3.put_object(Bucket=_TEST_BUCKET, Key=manifest_s3_key, Body=_MANIFEST_CONTENT.encode()) + s3.put_object(Bucket=str(_TEST_BUCKET), Key=str(manifest_s3_key), Body=_MANIFEST_CONTENT.encode()) else: manifest_local = tmp_path / "manifest.txt" manifest_local.write_text(_MANIFEST_CONTENT) - called_paths: list[str] = [] + called_paths: list[Path] = [] - def _fake_download(path, output_dir, **kwargs): # noqa: ARG001 + def _fake_download(path: Path, output_dir: Path, **kwargs: object) -> dict[str, str | int]: # noqa: ARG001 called_paths.append(path) return _MOCK_STATS - import cdm_data_loaders.utils.s3 as s3_mod - with ( patch.object(s3_mod, "get_s3_client", return_value=s3), patch.object(s3_mod, "_s3_client", s3), @@ -324,7 +326,7 @@ def _fake_download(path, output_dir, **kwargs): # noqa: ARG001 threads=1, ) - expected_paths = [l for l in _MANIFEST_CONTENT.splitlines() if l.strip()] + expected_paths: list[Path] = [Path(line) for line in _MANIFEST_CONTENT.splitlines() if line.strip()] assert sorted(called_paths) == sorted(expected_paths) reset_s3_client() @@ -336,17 +338,17 @@ def _fake_download(path, output_dir, **kwargs): # noqa: ARG001 @pytest.mark.parametrize( ("s3_key", "local_path", "should_raise"), [ - pytest.param("s3/key", "local/path", True, id="both_provided_raises"), + pytest.param(PurePosixPath("s3") / "key", Path("local") / "path", True, id="both_provided_raises"), pytest.param(None, None, True, id="neither_provided_raises"), - pytest.param("s3/key", None, False, id="s3_only_ok"), - pytest.param(None, "local/path", False, id="local_only_ok"), + pytest.param(PurePosixPath("s3") / "key", None, False, id="s3_only_ok"), + pytest.param(None, Path("local") / "path", False, id="local_only_ok"), ], ) @mock_aws def test_download_and_stage_exactly_one_source_required( tmp_path: Path, - s3_key: str | None, - local_path: str | None, + s3_key: PurePosixPath | None, + local_path: Path | None, should_raise: bool, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -365,14 +367,12 @@ def test_download_and_stage_exactly_one_source_required( s3 = _make_moto_s3(monkeypatch) # For s3_only: seed the object; for local_only: create the file if s3_key is not None: - s3.put_object(Bucket=_TEST_BUCKET, Key=s3_key, Body=_MANIFEST_CONTENT.encode()) + s3.put_object(Bucket=str(_TEST_BUCKET), Key=str(s3_key), Body=_MANIFEST_CONTENT.encode()) if local_path is not None: real_local = tmp_path / "manifest.txt" real_local.write_text(_MANIFEST_CONTENT) local_path = real_local - import cdm_data_loaders.utils.s3 as s3_mod - with ( patch.object(s3_mod, "get_s3_client", return_value=s3), patch.object(s3_mod, "_s3_client", s3), @@ -410,15 +410,13 @@ def test_download_and_stage_uploads_to_staging(tmp_path: Path, monkeypatch: pyte assembly_rel = "raw_data/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT" - def _fake_download(path, output_dir, **kwargs): # noqa: ARG001 + def _fake_download(path: Path, output_dir: Path, **kwargs: Path) -> dict[str, str | int]: # noqa: ARG001 asm_dir = Path(output_dir) / assembly_rel asm_dir.mkdir(parents=True) (asm_dir / "genomic.fna.gz").write_bytes(b"fasta_data") (asm_dir / "genomic.fna.gz.md5").write_bytes(b"abc123") return {**_MOCK_STATS, "files_downloaded": 2} - import cdm_data_loaders.utils.s3 as s3_mod - with ( patch.object(s3_mod, "get_s3_client", return_value=s3), patch.object(s3_mod, "_s3_client", s3), @@ -438,12 +436,16 @@ def _fake_download(path, output_dir, **kwargs): # noqa: ARG001 ) paginator = s3.get_paginator("list_objects_v2") - uploaded_keys = {obj["Key"] for page in paginator.paginate(Bucket=_TEST_BUCKET) for obj in page.get("Contents", [])} + uploaded_keys: set[PurePosixPath] = { + PurePosixPath(obj["Key"]) + for page in paginator.paginate(Bucket=str(_TEST_BUCKET)) + for obj in page.get("Contents", []) + } expected_keys = { - f"{_STAGING_PREFIX}{assembly_rel}/genomic.fna.gz", - f"{_STAGING_PREFIX}{assembly_rel}/genomic.fna.gz.md5", - f"{_STAGING_PREFIX}download_report.json", + _STAGING_PREFIX / assembly_rel / "genomic.fna.gz", + _STAGING_PREFIX / assembly_rel / "genomic.fna.gz.md5", + _STAGING_PREFIX / "download_report.json", } assert uploaded_keys == expected_keys assert report["staged_objects"] == len(expected_keys) @@ -463,14 +465,12 @@ def test_download_and_stage_dry_run_skips_upload(tmp_path: Path, monkeypatch: py manifest_local = tmp_path / "manifest.txt" manifest_local.write_text(_MANIFEST_CONTENT) - def _fake_download(path, output_dir, **kwargs): # noqa: ARG001 - asm_dir = Path(output_dir) / "raw_data/GCF/000/001/215/GCF_000001215.4" + def _fake_download(path: Path, output_dir: Path, **kwargs: object) -> dict[str, str | int]: # noqa: ARG001 + asm_dir = Path(output_dir) / "raw_data" / "GCF" / "000" / "001" / "215" / "GCF_000001215.4" asm_dir.mkdir(parents=True, exist_ok=True) (asm_dir / "genomic.fna.gz").write_bytes(b"fasta") return _MOCK_STATS - import cdm_data_loaders.utils.s3 as s3_mod - with ( patch.object(s3_mod, "get_s3_client", return_value=s3), patch.object(s3_mod, "_s3_client", s3), @@ -489,7 +489,7 @@ def _fake_download(path, output_dir, **kwargs): # noqa: ARG001 threads=1, ) - listed = s3.list_objects_v2(Bucket=_TEST_BUCKET) + listed = s3.list_objects_v2(Bucket=str(_TEST_BUCKET)) assert listed.get("KeyCount", 0) == 0 assert report["staged_objects"] == 0 assert report["dry_run"] is True @@ -516,8 +516,6 @@ def test_download_and_stage_limit_forwarded(tmp_path: Path, limit: int, monkeypa manifest_local = tmp_path / "manifest.txt" manifest_local.write_text(_MANIFEST_CONTENT) - import cdm_data_loaders.utils.s3 as s3_mod - with ( patch.object(s3_mod, "get_s3_client", return_value=s3), patch.object(s3_mod, "_s3_client", s3), @@ -555,8 +553,6 @@ def test_download_and_stage_report_shape(tmp_path: Path, monkeypatch: pytest.Mon manifest_local = tmp_path / "manifest.txt" manifest_local.write_text(_MANIFEST_CONTENT) - import cdm_data_loaders.utils.s3 as s3_mod - with ( patch.object(s3_mod, "get_s3_client", return_value=s3), patch.object(s3_mod, "_s3_client", s3), diff --git a/tests/utils/test_ftp_client.py b/tests/utils/test_ftp_client.py index 385fd329..a44e89a2 100644 --- a/tests/utils/test_ftp_client.py +++ b/tests/utils/test_ftp_client.py @@ -4,7 +4,7 @@ import time from collections.abc import Callable from ftplib import FTP, error_temp -from pathlib import Path +from pathlib import Path, PurePosixPath from unittest.mock import MagicMock, patch import pytest @@ -95,8 +95,8 @@ def fake_retrlines(_cmd: str, callback: Callable[[str], None]) -> None: callback(name) mock_ftp.retrlines.side_effect = fake_retrlines - result = ftp_list_dir(mock_ftp, "/some/path") - assert result == ["file1.txt", "file2.gz"] + result = ftp_list_dir(mock_ftp, PurePosixPath("/") / "some" / "path") + assert result == [PurePosixPath("file1.txt"), PurePosixPath("file2.gz")] mock_ftp.cwd.assert_called_once_with("/some/path") def test_retries_on_error_temp(self) -> None: @@ -112,8 +112,8 @@ def fake_retrlines(_cmd: str, callback: Callable[[str], None]) -> None: callback("file.txt") mock_ftp.retrlines.side_effect = fake_retrlines - result = ftp_list_dir(mock_ftp, "/path", retries=3) - assert result == ["file.txt"] + result = ftp_list_dir(mock_ftp, PurePosixPath("/path"), retries=3) + assert result == [PurePosixPath("file.txt")] assert call_count == _EXPECTED_RETRY_COUNT def test_raises_after_exhausted_retries(self) -> None: @@ -121,7 +121,7 @@ def test_raises_after_exhausted_retries(self) -> None: mock_ftp = MagicMock(spec=FTP) mock_ftp.retrlines.side_effect = error_temp(_ERR_421) # noqa: S321 with pytest.raises(error_temp): - ftp_list_dir(mock_ftp, "/path", retries=_EXPECTED_RETRY_COUNT) + ftp_list_dir(mock_ftp, PurePosixPath("/") / "path", retries=_EXPECTED_RETRY_COUNT) class TestFtpDownloadFile: @@ -136,7 +136,7 @@ def fake_retrbinary(_cmd: str, callback: Callable[[bytes], None]) -> None: mock_ftp.retrbinary.side_effect = fake_retrbinary local = tmp_path / "out.bin" - ftp_download_file(mock_ftp, "remote.bin", str(local)) + ftp_download_file(mock_ftp, PurePosixPath("remote.bin"), local) assert local.read_bytes() == b"file data" def test_retries_on_error_temp(self, tmp_path: Path) -> None: @@ -153,8 +153,8 @@ def fake_retrbinary(_cmd: str, callback: Callable[[bytes], None]) -> None: callback(b"ok") mock_ftp.retrbinary.side_effect = fake_retrbinary - local = str(tmp_path / "out.bin") - ftp_download_file(mock_ftp, "remote.bin", local, retries=3) + local = tmp_path / "out.bin" + ftp_download_file(mock_ftp, PurePosixPath("remote.bin"), local, retries=3) assert call_count == _EXPECTED_RETRY_COUNT @@ -170,7 +170,7 @@ def fake_retrlines(_cmd: str, callback: Callable[[str], None]) -> None: callback(line) mock_ftp.retrlines.side_effect = fake_retrlines - result = ftp_retrieve_text(mock_ftp, "remote.txt") + result = ftp_retrieve_text(mock_ftp, PurePosixPath("remote.txt")) assert result == "line1\nline2" From cbfb35040cad54ce7083a980808235cd14a515cc Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Tue, 16 Jun 2026 08:17:37 -0700 Subject: [PATCH 128/128] Updating vcrpy to version compatible with newest AOIHTTP --- pyproject.toml | 6 +- uv.lock | 552 +++++++++++++++++++++++++++++++++---------------- 2 files changed, 375 insertions(+), 183 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 66c4f20f..bece419f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ "click>=8.4.1", "defusedxml>=0.7.1", "delta-spark>=4.2.0", - "dlt[deltalake,duckdb,filesystem,parquet]>=1.27.2", + "dlt[deltalake,duckdb,filesystem,parquet,pyiceberg]>=1.27.2", "frictionless[aws]>=5.19.0", "frozendict>=2.4.7", "ipykernel>=7.2.0", @@ -34,6 +34,7 @@ uniref = "cdm_data_loaders.pipelines.uniref:cli" [dependency-groups] dev = [ "berdl-notebook-utils>=0.0.1", + "cdm-task-service-client>=0.2.1", "moto[s3]>=5.2.1", "pytest>=9.0.2", "pytest-asyncio>=1.3.0", @@ -209,7 +210,7 @@ S3_ENDPOINT_URL = "http://localhost:9000" AWS_CONFIG_FILE = "/dev/null" # ensure tests don't pick up creds from a real config file AWS_DEFAULT_REGION = "us-east-1" # default to use containerized S3 test store -AWS_ENDPOINT_URL = { value = "http://localhost:9000", skip_if_set = true } +AWS_ENDPOINT_URL = { value = "http://localhost:9000", skip_if_set = true } AWS_ENDPOINT_URL_S3 = { unset = true } AWS_ACCESS_KEY_ID = "test_access_key" AWS_SECRET_ACCESS_KEY = "test_access_secret" @@ -217,3 +218,4 @@ AWS_SECRET_ACCESS_KEY = "test_access_secret" [tool.uv.sources] cdm-schema = { git = "https://github.com/kbase/cdm-schema.git" } berdl-notebook-utils = { git = "https://github.com/BERDataLakehouse/spark_notebook.git", subdirectory = "notebook_utils" } +cdm-task-service-client = { git = "https://github.com/kbase/cdm-task-service-client.git" } diff --git a/uv.lock b/uv.lock index 31424242..25b4f0cb 100644 --- a/uv.lock +++ b/uv.lock @@ -41,7 +41,7 @@ wheels = [ [[package]] name = "aiohttp" -version = "3.13.5" +version = "3.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohappyeyeballs" }, @@ -52,59 +52,72 @@ dependencies = [ { name = "propcache" }, { name = "yarl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/77/9a/152096d4808df8e4268befa55fba462f440f14beab85e8ad9bf990516918/aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1", size = 7858271, upload-time = "2026-03-31T22:01:03.343Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/78/e9/d76bf503005709e390122d34e15256b88f7008e246c4bdbe915cd4f1adce/aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61", size = 742930, upload-time = "2026-03-31T21:58:13.155Z" }, - { url = "https://files.pythonhosted.org/packages/57/00/4b7b70223deaebd9bb85984d01a764b0d7bd6526fcdc73cca83bcbe7243e/aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832", size = 496927, upload-time = "2026-03-31T21:58:15.073Z" }, - { url = "https://files.pythonhosted.org/packages/9c/f5/0fb20fb49f8efdcdce6cd8127604ad2c503e754a8f139f5e02b01626523f/aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9", size = 497141, upload-time = "2026-03-31T21:58:17.009Z" }, - { url = "https://files.pythonhosted.org/packages/3b/86/b7c870053e36a94e8951b803cb5b909bfbc9b90ca941527f5fcafbf6b0fa/aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090", size = 1732476, upload-time = "2026-03-31T21:58:18.925Z" }, - { url = "https://files.pythonhosted.org/packages/b5/e5/4e161f84f98d80c03a238671b4136e6530453d65262867d989bbe78244d0/aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b", size = 1706507, upload-time = "2026-03-31T21:58:21.094Z" }, - { url = "https://files.pythonhosted.org/packages/d4/56/ea11a9f01518bd5a2a2fcee869d248c4b8a0cfa0bb13401574fa31adf4d4/aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a", size = 1773465, upload-time = "2026-03-31T21:58:23.159Z" }, - { url = "https://files.pythonhosted.org/packages/eb/40/333ca27fb74b0383f17c90570c748f7582501507307350a79d9f9f3c6eb1/aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8", size = 1873523, upload-time = "2026-03-31T21:58:25.59Z" }, - { url = "https://files.pythonhosted.org/packages/f0/d2/e2f77eef1acb7111405433c707dc735e63f67a56e176e72e9e7a2cd3f493/aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665", size = 1754113, upload-time = "2026-03-31T21:58:27.624Z" }, - { url = "https://files.pythonhosted.org/packages/fb/56/3f653d7f53c89669301ec9e42c95233e2a0c0a6dd051269e6e678db4fdb0/aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540", size = 1562351, upload-time = "2026-03-31T21:58:29.918Z" }, - { url = "https://files.pythonhosted.org/packages/ec/a6/9b3e91eb8ae791cce4ee736da02211c85c6f835f1bdfac0594a8a3b7018c/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb", size = 1693205, upload-time = "2026-03-31T21:58:32.214Z" }, - { url = "https://files.pythonhosted.org/packages/98/fc/bfb437a99a2fcebd6b6eaec609571954de2ed424f01c352f4b5504371dd3/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46", size = 1730618, upload-time = "2026-03-31T21:58:34.728Z" }, - { url = "https://files.pythonhosted.org/packages/e4/b6/c8534862126191a034f68153194c389addc285a0f1347d85096d349bbc15/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8", size = 1745185, upload-time = "2026-03-31T21:58:36.909Z" }, - { url = "https://files.pythonhosted.org/packages/0b/93/4ca8ee2ef5236e2707e0fd5fecb10ce214aee1ff4ab307af9c558bda3b37/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d", size = 1557311, upload-time = "2026-03-31T21:58:39.38Z" }, - { url = "https://files.pythonhosted.org/packages/57/ae/76177b15f18c5f5d094f19901d284025db28eccc5ae374d1d254181d33f4/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6", size = 1773147, upload-time = "2026-03-31T21:58:41.476Z" }, - { url = "https://files.pythonhosted.org/packages/01/a4/62f05a0a98d88af59d93b7fcac564e5f18f513cb7471696ac286db970d6a/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c", size = 1730356, upload-time = "2026-03-31T21:58:44.049Z" }, - { url = "https://files.pythonhosted.org/packages/e4/85/fc8601f59dfa8c9523808281f2da571f8b4699685f9809a228adcc90838d/aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc", size = 432637, upload-time = "2026-03-31T21:58:46.167Z" }, - { url = "https://files.pythonhosted.org/packages/c0/1b/ac685a8882896acf0f6b31d689e3792199cfe7aba37969fa91da63a7fa27/aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83", size = 458896, upload-time = "2026-03-31T21:58:48.119Z" }, - { url = "https://files.pythonhosted.org/packages/5d/ce/46572759afc859e867a5bc8ec3487315869013f59281ce61764f76d879de/aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c", size = 745721, upload-time = "2026-03-31T21:58:50.229Z" }, - { url = "https://files.pythonhosted.org/packages/13/fe/8a2efd7626dbe6049b2ef8ace18ffda8a4dfcbe1bcff3ac30c0c7575c20b/aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be", size = 497663, upload-time = "2026-03-31T21:58:52.232Z" }, - { url = "https://files.pythonhosted.org/packages/9b/91/cc8cc78a111826c54743d88651e1687008133c37e5ee615fee9b57990fac/aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25", size = 499094, upload-time = "2026-03-31T21:58:54.566Z" }, - { url = "https://files.pythonhosted.org/packages/0a/33/a8362cb15cf16a3af7e86ed11962d5cd7d59b449202dc576cdc731310bde/aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56", size = 1726701, upload-time = "2026-03-31T21:58:56.864Z" }, - { url = "https://files.pythonhosted.org/packages/45/0c/c091ac5c3a17114bd76cbf85d674650969ddf93387876cf67f754204bd77/aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2", size = 1683360, upload-time = "2026-03-31T21:58:59.072Z" }, - { url = "https://files.pythonhosted.org/packages/23/73/bcee1c2b79bc275e964d1446c55c54441a461938e70267c86afaae6fba27/aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a", size = 1773023, upload-time = "2026-03-31T21:59:01.776Z" }, - { url = "https://files.pythonhosted.org/packages/c7/ef/720e639df03004fee2d869f771799d8c23046dec47d5b81e396c7cda583a/aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be", size = 1853795, upload-time = "2026-03-31T21:59:04.568Z" }, - { url = "https://files.pythonhosted.org/packages/bd/c9/989f4034fb46841208de7aeeac2c6d8300745ab4f28c42f629ba77c2d916/aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b", size = 1730405, upload-time = "2026-03-31T21:59:07.221Z" }, - { url = "https://files.pythonhosted.org/packages/ce/75/ee1fd286ca7dc599d824b5651dad7b3be7ff8d9a7e7b3fe9820d9180f7db/aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94", size = 1558082, upload-time = "2026-03-31T21:59:09.484Z" }, - { url = "https://files.pythonhosted.org/packages/c3/20/1e9e6650dfc436340116b7aa89ff8cb2bbdf0abc11dfaceaad8f74273a10/aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d", size = 1692346, upload-time = "2026-03-31T21:59:12.068Z" }, - { url = "https://files.pythonhosted.org/packages/d8/40/8ebc6658d48ea630ac7903912fe0dd4e262f0e16825aa4c833c56c9f1f56/aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7", size = 1698891, upload-time = "2026-03-31T21:59:14.552Z" }, - { url = "https://files.pythonhosted.org/packages/d8/78/ea0ae5ec8ba7a5c10bdd6e318f1ba5e76fcde17db8275188772afc7917a4/aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772", size = 1742113, upload-time = "2026-03-31T21:59:17.068Z" }, - { url = "https://files.pythonhosted.org/packages/8a/66/9d308ed71e3f2491be1acb8769d96c6f0c47d92099f3bc9119cada27b357/aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5", size = 1553088, upload-time = "2026-03-31T21:59:19.541Z" }, - { url = "https://files.pythonhosted.org/packages/da/a6/6cc25ed8dfc6e00c90f5c6d126a98e2cf28957ad06fa1036bd34b6f24a2c/aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1", size = 1757976, upload-time = "2026-03-31T21:59:22.311Z" }, - { url = "https://files.pythonhosted.org/packages/c1/2b/cce5b0ffe0de99c83e5e36d8f828e4161e415660a9f3e58339d07cce3006/aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b", size = 1712444, upload-time = "2026-03-31T21:59:24.635Z" }, - { url = "https://files.pythonhosted.org/packages/6c/cf/9e1795b4160c58d29421eafd1a69c6ce351e2f7c8d3c6b7e4ca44aea1a5b/aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3", size = 438128, upload-time = "2026-03-31T21:59:27.291Z" }, - { url = "https://files.pythonhosted.org/packages/22/4d/eaedff67fc805aeba4ba746aec891b4b24cebb1a7d078084b6300f79d063/aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162", size = 464029, upload-time = "2026-03-31T21:59:29.429Z" }, - { url = "https://files.pythonhosted.org/packages/79/11/c27d9332ee20d68dd164dc12a6ecdef2e2e35ecc97ed6cf0d2442844624b/aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a", size = 778758, upload-time = "2026-03-31T21:59:31.547Z" }, - { url = "https://files.pythonhosted.org/packages/04/fb/377aead2e0a3ba5f09b7624f702a964bdf4f08b5b6728a9799830c80041e/aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254", size = 512883, upload-time = "2026-03-31T21:59:34.098Z" }, - { url = "https://files.pythonhosted.org/packages/bb/a6/aa109a33671f7a5d3bd78b46da9d852797c5e665bfda7d6b373f56bff2ec/aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36", size = 516668, upload-time = "2026-03-31T21:59:36.497Z" }, - { url = "https://files.pythonhosted.org/packages/79/b3/ca078f9f2fa9563c36fb8ef89053ea2bb146d6f792c5104574d49d8acb63/aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f", size = 1883461, upload-time = "2026-03-31T21:59:38.723Z" }, - { url = "https://files.pythonhosted.org/packages/b7/e3/a7ad633ca1ca497b852233a3cce6906a56c3225fb6d9217b5e5e60b7419d/aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800", size = 1747661, upload-time = "2026-03-31T21:59:41.187Z" }, - { url = "https://files.pythonhosted.org/packages/33/b9/cd6fe579bed34a906d3d783fe60f2fa297ef55b27bb4538438ee49d4dc41/aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf", size = 1863800, upload-time = "2026-03-31T21:59:43.84Z" }, - { url = "https://files.pythonhosted.org/packages/c0/3f/2c1e2f5144cefa889c8afd5cf431994c32f3b29da9961698ff4e3811b79a/aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b", size = 1958382, upload-time = "2026-03-31T21:59:46.187Z" }, - { url = "https://files.pythonhosted.org/packages/66/1d/f31ec3f1013723b3babe3609e7f119c2c2fb6ef33da90061a705ef3e1bc8/aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a", size = 1803724, upload-time = "2026-03-31T21:59:48.656Z" }, - { url = "https://files.pythonhosted.org/packages/0e/b4/57712dfc6f1542f067daa81eb61da282fab3e6f1966fca25db06c4fc62d5/aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8", size = 1640027, upload-time = "2026-03-31T21:59:51.284Z" }, - { url = "https://files.pythonhosted.org/packages/25/3c/734c878fb43ec083d8e31bf029daae1beafeae582d1b35da234739e82ee7/aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be", size = 1806644, upload-time = "2026-03-31T21:59:53.753Z" }, - { url = "https://files.pythonhosted.org/packages/20/a5/f671e5cbec1c21d044ff3078223f949748f3a7f86b14e34a365d74a5d21f/aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b", size = 1791630, upload-time = "2026-03-31T21:59:56.239Z" }, - { url = "https://files.pythonhosted.org/packages/0b/63/fb8d0ad63a0b8a99be97deac8c04dacf0785721c158bdf23d679a87aa99e/aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6", size = 1809403, upload-time = "2026-03-31T21:59:59.103Z" }, - { url = "https://files.pythonhosted.org/packages/59/0c/bfed7f30662fcf12206481c2aac57dedee43fe1c49275e85b3a1e1742294/aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037", size = 1634924, upload-time = "2026-03-31T22:00:02.116Z" }, - { url = "https://files.pythonhosted.org/packages/17/d6/fd518d668a09fd5a3319ae5e984d4d80b9a4b3df4e21c52f02251ef5a32e/aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500", size = 1836119, upload-time = "2026-03-31T22:00:04.756Z" }, - { url = "https://files.pythonhosted.org/packages/78/b7/15fb7a9d52e112a25b621c67b69c167805cb1f2ab8f1708a5c490d1b52fe/aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9", size = 1772072, upload-time = "2026-03-31T22:00:07.494Z" }, - { url = "https://files.pythonhosted.org/packages/7e/df/57ba7f0c4a553fc2bd8b6321df236870ec6fd64a2a473a8a13d4f733214e/aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8", size = 471819, upload-time = "2026-03-31T22:00:10.277Z" }, - { url = "https://files.pythonhosted.org/packages/62/29/2f8418269e46454a26171bfdd6a055d74febf32234e474930f2f60a17145/aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9", size = 505441, upload-time = "2026-03-31T22:00:12.791Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/ee/ab/93ce242f899b68c51b0578c027aafa791ab3614cb9345fa5d37b5f5c8e3e/aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b", size = 7940674, upload-time = "2026-06-01T19:41:02.763Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/21/61/d11f7d9a3144bffe825247d6367cd93053666da50b94707c9129c78868d5/aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127", size = 502399, upload-time = "2026-06-01T19:38:25.955Z" }, + { url = "https://files.pythonhosted.org/packages/4f/9b/a7e317625d36356844f8bb022cabd305b541f968856cc3c2e0b58e53ee6e/aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981", size = 510068, upload-time = "2026-06-01T19:38:27.828Z" }, + { url = "https://files.pythonhosted.org/packages/11/41/cc2d2cfbfbdc3126ba258f3cd27d1ac8a33492ae3c35a4583ee21f0ba7f1/aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6", size = 481670, upload-time = "2026-06-01T19:38:29.836Z" }, + { url = "https://files.pythonhosted.org/packages/3c/07/381f4023c3b08cb616e520f566d8c58957abad54e56441d41fe67cfb0195/aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2", size = 487591, upload-time = "2026-06-01T19:38:31.704Z" }, + { url = "https://files.pythonhosted.org/packages/fb/4d/4506fdb7a022bdf70011a3bbb4ca00c5c570026ef6a3c5bd7bc70c39089c/aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50", size = 496503, upload-time = "2026-06-01T19:38:33.6Z" }, + { url = "https://files.pythonhosted.org/packages/ef/7d/c814111e04894a45d9e2defc94443879a6f118d9633d5fedfe6e2e8af5f0/aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9", size = 745870, upload-time = "2026-06-01T19:38:36.013Z" }, + { url = "https://files.pythonhosted.org/packages/c6/ee/80eee0efddfe187e7cd05027086b7ce1c0e492e82a4eda58f5c5543a44a0/aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c", size = 505588, upload-time = "2026-06-01T19:38:38.282Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f8/0f28f04eef75d52fc9c715dde7ce9c0abb810fd20cfeb0fea7afd2ab1e98/aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8", size = 504492, upload-time = "2026-06-01T19:38:40.611Z" }, + { url = "https://files.pythonhosted.org/packages/ff/db/44c755232085545065c94378dfce38641b1aee647f4939fcd32f5b32e719/aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83", size = 1752111, upload-time = "2026-06-01T19:38:42.682Z" }, + { url = "https://files.pythonhosted.org/packages/5e/6a/42e030a46743841414402a3b00cd3d78419055e86c66fb5822c14b5abfc6/aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61", size = 1729674, upload-time = "2026-06-01T19:38:44.79Z" }, + { url = "https://files.pythonhosted.org/packages/34/26/3199beb415202e3108e7b83ecebe10914d806d33fb9860c3e4aa60a19be3/aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277", size = 1798808, upload-time = "2026-06-01T19:38:47.01Z" }, + { url = "https://files.pythonhosted.org/packages/bd/94/b9b6fcf0ee17c21d0d19fb8c22bf83ad18f82e702a9c3bd901a868f5e446/aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882", size = 1891921, upload-time = "2026-06-01T19:38:49.233Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a3/3800dbd095cb2bb165a7ea5d94d790914677e27f45638c7d80e3f34c8945/aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096", size = 1777241, upload-time = "2026-06-01T19:38:52.04Z" }, + { url = "https://files.pythonhosted.org/packages/21/2a/45be91ad1b860508557448d4cc2e165a2ee68dd865657b73bf66cc5a00fb/aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988", size = 1579554, upload-time = "2026-06-01T19:38:54.508Z" }, + { url = "https://files.pythonhosted.org/packages/b4/3d/dc94df99ed1511fdf28314f722643ed334112643cab00223577085e788c4/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c", size = 1714864, upload-time = "2026-06-01T19:38:56.788Z" }, + { url = "https://files.pythonhosted.org/packages/ae/e4/1f1c8acbb3acd5c8f795473b92c9c3d44eb60a5692c6104256c8a1c83a0c/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3", size = 1749803, upload-time = "2026-06-01T19:38:59.367Z" }, + { url = "https://files.pythonhosted.org/packages/0b/c8/c45ea6e7ed84cebba939b9c334498a045ba19d79c61b0110df5f21580de3/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac", size = 1765023, upload-time = "2026-06-01T19:39:01.651Z" }, + { url = "https://files.pythonhosted.org/packages/a8/a1/a932941784432962fe390e1066823aaef64b4e5ac9fa595df57b5fe472a9/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e", size = 1571671, upload-time = "2026-06-01T19:39:04.044Z" }, + { url = "https://files.pythonhosted.org/packages/b0/01/e1280feac522597a4d46eb67a0cdfa053cfae263033030b761ab146f29fb/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec", size = 1789904, upload-time = "2026-06-01T19:39:06.294Z" }, + { url = "https://files.pythonhosted.org/packages/fa/10/ab28818262f4d26bdb47ed5f1fc7999b69e2fc6e0370b02d0f49011f45ea/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869", size = 1754516, upload-time = "2026-06-01T19:39:08.788Z" }, + { url = "https://files.pythonhosted.org/packages/af/cc/c122eabd7a1b7e0c9bbdd6be60e4715905b858399145d9df872bb94f1427/aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456", size = 448656, upload-time = "2026-06-01T19:39:11.171Z" }, + { url = "https://files.pythonhosted.org/packages/41/a5/bab07d79848a00eedd8ed979ccb302aaea3ac6eb9fa16bd0ed87135869b4/aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11", size = 475803, upload-time = "2026-06-01T19:39:13.439Z" }, + { url = "https://files.pythonhosted.org/packages/d1/a0/f03ade8566c153666a3871afccbedf6d99911da006325e1fc6cf72a2de99/aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b", size = 443889, upload-time = "2026-06-01T19:39:15.945Z" }, + { url = "https://files.pythonhosted.org/packages/28/03/5f36ab196a88ba5e9648ae5643e6531e67a3a8c0e96f9c6510ff41540fec/aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f", size = 503330, upload-time = "2026-06-01T19:39:18.195Z" }, + { url = "https://files.pythonhosted.org/packages/2c/ce/8b49ec2f30f68e02f314f4832186cd45e583360a5a386058be36855d23b6/aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42", size = 509822, upload-time = "2026-06-01T19:39:20.396Z" }, + { url = "https://files.pythonhosted.org/packages/1a/fe/6edbf5d39bf29322b6816365b17ed8ede4dace164a3aea1abcd30110eb78/aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3", size = 483329, upload-time = "2026-06-01T19:39:22.607Z" }, + { url = "https://files.pythonhosted.org/packages/1b/5a/fae531bdbc6456fb6241f46b7b81e4d8a0dd3fc09118a0055dc7141ac1ec/aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b", size = 489502, upload-time = "2026-06-01T19:39:24.881Z" }, + { url = "https://files.pythonhosted.org/packages/36/f4/48a7b0414db7fed77a03d5dde34508c026afd83510ab6bca08c313855776/aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8", size = 497357, upload-time = "2026-06-01T19:39:27.197Z" }, + { url = "https://files.pythonhosted.org/packages/75/75/e85a13a370acc007fca5feb1fd1b88ac2d8426e6dadd625479b7cadd55a3/aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76", size = 750898, upload-time = "2026-06-01T19:39:29.563Z" }, + { url = "https://files.pythonhosted.org/packages/9e/e4/3d637f800c724eff0e2bed64df72557444482366fd0a35b0cec0e6968f6c/aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e", size = 506986, upload-time = "2026-06-01T19:39:31.872Z" }, + { url = "https://files.pythonhosted.org/packages/1d/df/35161f3598bf7501d2b2a805b41ab4f45a2e34150c421bcb4ef8c0d281a7/aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72", size = 508033, upload-time = "2026-06-01T19:39:34.137Z" }, + { url = "https://files.pythonhosted.org/packages/e5/39/b36e5d3d31e850fb4691dd3e941684ac490a2559249f6fa634b6b0fdf020/aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3", size = 1746213, upload-time = "2026-06-01T19:39:36.654Z" }, + { url = "https://files.pythonhosted.org/packages/b1/28/24e1409e605a9aa5d84abe0e2acb365354b70ae56d40948101cabe3341ab/aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f", size = 1705862, upload-time = "2026-06-01T19:39:38.968Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d0/e5eb3ff1daeaf644c7e36a957517672494122628e067c38b263fa04eda77/aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3", size = 1798909, upload-time = "2026-06-01T19:39:41.334Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ba/8943f906f0570342886ababb9a722a44e360f786a028c5e0b0e29e3f735b/aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6", size = 1868892, upload-time = "2026-06-01T19:39:43.807Z" }, + { url = "https://files.pythonhosted.org/packages/3a/05/27df32c844b2156e1675a8d8ec22d963e3c8ba469ed7ceb1863320c7b521/aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a", size = 1751659, upload-time = "2026-06-01T19:39:46.398Z" }, + { url = "https://files.pythonhosted.org/packages/7f/62/da182e5910ab912b2e88aa919b61a16046a37a95714a5795b02eb57b2d18/aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c", size = 1578775, upload-time = "2026-06-01T19:39:48.902Z" }, + { url = "https://files.pythonhosted.org/packages/66/e3/53c67097e8a5ce98625e91e3fa7f43c9c6940de680345d03b3509a72a078/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b", size = 1710090, upload-time = "2026-06-01T19:39:51.392Z" }, + { url = "https://files.pythonhosted.org/packages/dd/55/0e2732ca598c7a4dfe8a775662376d0ca2977cb1030e48386d4da5d9a456/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4", size = 1715016, upload-time = "2026-06-01T19:39:53.807Z" }, + { url = "https://files.pythonhosted.org/packages/5a/96/f0b73730798c9ca525afc30b39f1f81bbe24e245d9654c54d3b39d63212d/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c", size = 1763810, upload-time = "2026-06-01T19:39:56.31Z" }, + { url = "https://files.pythonhosted.org/packages/71/cc/11acb6c4518f448323405a7312b6f255d0f974a34373ad1db7633c4aadc8/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3", size = 1573064, upload-time = "2026-06-01T19:39:58.718Z" }, + { url = "https://files.pythonhosted.org/packages/de/2d/28c31dde0a7dc98c0ee7d0da2ddcec3f7688c4fc131e5989e278d0c03c0a/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb", size = 1775765, upload-time = "2026-06-01T19:40:01.195Z" }, + { url = "https://files.pythonhosted.org/packages/b8/69/155c4ef3aec96417d47024800472b33b16c5d8a665371dcd044c2afdf25d/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52", size = 1733716, upload-time = "2026-06-01T19:40:03.631Z" }, + { url = "https://files.pythonhosted.org/packages/5f/44/6126116fd8a316b712bb615660b855c78466bb67ba1bb1742427eafcf7ac/aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d", size = 453684, upload-time = "2026-06-01T19:40:06.277Z" }, + { url = "https://files.pythonhosted.org/packages/a2/d7/eff4c58a88c5cac5e38b55f44fb8a6d3929c3cbd77356e383e094d3220bd/aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7", size = 481758, upload-time = "2026-06-01T19:40:08.653Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ed/17b5bd9fbcb46e688f02e572f517754a9a75831e7b54702f027761dc4fa5/aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6", size = 450557, upload-time = "2026-06-01T19:40:11.03Z" }, + { url = "https://files.pythonhosted.org/packages/12/34/6180103ce9aabc8ebff3f7bb55a1228ffe60f61042823031d9692cb7b101/aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733", size = 787878, upload-time = "2026-06-01T19:40:13.401Z" }, + { url = "https://files.pythonhosted.org/packages/92/e9/08954a40e8b7baa3d8beadd2b074b186e9b1e9c8ddabc288678a6265de50/aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228", size = 524400, upload-time = "2026-06-01T19:40:15.972Z" }, + { url = "https://files.pythonhosted.org/packages/08/6a/b5965a634ac4d5ba99a463314cf4ab214ca073fcdc38a15e0294273701fc/aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095", size = 527904, upload-time = "2026-06-01T19:40:18.28Z" }, + { url = "https://files.pythonhosted.org/packages/06/b4/932bcdd850c354d9bcca30f360e475d7852e30413fbbd44b182782ed5432/aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde", size = 1912162, upload-time = "2026-06-01T19:40:20.825Z" }, + { url = "https://files.pythonhosted.org/packages/c6/85/ce79bab0310d2e3fd2d7bc7e44412abeff7c8338f8a21dd0f2f1714989e5/aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a", size = 1778813, upload-time = "2026-06-01T19:40:23.726Z" }, + { url = "https://files.pythonhosted.org/packages/05/54/ba62ac2d1bc87e010aad23751e383b8794e45d931df67677313a2da78823/aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936", size = 1899969, upload-time = "2026-06-01T19:40:26.406Z" }, + { url = "https://files.pythonhosted.org/packages/dc/82/7cc7907725d83a19f31551334061e1ab8e108b1d7ac52632a2a844a4acb5/aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e", size = 1991771, upload-time = "2026-06-01T19:40:29.061Z" }, + { url = "https://files.pythonhosted.org/packages/d0/1c/a57de71a4508c93a830b77c28af3d08cd97f606dedfc6b94275347744508/aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b", size = 1868606, upload-time = "2026-06-01T19:40:31.843Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ae/3839726cd49150a53ed340cc24ce5ba09d4c2117020ef9d45542bec5eb2f/aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a", size = 1665437, upload-time = "2026-06-01T19:40:35.01Z" }, + { url = "https://files.pythonhosted.org/packages/35/1e/c237923232c7da7f0392ea25d89fc5e60c0e93f685f4ebca8e7bcdd5271c/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de", size = 1834090, upload-time = "2026-06-01T19:40:37.733Z" }, + { url = "https://files.pythonhosted.org/packages/98/02/a5a7a2524f92d3911761b405a7c067c751891942144adc13e2ad79611e39/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce", size = 1816907, upload-time = "2026-06-01T19:40:40.46Z" }, + { url = "https://files.pythonhosted.org/packages/fa/76/a8b9f0d09234d516af9f2d7dd715557f33b5da3b0b56ead41d1170e86e3c/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c", size = 1840382, upload-time = "2026-06-01T19:40:43.48Z" }, + { url = "https://files.pythonhosted.org/packages/c9/8e/140e715a0a4bbc211979ea30ec8396ad2ed5bf90ab87d8058fc4668b1923/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7", size = 1659497, upload-time = "2026-06-01T19:40:46.265Z" }, + { url = "https://files.pythonhosted.org/packages/10/c7/7ba5de8af9650b9767b063c675427b8685f43fa7ce563673a7bc3af60f08/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928", size = 1870829, upload-time = "2026-06-01T19:40:49.583Z" }, + { url = "https://files.pythonhosted.org/packages/cc/bc/2aaab2f85cadb26ea59c091fa2b8e370d625154b5c14b478f1b489d07551/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0", size = 1832281, upload-time = "2026-06-01T19:40:52.303Z" }, + { url = "https://files.pythonhosted.org/packages/39/98/31b9ad9fbc01f0075ee7221002df5fd2d10b647f451ca5f30edc802d9dd6/aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6", size = 490597, upload-time = "2026-06-01T19:40:54.937Z" }, + { url = "https://files.pythonhosted.org/packages/59/1f/299b21441c8de42ff70fddc7cfe65e92f810abcf740739a09b56f7835364/aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2", size = 525789, upload-time = "2026-06-01T19:40:57.306Z" }, + { url = "https://files.pythonhosted.org/packages/70/11/7f83fcba9ee05d4c54d61b3f8104da0d43a59adac44dd28effc0c9a10422/aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24", size = 467399, upload-time = "2026-06-01T19:40:59.993Z" }, ] [[package]] @@ -331,7 +344,7 @@ wheels = [ [[package]] name = "berdl-notebook-utils" version = "0.0.1" -source = { git = "https://github.com/BERDataLakehouse/spark_notebook.git?subdirectory=notebook_utils#afad8c0925cfd2d9fec53cd1bf62b8ac4ed44f84" } +source = { git = "https://github.com/BERDataLakehouse/spark_notebook.git?subdirectory=notebook_utils#b89af8ba4d136bd982f5b4060eafd7b2c704c724" } dependencies = [ { name = "attrs" }, { name = "cacheout" }, @@ -428,6 +441,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/72/14/a89bb55107b8a9b586c8878f47d0b7750c3688c209f05f915e70de74880d/cacheout-0.16.0-py3-none-any.whl", hash = "sha256:1a52d9aa8b1e9720d8453b061348f15795578231f9ec4ad376fec49e717d0ed8", size = 21837, upload-time = "2023-12-22T17:44:31.556Z" }, ] +[[package]] +name = "cachetools" +version = "6.2.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/39/91/d9ae9a66b01102a18cd16db0cf4cd54187ffe10f0865cc80071a4104fbb3/cachetools-6.2.6.tar.gz", hash = "sha256:16c33e1f276b9a9c0b49ab5782d901e3ad3de0dd6da9bf9bcd29ac5672f2f9e6", size = 32363, upload-time = "2026-01-27T20:32:59.956Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/45/f458fa2c388e79dd9d8b9b0c99f1d31b568f27388f2fdba7bb66bbc0c6ed/cachetools-6.2.6-py3-none-any.whl", hash = "sha256:8c9717235b3c651603fff0076db52d6acbfd1b338b8ed50256092f7ce9c85bda", size = 11668, upload-time = "2026-01-27T20:32:58.527Z" }, +] + [[package]] name = "cdm-data-loaders" version = "0.1.8" @@ -438,7 +460,7 @@ dependencies = [ { name = "click" }, { name = "defusedxml" }, { name = "delta-spark" }, - { name = "dlt", extra = ["deltalake", "duckdb", "filesystem", "parquet"] }, + { name = "dlt", extra = ["deltalake", "duckdb", "filesystem", "parquet", "pyiceberg"] }, { name = "frictionless", extra = ["aws"] }, { name = "frozendict" }, { name = "ipykernel" }, @@ -451,6 +473,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "berdl-notebook-utils" }, + { name = "cdm-task-service-client" }, { name = "moto", extra = ["s3"] }, { name = "pytest" }, { name = "pytest-asyncio" }, @@ -475,7 +498,7 @@ requires-dist = [ { name = "click", specifier = ">=8.4.1" }, { name = "defusedxml", specifier = ">=0.7.1" }, { name = "delta-spark", specifier = ">=4.2.0" }, - { name = "dlt", extras = ["deltalake", "duckdb", "filesystem", "parquet"], specifier = ">=1.27.2" }, + { name = "dlt", extras = ["deltalake", "duckdb", "filesystem", "parquet", "pyiceberg"], specifier = ">=1.27.2" }, { name = "frictionless", extras = ["aws"], specifier = ">=5.19.0" }, { name = "frozendict", specifier = ">=2.4.7" }, { name = "ipykernel", specifier = ">=7.2.0" }, @@ -488,6 +511,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "berdl-notebook-utils", git = "https://github.com/BERDataLakehouse/spark_notebook.git?subdirectory=notebook_utils" }, + { name = "cdm-task-service-client", git = "https://github.com/kbase/cdm-task-service-client.git" }, { name = "moto", extras = ["s3"], specifier = ">=5.2.1" }, { name = "pytest", specifier = ">=9.0.2" }, { name = "pytest-asyncio", specifier = ">=1.3.0" }, @@ -835,19 +859,19 @@ dependencies = [ [[package]] name = "debugpy" -version = "1.8.20" +version = "1.8.21" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e0/b7/cd8080344452e4874aae67c40d8940e2b4d47b01601a8fd9f44786c757c7/debugpy-1.8.20.tar.gz", hash = "sha256:55bc8701714969f1ab89a6d5f2f3d40c36f91b2cbe2f65d98bf8196f6a6a2c33", size = 1645207, upload-time = "2026-01-29T23:03:28.199Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/aa/12037145b7a56eaa5b29b41872f7a21b538e807e13f32c4d3c46e59be084/debugpy-1.8.21.tar.gz", hash = "sha256:a3c53278e84c94e11bd87c53970ec391d1a67396c8b22609fcac576520e611a6", size = 1697577, upload-time = "2026-06-01T19:30:35.156Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/15/e2/fc500524cc6f104a9d049abc85a0a8b3f0d14c0a39b9c140511c61e5b40b/debugpy-1.8.20-cp313-cp313-macosx_15_0_universal2.whl", hash = "sha256:5dff4bb27027821fdfcc9e8f87309a28988231165147c31730128b1c983e282a", size = 2539560, upload-time = "2026-01-29T23:03:48.738Z" }, - { url = "https://files.pythonhosted.org/packages/90/83/fb33dcea789ed6018f8da20c5a9bc9d82adc65c0c990faed43f7c955da46/debugpy-1.8.20-cp313-cp313-manylinux_2_34_x86_64.whl", hash = "sha256:84562982dd7cf5ebebfdea667ca20a064e096099997b175fe204e86817f64eaf", size = 4293272, upload-time = "2026-01-29T23:03:50.169Z" }, - { url = "https://files.pythonhosted.org/packages/a6/25/b1e4a01bfb824d79a6af24b99ef291e24189080c93576dfd9b1a2815cd0f/debugpy-1.8.20-cp313-cp313-win32.whl", hash = "sha256:da11dea6447b2cadbf8ce2bec59ecea87cc18d2c574980f643f2d2dfe4862393", size = 5331208, upload-time = "2026-01-29T23:03:51.547Z" }, - { url = "https://files.pythonhosted.org/packages/13/f7/a0b368ce54ffff9e9028c098bd2d28cfc5b54f9f6c186929083d4c60ba58/debugpy-1.8.20-cp313-cp313-win_amd64.whl", hash = "sha256:eb506e45943cab2efb7c6eafdd65b842f3ae779f020c82221f55aca9de135ed7", size = 5372930, upload-time = "2026-01-29T23:03:53.585Z" }, - { url = "https://files.pythonhosted.org/packages/33/2e/f6cb9a8a13f5058f0a20fe09711a7b726232cd5a78c6a7c05b2ec726cff9/debugpy-1.8.20-cp314-cp314-macosx_15_0_universal2.whl", hash = "sha256:9c74df62fc064cd5e5eaca1353a3ef5a5d50da5eb8058fcef63106f7bebe6173", size = 2538066, upload-time = "2026-01-29T23:03:54.999Z" }, - { url = "https://files.pythonhosted.org/packages/c5/56/6ddca50b53624e1ca3ce1d1e49ff22db46c47ea5fb4c0cc5c9b90a616364/debugpy-1.8.20-cp314-cp314-manylinux_2_34_x86_64.whl", hash = "sha256:077a7447589ee9bc1ff0cdf443566d0ecf540ac8aa7333b775ebcb8ce9f4ecad", size = 4269425, upload-time = "2026-01-29T23:03:56.518Z" }, - { url = "https://files.pythonhosted.org/packages/c5/d9/d64199c14a0d4c476df46c82470a3ce45c8d183a6796cfb5e66533b3663c/debugpy-1.8.20-cp314-cp314-win32.whl", hash = "sha256:352036a99dd35053b37b7803f748efc456076f929c6a895556932eaf2d23b07f", size = 5331407, upload-time = "2026-01-29T23:03:58.481Z" }, - { url = "https://files.pythonhosted.org/packages/e0/d9/1f07395b54413432624d61524dfd98c1a7c7827d2abfdb8829ac92638205/debugpy-1.8.20-cp314-cp314-win_amd64.whl", hash = "sha256:a98eec61135465b062846112e5ecf2eebb855305acc1dfbae43b72903b8ab5be", size = 5372521, upload-time = "2026-01-29T23:03:59.864Z" }, - { url = "https://files.pythonhosted.org/packages/e0/c3/7f67dea8ccf8fdcb9c99033bbe3e90b9e7395415843accb81428c441be2d/debugpy-1.8.20-py2.py3-none-any.whl", hash = "sha256:5be9bed9ae3be00665a06acaa48f8329d2b9632f15fd09f6a9a8c8d9907e54d7", size = 5337658, upload-time = "2026-01-29T23:04:17.404Z" }, + { url = "https://files.pythonhosted.org/packages/77/6b/d817e1f8cc77aa055d37fba092e0febfdff40fe652d8d53d4cd7a86ad98d/debugpy-1.8.21-cp313-cp313-macosx_15_0_universal2.whl", hash = "sha256:13678151fc401e2d68c9880b91e28714f797d40422994572b24560ef80910a88", size = 2477398, upload-time = "2026-06-01T19:30:57.644Z" }, + { url = "https://files.pythonhosted.org/packages/48/57/412421516afc3055fa577516f00beec3d663f9b0ab330639547ae6c57720/debugpy-1.8.21-cp313-cp313-manylinux_2_34_x86_64.whl", hash = "sha256:ecbd158386c31ffe71d46f72d44d56e66331ab9b16cad649156d514368f23ab2", size = 3962096, upload-time = "2026-06-01T19:30:59.235Z" }, + { url = "https://files.pythonhosted.org/packages/c1/62/2c616337cf6ba7b07ebbc97f02c6c945a8e2f76b365e33ee809c32ee36d1/debugpy-1.8.21-cp313-cp313-win32.whl", hash = "sha256:2c2ae706dec41d99a9ca1f7ebc987a83e65578363be6f6b3ac9067504917fae1", size = 5336288, upload-time = "2026-06-01T19:31:00.79Z" }, + { url = "https://files.pythonhosted.org/packages/f8/99/9175103392f84c4b1bf7622888cdc68da07f0ff7d9e581266428f6776033/debugpy-1.8.21-cp313-cp313-win_amd64.whl", hash = "sha256:aa648733047443eb1d07682c4ef287d36a54507b643ffdf38b09a3ef002c72a0", size = 5376567, upload-time = "2026-06-01T19:31:02.56Z" }, + { url = "https://files.pythonhosted.org/packages/ce/3d/f4bbb323a548bfab2af3d6b4ffd9bf22636e55956a1285d317a1de643aad/debugpy-1.8.21-cp314-cp314-macosx_15_0_universal2.whl", hash = "sha256:9bb2a685287a2ac9b181cde89edcec64845cb51de7faaa75badb9a698bc24782", size = 2477209, upload-time = "2026-06-01T19:31:04.157Z" }, + { url = "https://files.pythonhosted.org/packages/8c/2d/6e7ec524984a1702777868de49a4c53202bddac2a432a76a093469587750/debugpy-1.8.21-cp314-cp314-manylinux_2_34_x86_64.whl", hash = "sha256:3d6922439bf33fd38a3e2c447869ebc7b97da5cd3d329ff1ef9bc06c4903437e", size = 3927115, upload-time = "2026-06-01T19:31:05.863Z" }, + { url = "https://files.pythonhosted.org/packages/97/47/d1aa6d64005a98a9144647d99306b419396f9ad7bf1d73c119e17a81fb4d/debugpy-1.8.21-cp314-cp314-win32.whl", hash = "sha256:15d4963bd5ffa48f0da0947fd06757fa7621945048a14ad7705431566d3c0e7c", size = 5336724, upload-time = "2026-06-01T19:31:07.711Z" }, + { url = "https://files.pythonhosted.org/packages/5f/67/b905b90d163af11878c1af8abafa4a25206335e112e284e413454543a6da/debugpy-1.8.21-cp314-cp314-win_amd64.whl", hash = "sha256:fe0744a12353406de0ae8ccff0d0a4a666f00801a3db8fd04e7a5f761cd520e8", size = 5373803, upload-time = "2026-06-01T19:31:09.469Z" }, + { url = "https://files.pythonhosted.org/packages/95/51/67e7cf11a53e40694f720457d5b3a1cdaaa3d5a9a633e482f225456b93ff/debugpy-1.8.21-py2.py3-none-any.whl", hash = "sha256:b1e37d333663c8851516a47364ef473da127f9caebe4417e6df6f5825a7e9a92", size = 5352888, upload-time = "2026-06-01T19:31:25.186Z" }, ] [[package]] @@ -971,6 +995,12 @@ filesystem = [ parquet = [ { name = "pyarrow" }, ] +pyiceberg = [ + { name = "pyarrow" }, + { name = "pyiceberg" }, + { name = "pyiceberg-core" }, + { name = "sqlalchemy" }, +] [[package]] name = "dnspython" @@ -1272,47 +1302,47 @@ wheels = [ [[package]] name = "grpcio" -version = "1.80.0" +version = "1.81.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b7/48/af6173dbca4454f4637a4678b67f52ca7e0c1ed7d5894d89d434fecede05/grpcio-1.80.0.tar.gz", hash = "sha256:29aca15edd0688c22ba01d7cc01cb000d72b2033f4a3c72a81a19b56fd143257", size = 12978905, upload-time = "2026-03-30T08:49:10.502Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2f/3a/7c3c25789e3f069e581dc342e03613c5b1cb012c4e8c7d9d5cf960a75856/grpcio-1.80.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:e9e408fc016dffd20661f0126c53d8a31c2821b5c13c5d67a0f5ed5de93319ad", size = 6017243, upload-time = "2026-03-30T08:47:40.075Z" }, - { url = "https://files.pythonhosted.org/packages/04/19/21a9806eb8240e174fd1ab0cd5b9aa948bb0e05c2f2f55f9d5d7405e6d08/grpcio-1.80.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:92d787312e613754d4d8b9ca6d3297e69994a7912a32fa38c4c4e01c272974b0", size = 12010840, upload-time = "2026-03-30T08:47:43.11Z" }, - { url = "https://files.pythonhosted.org/packages/18/3a/23347d35f76f639e807fb7a36fad3068aed100996849a33809591f26eca6/grpcio-1.80.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8ac393b58aa16991a2f1144ec578084d544038c12242da3a215966b512904d0f", size = 6567644, upload-time = "2026-03-30T08:47:46.806Z" }, - { url = "https://files.pythonhosted.org/packages/ff/40/96e07ecb604a6a67ae6ab151e3e35b132875d98bc68ec65f3e5ab3e781d7/grpcio-1.80.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:68e5851ac4b9afe07e7f84483803ad167852570d65326b34d54ca560bfa53fb6", size = 7277830, upload-time = "2026-03-30T08:47:49.643Z" }, - { url = "https://files.pythonhosted.org/packages/9b/e2/da1506ecea1f34a5e365964644b35edef53803052b763ca214ba3870c856/grpcio-1.80.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:873ff5d17d68992ef6605330127425d2fc4e77e612fa3c3e0ed4e668685e3140", size = 6783216, upload-time = "2026-03-30T08:47:52.817Z" }, - { url = "https://files.pythonhosted.org/packages/44/83/3b20ff58d0c3b7f6caaa3af9a4174d4023701df40a3f39f7f1c8e7c48f9d/grpcio-1.80.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2bea16af2750fd0a899bf1abd9022244418b55d1f37da2202249ba4ba673838d", size = 7385866, upload-time = "2026-03-30T08:47:55.687Z" }, - { url = "https://files.pythonhosted.org/packages/47/45/55c507599c5520416de5eefecc927d6a0d7af55e91cfffb2e410607e5744/grpcio-1.80.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba0db34f7e1d803a878284cd70e4c63cb6ae2510ba51937bf8f45ba997cefcf7", size = 8391602, upload-time = "2026-03-30T08:47:58.303Z" }, - { url = "https://files.pythonhosted.org/packages/10/bb/dd06f4c24c01db9cf11341b547d0a016b2c90ed7dbbb086a5710df7dd1d7/grpcio-1.80.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8eb613f02d34721f1acf3626dfdb3545bd3c8505b0e52bf8b5710a28d02e8aa7", size = 7826752, upload-time = "2026-03-30T08:48:01.311Z" }, - { url = "https://files.pythonhosted.org/packages/f9/1e/9d67992ba23371fd63d4527096eb8c6b76d74d52b500df992a3343fd7251/grpcio-1.80.0-cp313-cp313-win32.whl", hash = "sha256:93b6f823810720912fd131f561f91f5fed0fda372b6b7028a2681b8194d5d294", size = 4142310, upload-time = "2026-03-30T08:48:04.594Z" }, - { url = "https://files.pythonhosted.org/packages/cf/e6/283326a27da9e2c3038bc93eeea36fb118ce0b2d03922a9cda6688f53c5b/grpcio-1.80.0-cp313-cp313-win_amd64.whl", hash = "sha256:e172cf795a3ba5246d3529e4d34c53db70e888fa582a8ffebd2e6e48bc0cba50", size = 4882833, upload-time = "2026-03-30T08:48:07.363Z" }, - { url = "https://files.pythonhosted.org/packages/c5/6d/e65307ce20f5a09244ba9e9d8476e99fb039de7154f37fb85f26978b59c3/grpcio-1.80.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:3d4147a97c8344d065d01bbf8b6acec2cf86fb0400d40696c8bdad34a64ffc0e", size = 6017376, upload-time = "2026-03-30T08:48:10.005Z" }, - { url = "https://files.pythonhosted.org/packages/69/10/9cef5d9650c72625a699c549940f0abb3c4bfdb5ed45a5ce431f92f31806/grpcio-1.80.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d8e11f167935b3eb089ac9038e1a063e6d7dbe995c0bb4a661e614583352e76f", size = 12018133, upload-time = "2026-03-30T08:48:12.927Z" }, - { url = "https://files.pythonhosted.org/packages/04/82/983aabaad82ba26113caceeb9091706a0696b25da004fe3defb5b346e15b/grpcio-1.80.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f14b618fc30de822681ee986cfdcc2d9327229dc4c98aed16896761cacd468b9", size = 6574748, upload-time = "2026-03-30T08:48:16.386Z" }, - { url = "https://files.pythonhosted.org/packages/07/d7/031666ef155aa0bf399ed7e19439656c38bbd143779ae0861b038ce82abd/grpcio-1.80.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4ed39fbdcf9b87370f6e8df4e39ca7b38b3e5e9d1b0013c7b6be9639d6578d14", size = 7277711, upload-time = "2026-03-30T08:48:19.627Z" }, - { url = "https://files.pythonhosted.org/packages/e8/43/f437a78f7f4f1d311804189e8f11fb311a01049b2e08557c1068d470cb2e/grpcio-1.80.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2dcc70e9f0ba987526e8e8603a610fb4f460e42899e74e7a518bf3c68fe1bf05", size = 6785372, upload-time = "2026-03-30T08:48:22.373Z" }, - { url = "https://files.pythonhosted.org/packages/93/3d/f6558e9c6296cb4227faa5c43c54a34c68d32654b829f53288313d16a86e/grpcio-1.80.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:448c884b668b868562b1bda833c5fce6272d26e1926ec46747cda05741d302c1", size = 7395268, upload-time = "2026-03-30T08:48:25.638Z" }, - { url = "https://files.pythonhosted.org/packages/06/21/0fdd77e84720b08843c371a2efa6f2e19dbebf56adc72df73d891f5506f0/grpcio-1.80.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a1dc80fe55685b4a543555e6eef975303b36c8db1023b1599b094b92aa77965f", size = 8392000, upload-time = "2026-03-30T08:48:28.974Z" }, - { url = "https://files.pythonhosted.org/packages/f5/68/67f4947ed55d2e69f2cc199ab9fd85e0a0034d813bbeef84df6d2ba4d4b7/grpcio-1.80.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:31b9ac4ad1aa28ffee5503821fafd09e4da0a261ce1c1281c6c8da0423c83b6e", size = 7828477, upload-time = "2026-03-30T08:48:32.054Z" }, - { url = "https://files.pythonhosted.org/packages/44/b6/8d4096691b2e385e8271911a0de4f35f0a6c7d05aff7098e296c3de86939/grpcio-1.80.0-cp314-cp314-win32.whl", hash = "sha256:367ce30ba67d05e0592470428f0ec1c31714cab9ef19b8f2e37be1f4c7d32fae", size = 4218563, upload-time = "2026-03-30T08:48:34.538Z" }, - { url = "https://files.pythonhosted.org/packages/e5/8c/bbe6baf2557262834f2070cf668515fa308b2d38a4bbf771f8f7872a7036/grpcio-1.80.0-cp314-cp314-win_amd64.whl", hash = "sha256:3b01e1f5464c583d2f567b2e46ff0d516ef979978f72091fd81f5ab7fa6e2e7f", size = 5019457, upload-time = "2026-03-30T08:48:37.308Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/15/f3/23f47b24f8d8c2028eba501db3acfbb2f592cbb5995eaa6e363a627b74d7/grpcio-1.81.0.tar.gz", hash = "sha256:a5acd7efd3b1fe9b4eb0bcaaa1507eed68a0ad0678b654c3f7b464df9ba9dca5", size = 13032272, upload-time = "2026-06-01T05:56:22.827Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/29/779ee53c931d0fd55c1d459fde43e485172caa3ac87cbd43d003a13a0185/grpcio-1.81.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:62bbe463c9f0f2ff24e31bd25f8dd8b4bae78900e315915a3195a0ef1471a855", size = 6054973, upload-time = "2026-06-01T05:55:25.043Z" }, + { url = "https://files.pythonhosted.org/packages/9e/b6/7211807926b5a17f8d9a5d47c739a163d6812fefe3e4714e81cf92945ed7/grpcio-1.81.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:43c121e135ae44d1559b430db2b2dfad7421cbbe40e1deba506c7dc62b439719", size = 12048662, upload-time = "2026-06-01T05:55:28.453Z" }, + { url = "https://files.pythonhosted.org/packages/64/89/b1b93ef6b34bd20bbaf707fa99133bc9cc302139d5ec6f77a165c7169796/grpcio-1.81.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f345de40ef2e65f63645d53d251824e6070e07804827c5b00ec2e44555f9f901", size = 6599116, upload-time = "2026-06-01T05:55:31.185Z" }, + { url = "https://files.pythonhosted.org/packages/eb/bc/c89f9b9d1c22895715356a1e009554dae66319e97826bb4d30bcda7d29e8/grpcio-1.81.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:8c0855a350886f713b9e458e2a10d208009dcaa849f574e39cd6067db1fe1279", size = 7307591, upload-time = "2026-06-01T05:55:33.463Z" }, + { url = "https://files.pythonhosted.org/packages/65/4a/1df2a4cb4a1386e066ab7e4175e34bb884b35ccb60d3621c09c84af6aabb/grpcio-1.81.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a524cd530900bd24511fcb7f2ed144da4ea37711c4b094475d0bceca7a93a170", size = 6811797, upload-time = "2026-06-01T05:55:36.731Z" }, + { url = "https://files.pythonhosted.org/packages/8d/dc/fa189d20601a1be25b08850cfb733879bbb1047b62a8feec3a60e3e1a87b/grpcio-1.81.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e7746ba3e6efc9e2b748eff59470a2b8684d5a9ec607c6580bcaa5be175820bc", size = 7415131, upload-time = "2026-06-01T05:55:39.451Z" }, + { url = "https://files.pythonhosted.org/packages/ad/a3/5625c48cb48d23c6631b3e5294f88e4c751f22a52591ae78859fab96dca1/grpcio-1.81.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:aaaa4f7f2057d795952e4eacf3f342be8b5b156992f6ac85023c8b98794ebd47", size = 8408398, upload-time = "2026-06-01T05:55:42.219Z" }, + { url = "https://files.pythonhosted.org/packages/75/34/0f8202c6809a46c2b4d69125ef3667c40b1c211f8e19930e5fa1f1197039/grpcio-1.81.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0fba53cb96004b2b7fb758b46b2288cb49d0b658316a4e73f3ef67230616ee65", size = 7844481, upload-time = "2026-06-01T05:55:44.849Z" }, + { url = "https://files.pythonhosted.org/packages/c0/95/c3366b5b5edf4c4adc90f2e29ca16e57965a8e56dc8d2ee89565ba1905bb/grpcio-1.81.0-cp313-cp313-win32.whl", hash = "sha256:c197e2ef75a442528072b29e9755da299110e8610e8bcbb59a6b4cf55384f005", size = 4182777, upload-time = "2026-06-01T05:55:47.459Z" }, + { url = "https://files.pythonhosted.org/packages/a9/a7/932f2f748511a32e641a2aba0d30dded3ed6e8bc330e0924e4d5d86853e6/grpcio-1.81.0-cp313-cp313-win_amd64.whl", hash = "sha256:194eddfacc84d80f50512e9fd4ee851d5f2499f18f299c95aa8fb4748f0537e0", size = 4928085, upload-time = "2026-06-01T05:55:50.158Z" }, + { url = "https://files.pythonhosted.org/packages/c5/1d/28b231333857deb840bc3d182ae087510170ea6d68f21393aeb0fe499530/grpcio-1.81.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:a9351055f52660b58f3d4890ea66188b5134399f82b11aa0c55bd4b99eff5390", size = 6055712, upload-time = "2026-06-01T05:55:52.889Z" }, + { url = "https://files.pythonhosted.org/packages/e8/b8/999c14f9dff0fc47549d2e827cba1343ddc18e1d1bf0d06d2cf628eecbd9/grpcio-1.81.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:300f3337b6425fd16ead9a4f9b2ac25801acb64aa5bc0b99eb69901645b2b1d2", size = 12057189, upload-time = "2026-06-01T05:55:55.952Z" }, + { url = "https://files.pythonhosted.org/packages/1e/3d/1fbde079572562af65351151d840525a13879eb7b481d35b55cd64c6127a/grpcio-1.81.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:97bbd623f7ded558fd4f7cb5a4f600c4d4de65c5dd364c83a5b14b2a10a2d3b5", size = 6608136, upload-time = "2026-06-01T05:55:59.069Z" }, + { url = "https://files.pythonhosted.org/packages/32/89/1f17cb6882abfd8e5a303a25d5d1665abef5a8c499a96198c65a651d1b85/grpcio-1.81.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:ff83d889e3ebf6341c8c7864ad8031591ad5ca61599072fc511644d1eb962d2b", size = 7307045, upload-time = "2026-06-01T05:56:02.376Z" }, + { url = "https://files.pythonhosted.org/packages/48/5a/f98e91b2e755652e637ea2144318b0229b290062199f761b445fe1fa6015/grpcio-1.81.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c4fe218c5a35e1d87a5a26544237f1fa41dfd9cbd3c856b0810a30061f8b0aaf", size = 6812794, upload-time = "2026-06-01T05:56:05.777Z" }, + { url = "https://files.pythonhosted.org/packages/0a/0c/77892d715ac41e7ec0ace2a50080ffb64e189188056f607a66fe0014d1ee/grpcio-1.81.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b8b025b6af43ee0ad4a70307025d77bcab5adde7c4597786010d802c203e9fc5", size = 7422767, upload-time = "2026-06-01T05:56:08.524Z" }, + { url = "https://files.pythonhosted.org/packages/3f/b8/aa04590c6564714d94954515f15a236e59d4b9b3ad01e615f1b706d7792d/grpcio-1.81.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:3d4e0ce5a40a998cf608c8ba60ecfe18fdf364a9aa193ae4ac3faeecd0e86757", size = 8408551, upload-time = "2026-06-01T05:56:11.283Z" }, + { url = "https://files.pythonhosted.org/packages/43/3d/4f4a3450a1973568910c6909cb74abbf2126f68aefae5976962f9f7ad50d/grpcio-1.81.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:aa948712c8e5fa40ec250870bda14bc7578e1bb832a8912d9d2a0f720518edbe", size = 7846468, upload-time = "2026-06-01T05:56:14.536Z" }, + { url = "https://files.pythonhosted.org/packages/88/f4/5827fd248221ad3b44161c23ce9b5f4ee405b04fc6da5fd402a9aa87a84a/grpcio-1.81.0-cp314-cp314-win32.whl", hash = "sha256:fbbe81314a9d92156abce8b62c09364eb8bafc0ca2a19919a45ec64b5c6cb664", size = 4264427, upload-time = "2026-06-01T05:56:17.192Z" }, + { url = "https://files.pythonhosted.org/packages/0c/e8/127dc2b246096ad50ef7c8d9b7b31d757787aeb796368bcdd4454e4204c4/grpcio-1.81.0-cp314-cp314-win_amd64.whl", hash = "sha256:b93cee313cae4e113fbb3a0ce1ea5633db6f63cfde2b2dc1d817429026b2a50b", size = 5070848, upload-time = "2026-06-01T05:56:19.735Z" }, ] [[package]] name = "grpcio-status" -version = "1.80.0" +version = "1.81.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "googleapis-common-protos" }, { name = "grpcio" }, { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b1/ed/105f619bdd00cb47a49aa2feea6232ea2bbb04199d52a22cc6a7d603b5cb/grpcio_status-1.80.0.tar.gz", hash = "sha256:df73802a4c89a3ea88aa2aff971e886fccce162bc2e6511408b3d67a144381cd", size = 13901, upload-time = "2026-03-30T08:54:34.784Z" } +sdist = { url = "https://files.pythonhosted.org/packages/01/b6/cdc177114997d15c887fb09ccfd16705c8ceb8b4ca2487902b54a7bfd1af/grpcio_status-1.81.0.tar.gz", hash = "sha256:b6fe9788cfdd1f0f63c0528a1e0bfdb41e8ff0583e920d2d8e8888598c01bb69", size = 13900, upload-time = "2026-06-01T06:00:32.638Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/76/80/58cd2dfc19a07d022abe44bde7c365627f6c7cb6f692ada6c65ca437d09a/grpcio_status-1.80.0-py3-none-any.whl", hash = "sha256:4b56990363af50dbf2c2ebb80f1967185c07d87aa25aa2bea45ddb75fc181dbe", size = 14638, upload-time = "2026-03-30T08:54:01.569Z" }, + { url = "https://files.pythonhosted.org/packages/f3/b7/5aa346bf1cdecd4ed64b86c10a4d5a089ce3da89145f8328caf0b22b240d/grpcio_status-1.81.0-py3-none-any.whl", hash = "sha256:10eb4c2309db902dc26c1873e80a821bf794be772c10dfd83030f7f59f165fab", size = 14634, upload-time = "2026-06-01T06:00:13.345Z" }, ] [[package]] @@ -1385,11 +1415,11 @@ wheels = [ [[package]] name = "idna" -version = "3.17" +version = "3.18" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b9/28/99c51f664567218d824af024c0251650fb27e4ca066df188dab0769c5b91/idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f", size = 196048, upload-time = "2026-05-28T14:32:38.55Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/de/a7/f76514cc40ad6234098ecdebda08732d75964776c51a42845b7da10649e2/idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c", size = 65316, upload-time = "2026-05-28T14:32:37.035Z" }, + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] [[package]] @@ -1448,7 +1478,7 @@ wheels = [ [[package]] name = "ipython" -version = "9.13.0" +version = "9.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, @@ -1458,14 +1488,14 @@ dependencies = [ { name = "matplotlib-inline" }, { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, { name = "prompt-toolkit" }, - { name = "psutil" }, + { name = "psutil", marker = "sys_platform != 'emscripten'" }, { name = "pygments" }, { name = "stack-data" }, { name = "traitlets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/87cda5842cf5c31837c06ddb588e11c3c35d8ece89b7a0108c06b8c9b00a/ipython-9.13.0.tar.gz", hash = "sha256:7e834b6afc99f020e3f05966ced34792f40267d64cb1ea9043886dab0dde5967", size = 4430549, upload-time = "2026-04-24T12:24:55.221Z" } +sdist = { url = "https://files.pythonhosted.org/packages/21/c2/c0064cf15d026501a1ef70e42efd9c3f818663089399aacc5e37a82901c1/ipython-9.14.0.tar.gz", hash = "sha256:6f27ff0f1d9ea050e0551f71568bc4b34d8aba579e8f111c5b4175f44ac6b4aa", size = 4432601, upload-time = "2026-05-29T15:13:24.611Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b9/86/3060e8029b7cc505cce9a0137431dda81d0a3fde93a8f0f50ee0bf37a795/ipython-9.13.0-py3-none-any.whl", hash = "sha256:57f9d4639e20818d328d287c7b549af3d05f12486ea8f2e7f73e52a36ec4d201", size = 627274, upload-time = "2026-04-24T12:24:53.038Z" }, + { url = "https://files.pythonhosted.org/packages/14/a3/9e59340f02c1dc8f8c0a05b09244712b8609eb5439f9996e887e2b82f452/ipython-9.14.0-py3-none-any.whl", hash = "sha256:8fd984a3372c14b12790b084ba6b5cff5678c0cb063244a0034f06a51f20d6c2", size = 627457, upload-time = "2026-05-29T15:13:22.942Z" }, ] [[package]] @@ -1730,16 +1760,16 @@ wheels = [ [[package]] name = "langchain" -version = "1.3.2" +version = "1.3.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, { name = "langgraph" }, { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d5/d0/c7f9d3d26c0e3f8bb146c6d707ee0fc1d30d8da65a59626e8a580085e929/langchain-1.3.2.tar.gz", hash = "sha256:ffd5f204a46b5fa1a38bf89ba3b45ca0902c02d18fa7d2a2eaeaeb1f5bf19d0a", size = 600598, upload-time = "2026-05-26T18:17:57.715Z" } +sdist = { url = "https://files.pythonhosted.org/packages/36/3f/034eb6cbef90bfccc89b7f8ed0c1d853dc9cb0bea17c7a269534c647ba3a/langchain-1.3.4.tar.gz", hash = "sha256:d6e0654c22848925534f5c0a706f9be481bb09a619ec60a738fbd1e5502e457a", size = 606617, upload-time = "2026-06-02T20:04:49.411Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/82/a54edcd1c48163de5642eb10fa2cb58b13a8889c659964f63f0306b58b1e/langchain-1.3.2-py3-none-any.whl", hash = "sha256:900f6b3f4ee08b9ba3cdbe667dbf42525bd6f66a4a07a7f1db26262673e41ed6", size = 121225, upload-time = "2026-05-26T18:17:56.075Z" }, + { url = "https://files.pythonhosted.org/packages/a5/29/9ffe99c7dc4891a0215ec59c423bea320f943c08a231bc5bae392a438a83/langchain-1.3.4-py3-none-any.whl", hash = "sha256:e51b05ab23d056bc6bf2d97d8c694fb92d6d5765126fef74565d007c27581672", size = 125286, upload-time = "2026-06-02T20:04:48.13Z" }, ] [[package]] @@ -1884,7 +1914,7 @@ wheels = [ [[package]] name = "langgraph" -version = "1.2.2" +version = "1.2.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, @@ -1894,9 +1924,9 @@ dependencies = [ { name = "pydantic" }, { name = "xxhash" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e6/5a/ffc12434ee8aecab830d58b4d204ddea45073eae7639c963310f671a5bf5/langgraph-1.2.2.tar.gz", hash = "sha256:f54a98458976b3ff0774683867df125fb52d8dbedeb2441d0b0656a51331cee5", size = 695730, upload-time = "2026-05-26T18:07:28.49Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/43/dac5a2621c1e57f8eb7f0703f6f6fe34a5caf62f8f0fb4d2bb395bb454ea/langgraph-1.2.4.tar.gz", hash = "sha256:5df076973a2d23efb13eceb279d1e5b46feebcbbeded0a86a2ef669abd9e4399", size = 720374, upload-time = "2026-06-02T17:07:37.347Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/42/9b/b08d578bba73e25351152dfd3d6d21e81210a5fff1b6f26e56f33197c8f5/langgraph-1.2.2-py3-none-any.whl", hash = "sha256:0a851bf4ba5939c5474a2fd57e6b439b5315283e254e42943bd392c2d71a5e03", size = 236376, upload-time = "2026-05-26T18:07:26.577Z" }, + { url = "https://files.pythonhosted.org/packages/48/9e/31ca236104966d7bb14ea9e93cfd73350aea8c41008ddf057b65794ed10d/langgraph-1.2.4-py3-none-any.whl", hash = "sha256:ffe3e1e31dce28907640f82525858470f293506d2b272d07ea3b3ce97974b067", size = 245402, upload-time = "2026-06-02T17:07:35.977Z" }, ] [[package]] @@ -1927,20 +1957,23 @@ wheels = [ [[package]] name = "langgraph-sdk" -version = "0.3.15" +version = "0.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, + { name = "langchain-core" }, + { name = "langchain-protocol" }, { name = "orjson" }, + { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/66/af/cdd4d6f3c05b3c1112ed3f12ef830faf15951b21d22cbc622a4becbbe25c/langgraph_sdk-0.3.15.tar.gz", hash = "sha256:29e805003d2c6e296823dd71992610976fd0428cefaa8b3304fd91f2247037de", size = 201924, upload-time = "2026-05-22T16:54:27.678Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b4/2b/bd8ac26d4e97f6df88ef05ce5b6a38945a3903e1025d926f4752aa88aa97/langgraph_sdk-0.4.2.tar.gz", hash = "sha256:b88f0f5f6328ac0680d6790614a905b2bcfa257f2276dba4e38f0e86db0aa738", size = 348327, upload-time = "2026-06-01T17:51:19.856Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/be/a5/0196d9c05749c25bc198e4909d68c998bc3120297e14944921baf2f4c384/langgraph_sdk-0.3.15-py3-none-any.whl", hash = "sha256:3838773acf7456d158165385d49f48f1e856f28b56ccd99ea139a8f27004815d", size = 98166, upload-time = "2026-05-22T16:54:26.013Z" }, + { url = "https://files.pythonhosted.org/packages/a0/05/aac507337cceae773c2cc9ab91eb6301963af7aeeb55b4217a00e15aff17/langgraph_sdk-0.4.2-py3-none-any.whl", hash = "sha256:75fa5096c1177ce39c847096a8fe3745ffd480ddb412995f836e9f5f884c43dd", size = 160521, upload-time = "2026-06-01T17:51:18.849Z" }, ] [[package]] name = "langsmith" -version = "0.8.7" +version = "0.8.8" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, @@ -1954,9 +1987,9 @@ dependencies = [ { name = "xxhash" }, { name = "zstandard" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2b/8a/9c4a27612f08b607fad56c1c093334d61d3e65d6b1783205118deb1da4bd/langsmith-0.8.7.tar.gz", hash = "sha256:1d0f2b4bcfbf26e9e37bf978dfe23e50d4c90bf1a0f26717879d56f941465a85", size = 4467388, upload-time = "2026-05-29T12:40:16.044Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/93/28df12b3b3c776077983b92f1299c623592b5999695af2a755fb90ff048b/langsmith-0.8.8.tar.gz", hash = "sha256:9d00e54f54d833c1914003527ff03ad0364741034330da72f0adbeaba852b6cf", size = 4468035, upload-time = "2026-05-31T22:14:57.698Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/6e/a99e22455925eccb45aa2c5de1c4773309420bcc0ee1f8fec5bb20e70d9d/langsmith-0.8.7-py3-none-any.whl", hash = "sha256:bfc9bd3276c75201edbf85d12367502aed60752e2da720daee46dccb96f0c3cb", size = 402326, upload-time = "2026-05-29T12:40:13.623Z" }, + { url = "https://files.pythonhosted.org/packages/8d/71/94a8f2b573278a0b0b7dfd37663c0ddd36867f9e2bba69addd183de0cd56/langsmith-0.8.8-py3-none-any.whl", hash = "sha256:9d60d724c0d187c036e184b3ffdf9fa5c6822aa0bb88144a5fb898e79be645af", size = 402712, upload-time = "2026-05-31T22:14:55.908Z" }, ] [[package]] @@ -2140,7 +2173,7 @@ wheels = [ [[package]] name = "mcp" -version = "1.27.1" +version = "1.27.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -2158,9 +2191,9 @@ dependencies = [ { name = "typing-inspection" }, { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/38/83/d1efe7c2980d8a3afa476f4e3d42d53dd54c0ab94c27bee5d755b45c8b73/mcp-1.27.1.tar.gz", hash = "sha256:0f47e1820f8f8f941466b39749eb1d1839a04caddca2bc60e9d46e8a99914924", size = 608458, upload-time = "2026-05-08T16:50:12.601Z" } +sdist = { url = "https://files.pythonhosted.org/packages/27/3c/347cf965d313f5d41764e7d46bea6ffe7d9ef13b983cc429b0340962a082/mcp-1.27.2.tar.gz", hash = "sha256:8e02db104096d1c25b28e64bde29a5c32b31bc241710213e12fd4d84985bdfef", size = 621116, upload-time = "2026-05-29T17:16:04.039Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/73/42d9596facebdb533b7f0b86c1b0364ef350d1f8ba78b1052e8a58b48b65/mcp-1.27.1-py3-none-any.whl", hash = "sha256:1af3c4203b329430fde7a87b4fcb6392a041f5cb851fd68fc674016ab4e7c06f", size = 216260, upload-time = "2026-05-08T16:50:10.547Z" }, + { url = "https://files.pythonhosted.org/packages/c9/11/252c6f971dc4f16af1d98a1c469d8ba523aab00d1bb76b4d3bc1ff32eacc/mcp-1.27.2-py3-none-any.whl", hash = "sha256:d6ff5160c6ca65d93013626efb3fc249de683c30b2d8570755ceddd490344de5", size = 220498, upload-time = "2026-05-29T17:16:02.442Z" }, ] [[package]] @@ -2199,6 +2232,72 @@ dependencies = [ { name = "python-dateutil" }, ] +[[package]] +name = "mmh3" +version = "5.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/1a/edb23803a168f070ded7a3014c6d706f63b90c84ccc024f89d794a3b7a6d/mmh3-5.2.1.tar.gz", hash = "sha256:bbea5b775f0ac84945191fb83f845a6fd9a21a03ea7f2e187defac7e401616ad", size = 33775, upload-time = "2026-03-05T15:55:57.716Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/a5/9daa0508a1569a54130f6198d5462a92deda870043624aa3ea72721aa765/mmh3-5.2.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:723b2681ed4cc07d3401bbea9c201ad4f2a4ca6ba8cddaff6789f715dd2b391e", size = 40832, upload-time = "2026-03-05T15:54:43.212Z" }, + { url = "https://files.pythonhosted.org/packages/0a/6b/3230c6d80c1f4b766dedf280a92c2241e99f87c1504ff74205ec8cebe451/mmh3-5.2.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:3619473a0e0d329fd4aec8075628f8f616be2da41605300696206d6f36920c3d", size = 41964, upload-time = "2026-03-05T15:54:44.204Z" }, + { url = "https://files.pythonhosted.org/packages/62/fb/648bfddb74a872004b6ee751551bfdda783fe6d70d2e9723bad84dbe5311/mmh3-5.2.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:e48d4dbe0f88e53081da605ae68644e5182752803bbc2beb228cca7f1c4454d6", size = 39114, upload-time = "2026-03-05T15:54:45.205Z" }, + { url = "https://files.pythonhosted.org/packages/95/c2/ab7901f87af438468b496728d11264cb397b3574d41506e71b92128e0373/mmh3-5.2.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a482ac121de6973897c92c2f31defc6bafb11c83825109275cffce54bb64933f", size = 39819, upload-time = "2026-03-05T15:54:46.509Z" }, + { url = "https://files.pythonhosted.org/packages/2f/ed/6f88dda0df67de1612f2e130ffea34cf84aaee5bff5b0aff4dbff2babe34/mmh3-5.2.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:17fbb47f0885ace8327ce1235d0416dc86a211dcd8cc1e703f41523be32cfec8", size = 40330, upload-time = "2026-03-05T15:54:47.864Z" }, + { url = "https://files.pythonhosted.org/packages/3d/66/7516d23f53cdf90f43fce24ab80c28f45e6851d78b46bef8c02084edf583/mmh3-5.2.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d51fde50a77f81330523562e3c2734ffdca9c4c9e9d355478117905e1cfe16c6", size = 56078, upload-time = "2026-03-05T15:54:48.9Z" }, + { url = "https://files.pythonhosted.org/packages/bc/34/4d152fdf4a91a132cb226b671f11c6b796eada9ab78080fb5ce1e95adaab/mmh3-5.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:19bbd3b841174ae6ed588536ab5e1b1fe83d046e668602c20266547298d939a9", size = 40498, upload-time = "2026-03-05T15:54:49.942Z" }, + { url = "https://files.pythonhosted.org/packages/d4/4c/8e3af1b6d85a299767ec97bd923f12b06267089c1472c27c1696870d1175/mmh3-5.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:be77c402d5e882b6fbacfd90823f13da8e0a69658405a39a569c6b58fdb17b03", size = 40033, upload-time = "2026-03-05T15:54:50.994Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f2/966ea560e32578d453c9e9db53d602cbb1d0da27317e232afa7c38ceba11/mmh3-5.2.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:fd96476f04db5ceba1cfa0f21228f67c1f7402296f0e73fee3513aa680ad237b", size = 97320, upload-time = "2026-03-05T15:54:52.072Z" }, + { url = "https://files.pythonhosted.org/packages/bb/0d/2c5f9893b38aeb6b034d1a44ecd55a010148054f6a516abe53b5e4057297/mmh3-5.2.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:707151644085dd0f20fe4f4b573d28e5130c4aaa5f587e95b60989c5926653b5", size = 103299, upload-time = "2026-03-05T15:54:53.569Z" }, + { url = "https://files.pythonhosted.org/packages/1c/fc/2ebaef4a4d4376f89761274dc274035ffd96006ab496b4ee5af9b08f21a9/mmh3-5.2.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3737303ca9ea0f7cb83028781148fcda4f1dac7821db0c47672971dabcf63593", size = 106222, upload-time = "2026-03-05T15:54:55.092Z" }, + { url = "https://files.pythonhosted.org/packages/57/09/ea7ffe126d0ba0406622602a2d05e1e1a6841cc92fc322eb576c95b27fad/mmh3-5.2.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2778fed822d7db23ac5008b181441af0c869455b2e7d001f4019636ac31b6fe4", size = 113048, upload-time = "2026-03-05T15:54:56.305Z" }, + { url = "https://files.pythonhosted.org/packages/85/57/9447032edf93a64aa9bef4d9aa596400b1756f40411890f77a284f6293ca/mmh3-5.2.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d57dea657357230cc780e13920d7fa7db059d58fe721c80020f94476da4ca0a1", size = 120742, upload-time = "2026-03-05T15:54:57.453Z" }, + { url = "https://files.pythonhosted.org/packages/53/82/a86cc87cc88c92e9e1a598fee509f0409435b57879a6129bf3b3e40513c7/mmh3-5.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:169e0d178cb59314456ab30772429a802b25d13227088085b0d49b9fe1533104", size = 99132, upload-time = "2026-03-05T15:54:58.583Z" }, + { url = "https://files.pythonhosted.org/packages/54/f7/6b16eb1b40ee89bb740698735574536bc20d6cdafc65ae702ea235578e05/mmh3-5.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7e4e1f580033335c6f76d1e0d6b56baf009d1a64d6a4816347e4271ba951f46d", size = 98686, upload-time = "2026-03-05T15:55:00.078Z" }, + { url = "https://files.pythonhosted.org/packages/e8/88/a601e9f32ad1410f438a6d0544298ea621f989bd34a0731a7190f7dec799/mmh3-5.2.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:2bd9f19f7f1fcebd74e830f4af0f28adad4975d40d80620be19ffb2b2af56c9f", size = 106479, upload-time = "2026-03-05T15:55:01.532Z" }, + { url = "https://files.pythonhosted.org/packages/d6/5c/ce29ae3dfc4feec4007a437a1b7435fb9507532a25147602cd5b52be86db/mmh3-5.2.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:c88653877aeb514c089d1b3d473451677b8b9a6d1497dbddf1ae7934518b06d2", size = 110030, upload-time = "2026-03-05T15:55:02.934Z" }, + { url = "https://files.pythonhosted.org/packages/13/30/ae444ef2ff87c805d525da4fa63d27cda4fe8a48e77003a036b8461cfd5c/mmh3-5.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fceef7fe67c81e1585198215e42ad3fdba3a25644beda8fbdaf85f4d7b93175a", size = 97536, upload-time = "2026-03-05T15:55:04.135Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f9/dc3787ee5c813cc27fe79f45ad4500d9b5437f23a7402435cc34e07c7718/mmh3-5.2.1-cp313-cp313-win32.whl", hash = "sha256:54b64fb2433bc71488e7a449603bf8bd31fbcf9cb56fbe1eb6d459e90b86c37b", size = 40769, upload-time = "2026-03-05T15:55:05.277Z" }, + { url = "https://files.pythonhosted.org/packages/43/67/850e0b5a1e97799822ebfc4ca0e8c6ece3ed8baf7dcdf64de817dfdda2ca/mmh3-5.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:cae6383181f1e345317742d2ddd88f9e7d2682fa4c9432e3a74e47d92dce0229", size = 41563, upload-time = "2026-03-05T15:55:06.283Z" }, + { url = "https://files.pythonhosted.org/packages/c0/cc/98c90b28e1da5458e19fbfaf4adb5289208d3bfccd45dd14eab216a2f0bb/mmh3-5.2.1-cp313-cp313-win_arm64.whl", hash = "sha256:022aa1a528604e6c83d0a7705fdef0b5355d897a9e0fa3a8d26709ceaa06965d", size = 39310, upload-time = "2026-03-05T15:55:07.323Z" }, + { url = "https://files.pythonhosted.org/packages/63/b4/65bc1fb2bb7f83e91c30865023b1847cf89a5f237165575e8c83aa536584/mmh3-5.2.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:d771f085fcdf4035786adfb1d8db026df1eb4b41dac1c3d070d1e49512843227", size = 40794, upload-time = "2026-03-05T15:55:09.773Z" }, + { url = "https://files.pythonhosted.org/packages/c4/86/7168b3d83be8eb553897b1fac9da8bbb06568e5cfe555ffc329ebb46f59d/mmh3-5.2.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:7f196cd7910d71e9d9860da0ff7a77f64d22c1ad931f1dd18559a06e03109fc0", size = 41923, upload-time = "2026-03-05T15:55:10.924Z" }, + { url = "https://files.pythonhosted.org/packages/bf/9b/b653ab611c9060ce8ff0ba25c0226757755725e789292f3ca138a58082cd/mmh3-5.2.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b1f12bd684887a0a5d55e6363ca87056f361e45451105012d329b86ec19dbe0b", size = 39131, upload-time = "2026-03-05T15:55:11.961Z" }, + { url = "https://files.pythonhosted.org/packages/9b/b4/5a2e0d34ab4d33543f01121e832395ea510132ea8e52cdf63926d9d81754/mmh3-5.2.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:d106493a60dcb4aef35a0fac85105e150a11cf8bc2b0d388f5a33272d756c966", size = 39825, upload-time = "2026-03-05T15:55:13.013Z" }, + { url = "https://files.pythonhosted.org/packages/bd/69/81699a8f39a3f8d368bec6443435c0c392df0d200ad915bf0d222b588e03/mmh3-5.2.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:44983e45310ee5b9f73397350251cdf6e63a466406a105f1d16cb5baa659270b", size = 40344, upload-time = "2026-03-05T15:55:14.026Z" }, + { url = "https://files.pythonhosted.org/packages/0c/b3/71c8c775807606e8fd8acc5c69016e1caf3200d50b50b6dd4b40ce10b76c/mmh3-5.2.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:368625fb01666655985391dbad3860dc0ba7c0d6b9125819f3121ee7292b4ac8", size = 56291, upload-time = "2026-03-05T15:55:15.137Z" }, + { url = "https://files.pythonhosted.org/packages/6f/75/2c24517d4b2ce9e4917362d24f274d3d541346af764430249ddcc4cb3a08/mmh3-5.2.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:72d1cc63bcc91e14933f77d51b3df899d6a07d184ec515ea7f56bff659e124d7", size = 40575, upload-time = "2026-03-05T15:55:16.518Z" }, + { url = "https://files.pythonhosted.org/packages/bf/b9/e4a360164365ac9f07a25f0f7928e3a66eb9ecc989384060747aa170e6aa/mmh3-5.2.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e8b4b5580280b9265af3e0409974fb79c64cf7523632d03fbf11df18f8b0181e", size = 40052, upload-time = "2026-03-05T15:55:17.735Z" }, + { url = "https://files.pythonhosted.org/packages/97/ca/120d92223a7546131bbbc31c9174168ee7a73b1366f5463ffe69d9e691fe/mmh3-5.2.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4cbbde66f1183db040daede83dd86c06d663c5bb2af6de1142b7c8c37923dd74", size = 97311, upload-time = "2026-03-05T15:55:18.959Z" }, + { url = "https://files.pythonhosted.org/packages/b6/71/c1a60c1652b8813ef9de6d289784847355417ee0f2980bca002fe87f4ae5/mmh3-5.2.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8ff038d52ef6aa0f309feeba00c5095c9118d0abf787e8e8454d6048db2037fc", size = 103279, upload-time = "2026-03-05T15:55:20.448Z" }, + { url = "https://files.pythonhosted.org/packages/48/29/ad97f4be1509cdcb28ae32c15593ce7c415db47ace37f8fad35b493faa9a/mmh3-5.2.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a4130d0b9ce5fad6af07421b1aecc7e079519f70d6c05729ab871794eded8617", size = 106290, upload-time = "2026-03-05T15:55:21.6Z" }, + { url = "https://files.pythonhosted.org/packages/77/29/1f86d22e281bd8827ba373600a4a8b0c0eae5ca6aa55b9a8c26d2a34decc/mmh3-5.2.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6e0bfe77d238308839699944164b96a2eeccaf55f2af400f54dc20669d8d5f2", size = 113116, upload-time = "2026-03-05T15:55:22.826Z" }, + { url = "https://files.pythonhosted.org/packages/a7/7c/339971ea7ed4c12d98f421f13db3ea576a9114082ccb59d2d1a0f00ccac1/mmh3-5.2.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f963eafc0a77a6c0562397da004f5876a9bcf7265a7bcc3205e29636bc4a1312", size = 120740, upload-time = "2026-03-05T15:55:24.3Z" }, + { url = "https://files.pythonhosted.org/packages/e4/92/3c7c4bdb8e926bb3c972d1e2907d77960c1c4b250b41e8366cf20c6e4373/mmh3-5.2.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:92883836caf50d5255be03d988d75bc93e3f86ba247b7ca137347c323f731deb", size = 99143, upload-time = "2026-03-05T15:55:25.456Z" }, + { url = "https://files.pythonhosted.org/packages/df/0a/33dd8706e732458c8375eae63c981292de07a406bad4ec03e5269654aa2c/mmh3-5.2.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:57b52603e89355ff318025dd55158f6e71396c0f1f609d548e9ea9c94cc6ce0a", size = 98703, upload-time = "2026-03-05T15:55:26.723Z" }, + { url = "https://files.pythonhosted.org/packages/51/04/76bbce05df76cbc3d396f13b2ea5b1578ef02b6a5187e132c6c33f99d596/mmh3-5.2.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f40a95186a72fa0b67d15fef0f157bfcda00b4f59c8a07cbe5530d41ac35d105", size = 106484, upload-time = "2026-03-05T15:55:28.214Z" }, + { url = "https://files.pythonhosted.org/packages/d3/8f/c6e204a2c70b719c1f62ffd9da27aef2dddcba875ea9c31ca0e87b975a46/mmh3-5.2.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:58370d05d033ee97224c81263af123dea3d931025030fd34b61227a768a8858a", size = 110012, upload-time = "2026-03-05T15:55:29.532Z" }, + { url = "https://files.pythonhosted.org/packages/e3/37/7181efd8e39db386c1ebc3e6b7d1f702a09d7c1197a6f2742ed6b5c16597/mmh3-5.2.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7be6dfb49e48fd0a7d91ff758a2b51336f1cd21f9d44b20f6801f072bd080cdd", size = 97508, upload-time = "2026-03-05T15:55:31.01Z" }, + { url = "https://files.pythonhosted.org/packages/42/0f/afa7ca2615fd85e1469474bb860e381443d0b868c083b62b41cb1d7ca32f/mmh3-5.2.1-cp314-cp314-win32.whl", hash = "sha256:54fe8518abe06a4c3852754bfd498b30cc58e667f376c513eac89a244ce781a4", size = 41387, upload-time = "2026-03-05T15:55:32.403Z" }, + { url = "https://files.pythonhosted.org/packages/71/0d/46d42a260ee1357db3d486e6c7a692e303c017968e14865e00efa10d09fc/mmh3-5.2.1-cp314-cp314-win_amd64.whl", hash = "sha256:3f796b535008708846044c43302719c6956f39ca2d93f2edda5319e79a29efbb", size = 42101, upload-time = "2026-03-05T15:55:33.646Z" }, + { url = "https://files.pythonhosted.org/packages/a4/7b/848a8378059d96501a41159fca90d6a99e89736b0afbe8e8edffeac8c74b/mmh3-5.2.1-cp314-cp314-win_arm64.whl", hash = "sha256:cd471ede0d802dd936b6fab28188302b2d497f68436025857ca72cd3810423fe", size = 39836, upload-time = "2026-03-05T15:55:35.026Z" }, + { url = "https://files.pythonhosted.org/packages/27/61/1dabea76c011ba8547c25d30c91c0ec22544487a8750997a27a0c9e1180b/mmh3-5.2.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:5174a697ce042fa77c407e05efe41e03aa56dae9ec67388055820fb48cf4c3ba", size = 57727, upload-time = "2026-03-05T15:55:36.162Z" }, + { url = "https://files.pythonhosted.org/packages/b7/32/731185950d1cf2d5e28979cc8593016ba1619a295faba10dda664a4931b5/mmh3-5.2.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0a3984146e414684a6be2862d84fcb1035f4984851cb81b26d933bab6119bf00", size = 41308, upload-time = "2026-03-05T15:55:37.254Z" }, + { url = "https://files.pythonhosted.org/packages/76/aa/66c76801c24b8c9418b4edde9b5e57c75e72c94e29c48f707e3962534f18/mmh3-5.2.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:bd6e7d363aa93bd3421b30b6af97064daf47bc96005bddba67c5ffbc6df426b8", size = 40758, upload-time = "2026-03-05T15:55:38.61Z" }, + { url = "https://files.pythonhosted.org/packages/9e/bb/79a1f638a02f0ae389f706d13891e2fbf7d8c0a22ecde67ba828951bb60a/mmh3-5.2.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:113f78e7463a36dbbcea05bfe688efd7fa759d0f0c56e73c974d60dcfec3dfcc", size = 109670, upload-time = "2026-03-05T15:55:40.13Z" }, + { url = "https://files.pythonhosted.org/packages/26/94/8cd0e187a288985bcfc79bf5144d1d712df9dee74365f59d26e3a1865be6/mmh3-5.2.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7e8ec5f606e0809426d2440e0683509fb605a8820a21ebd120dcdba61b74ef7f", size = 117399, upload-time = "2026-03-05T15:55:42.076Z" }, + { url = "https://files.pythonhosted.org/packages/42/94/dfea6059bd5c5beda565f58a4096e43f4858fb6d2862806b8bbd12cbb284/mmh3-5.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22b0f9971ec4e07e8223f2beebe96a6cfc779d940b6f27d26604040dd74d3a44", size = 120386, upload-time = "2026-03-05T15:55:43.481Z" }, + { url = "https://files.pythonhosted.org/packages/47/cb/f9c45e62aaa67220179f487772461d891bb582bb2f9783c944832c60efd9/mmh3-5.2.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85ffc9920ffc39c5eee1e3ac9100c913a0973996fbad5111f939bbda49204bb7", size = 125924, upload-time = "2026-03-05T15:55:44.638Z" }, + { url = "https://files.pythonhosted.org/packages/a5/83/fe54a4a7c11bc9f623dfc1707decd034245602b076dfc1dcc771a4163170/mmh3-5.2.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7aec798c2b01aaa65a55f1124f3405804184373abb318a3091325aece235f67c", size = 135280, upload-time = "2026-03-05T15:55:45.866Z" }, + { url = "https://files.pythonhosted.org/packages/97/67/fe7e9e9c143daddd210cd22aef89cbc425d58ecf238d2b7d9eb0da974105/mmh3-5.2.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:55dbbd8ffbc40d1697d5e2d0375b08599dae8746b0b08dea05eee4ce81648fac", size = 110050, upload-time = "2026-03-05T15:55:47.074Z" }, + { url = "https://files.pythonhosted.org/packages/43/c4/6d4b09fcbef80794de447c9378e39eefc047156b290fa3dd2d5257ca8227/mmh3-5.2.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:6c85c38a279ca9295a69b9b088a2e48aa49737bb1b34e6a9dc6297c110e8d912", size = 111158, upload-time = "2026-03-05T15:55:48.239Z" }, + { url = "https://files.pythonhosted.org/packages/81/a6/ca51c864bdb30524beb055a6d8826db3906af0834ec8c41d097a6e8573d5/mmh3-5.2.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:6290289fa5fb4c70fd7f72016e03633d60388185483ff3b162912c81205ae2cf", size = 116890, upload-time = "2026-03-05T15:55:49.405Z" }, + { url = "https://files.pythonhosted.org/packages/cc/04/5a1fe2e2ad843d03e89af25238cbc4f6840a8bb6c4329a98ab694c71deda/mmh3-5.2.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4fc6cd65dc4d2fdb2625e288939a3566e36127a84811a4913f02f3d5931da52d", size = 123121, upload-time = "2026-03-05T15:55:50.61Z" }, + { url = "https://files.pythonhosted.org/packages/af/4d/3c820c6f4897afd25905270a9f2330a23f77a207ea7356f7aadace7273c0/mmh3-5.2.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:623f938f6a039536cc02b7582a07a080f13fdfd48f87e63201d92d7e34d09a18", size = 110187, upload-time = "2026-03-05T15:55:52.143Z" }, + { url = "https://files.pythonhosted.org/packages/21/54/1d71cd143752361c0aebef16ad3f55926a6faf7b112d355745c1f8a25f7f/mmh3-5.2.1-cp314-cp314t-win32.whl", hash = "sha256:29bc3973676ae334412efdd367fcd11d036b7be3efc1ce2407ef8676dabfeb82", size = 41934, upload-time = "2026-03-05T15:55:53.564Z" }, + { url = "https://files.pythonhosted.org/packages/9d/e4/63a2a88f31d93dea03947cccc2a076946857e799ea4f7acdecbf43b324aa/mmh3-5.2.1-cp314-cp314t-win_amd64.whl", hash = "sha256:28cfab66577000b9505a0d068c731aee7ca85cd26d4d63881fab17857e0fe1fb", size = 43036, upload-time = "2026-03-05T15:55:55.252Z" }, + { url = "https://files.pythonhosted.org/packages/a0/0f/59204bf136d1201f8d7884cfbaf7498c5b4674e87a4c693f9bde63741ce1/mmh3-5.2.1-cp314-cp314t-win_arm64.whl", hash = "sha256:dfd51b4c56b673dfbc43d7d27ef857dd91124801e2806c69bb45585ce0fa019b", size = 40391, upload-time = "2026-03-05T15:55:56.697Z" }, +] + [[package]] name = "more-click" version = "0.1.3" @@ -2390,7 +2489,7 @@ wheels = [ [[package]] name = "openai" -version = "2.38.0" +version = "2.40.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -2402,9 +2501,9 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8f/12/cfa322c5f5dd8fa21aab9a7a8e979e7a11123800f86ca8d82eb68a83d213/openai-2.38.0.tar.gz", hash = "sha256:798694c6cf74145541fda94325b6f8f72d8e1fd0262cc137c8d728177a6a4ce3", size = 772764, upload-time = "2026-05-21T21:23:42.105Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/9f/136562ec6c3b1a50fe06eb0bb34ed21f0d7426ec0140e5cc43ac785b69a5/openai-2.40.0.tar.gz", hash = "sha256:9a756f91f274a24ad6026cbcb2042fd356c8d4a10e8f347b08d34465e585f7a2", size = 781177, upload-time = "2026-06-01T21:48:23.878Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0a/bf/ccff9be562e24207716d04ef9dc931c76aff0c89a7265da43e2104d7fe06/openai-2.38.0-py3-none-any.whl", hash = "sha256:ec6661c57b2dcc47414a767e6e3335c7ed3d19c9696999283a3c82e95c756a3c", size = 1344910, upload-time = "2026-05-21T21:23:39.636Z" }, + { url = "https://files.pythonhosted.org/packages/f6/46/180e14be801a75bc13f234cb1b594b232adeb9c84e60a9ab1832e8333591/openai-2.40.0-py3-none-any.whl", hash = "sha256:2b205637ff214477f9ce9ab035e9f494db0e3fa8f1e599008953735fbf6ff1ff", size = 1350935, upload-time = "2026-06-01T21:48:21.462Z" }, ] [[package]] @@ -2590,11 +2689,11 @@ wheels = [ [[package]] name = "petl" -version = "1.7.17" +version = "1.7.19" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a5/07/16a40e30700f7c6975e62cba01c2f94afbb347dd8ed224a5918c686a47fe/petl-1.7.17.tar.gz", hash = "sha256:802696187c2ef35894c4acf3c0ff9fecff6035cb335944c194416b9a18e8390b", size = 424376, upload-time = "2025-07-10T20:47:13.523Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2e/97/e357f6b04f645712ec336d5ff5a1ef53aad6149fb53f39fd316c1b9a11cf/petl-1.7.19.tar.gz", hash = "sha256:238ff9ad3a85fcbbd041a8c24cfbcf6dd9e969f736b71b8ae25064648db0b0a0", size = 427585, upload-time = "2026-06-01T21:44:10.717Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/5c/ea831abc18dd3268046d7d9a0119f1f8ddc69642e0a5245f839602b8114d/petl-1.7.17-py3-none-any.whl", hash = "sha256:53785128bcdf46eb4472638ad572acc6d87cc83f80b567fed06ee4a947eea5d1", size = 233093, upload-time = "2025-07-10T20:47:12.03Z" }, + { url = "https://files.pythonhosted.org/packages/61/30/040c8afebe44b44d8ecef4a1f6ee947cbddba7eaf7527f18653cfff3f073/petl-1.7.19-py3-none-any.whl", hash = "sha256:51565902062d0b94284da6395581eb97ea8db0be1cf687e01861be825718c435", size = 234352, upload-time = "2026-06-01T21:44:09.216Z" }, ] [[package]] @@ -2727,17 +2826,17 @@ wheels = [ [[package]] name = "protobuf" -version = "6.33.6" +version = "7.35.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } +sdist = { url = "https://files.pythonhosted.org/packages/60/fd/5b1491d9e4b586d621c54f4c36b888714164b6875f8d6afa3f9072906a51/protobuf-7.35.0.tar.gz", hash = "sha256:a2efd84605f41e559f1881b0912b44099d0a2ac9bf46b3474823f10fb393b0e6", size = 458677, upload-time = "2026-05-19T23:02:29.197Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, - { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, - { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, - { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, - { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, - { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, - { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, + { url = "https://files.pythonhosted.org/packages/83/ee/93d06e358a4aa32280b00e722d3ea0a1f25fc3cc5778d80581c9cca2c10e/protobuf-7.35.0-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:66be6c513931c794fa92c080ffee41671390da3d79da219cf9c0c0907f035dda", size = 433225, upload-time = "2026-05-19T23:02:19.884Z" }, + { url = "https://files.pythonhosted.org/packages/8b/39/1c76c2da93f3c507e958e0aecee2391cc44d4625de6c728bbc555195b5a8/protobuf-7.35.0-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:fcbe42a4ac09d3ec9c987ddfcd956afd0b15f1ff613bd8371bde9405ffd5c8e5", size = 328847, upload-time = "2026-05-19T23:02:22.3Z" }, + { url = "https://files.pythonhosted.org/packages/91/1a/39f7ce90a238c1a987a4d81ec26379e02ca0aff367de68e4a1fa474215b9/protobuf-7.35.0-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:4cbf5cc286130e06a6c9bbefac442431173906dfcc979712183d4adcc01b37ee", size = 344030, upload-time = "2026-05-19T23:02:23.591Z" }, + { url = "https://files.pythonhosted.org/packages/70/5b/6baf9008817964454055ff3fe65f1de0b5f1e26c80c82f7fb108b7cd4ea3/protobuf-7.35.0-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:6c0f98f10c8a05ea30f8993dfef2de093d27b490fdae78bb60c8343795d55011", size = 327130, upload-time = "2026-05-19T23:02:24.637Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e5/e46adb0badc388bfb84877a5f9f026aff63f60e611016cf64dbe77e05446/protobuf-7.35.0-cp310-abi3-win32.whl", hash = "sha256:4c4617b83ade0e279d1d2bfe04025a1adb87f9ed657de038620dc0ff959357f6", size = 428946, upload-time = "2026-05-19T23:02:25.741Z" }, + { url = "https://files.pythonhosted.org/packages/a7/ab/547fbd9e16d879dd13c167478f8ae0a83a428008ca07a5e06acdc23ad473/protobuf-7.35.0-cp310-abi3-win_amd64.whl", hash = "sha256:f05bcadf9a2a6b8dda047007075135fb7d08c73d9177aabc067e1be46881a201", size = 439996, upload-time = "2026-05-19T23:02:26.808Z" }, + { url = "https://files.pythonhosted.org/packages/b8/ef/50433d346c56657a70d27f156c7b349ac59a068b01de4eb796e747eecc43/protobuf-7.35.0-py3-none-any.whl", hash = "sha256:c13f325cf242bad135c350629eeb5d54b24228eb472fb3e2e9ebbd4c5dc20ca0", size = 171659, upload-time = "2026-05-19T23:02:27.842Z" }, ] [[package]] @@ -2978,6 +3077,48 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] +[[package]] +name = "pyiceberg" +version = "0.11.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachetools" }, + { name = "click" }, + { name = "fsspec" }, + { name = "mmh3" }, + { name = "pydantic" }, + { name = "pyparsing" }, + { name = "pyroaring" }, + { name = "requests" }, + { name = "rich" }, + { name = "strictyaml" }, + { name = "tenacity" }, + { name = "zstandard" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ce/f0/7616676603fdbd05ab97816337a9b31be08a5f9e1ffd636260812b217e0f/pyiceberg-0.11.1.tar.gz", hash = "sha256:366fe0d5a74e3cf1d4e7cbf3c49e308da60e7835ea268667be9185388f05d7a5", size = 1076075, upload-time = "2026-03-03T00:10:27.61Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/4c/a122d80d98cb6125d87024681263406433f0c25c699d503f5633521e6809/pyiceberg-0.11.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b7ec5db19feab98a31fcd5caccf4a9a4e83f96933d1ca393ba7aea665710c2bb", size = 532644, upload-time = "2026-03-03T00:10:18.574Z" }, + { url = "https://files.pythonhosted.org/packages/10/94/9a8fa5fc580e6dccd34bbbf51e7658cd7b49540e2458783addeff5e22a91/pyiceberg-0.11.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cec0616d2ba6e7dda6327089a2f34ec723aa9ac2c389857ef0b83f65fb135dd6", size = 532787, upload-time = "2026-03-03T00:10:19.656Z" }, + { url = "https://files.pythonhosted.org/packages/b3/ab/ab7c88828bc17d77dbbc5a765419dfec2135629e1d74cdd0762cd38ad867/pyiceberg-0.11.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ddb360da76c62c7c23ec3da40e1af48e6712a563905fea2d1a8911ff7a3b6c4d", size = 722202, upload-time = "2026-03-03T00:10:21.012Z" }, + { url = "https://files.pythonhosted.org/packages/df/38/079cf1c0bf86da315472a926eec0dba10135f43374a2e267336eb98d8c76/pyiceberg-0.11.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d8790f420ebc484236017edba59182cf2a21bd3e4224a0bd0760a9c7268e96a", size = 724037, upload-time = "2026-03-03T00:10:22.176Z" }, + { url = "https://files.pythonhosted.org/packages/08/6b/08eaef477debb110438d943ef3f5985096f660ccb735d6344701cbd075a9/pyiceberg-0.11.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ae27ba4d37925d5b2cff192acaa70c8bb114d632bbc527cc91fea0370702b866", size = 716035, upload-time = "2026-03-03T00:10:23.789Z" }, + { url = "https://files.pythonhosted.org/packages/0b/59/7671d6a630ab1d85c6e7ca8ddf438dc63a0b0dd183bc4be69bf25c0fa5f6/pyiceberg-0.11.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:db66a4e0fdfbf4090631d59c3f65e960d9a5561e9259f6f3993cbe91e396837e", size = 720887, upload-time = "2026-03-03T00:10:24.824Z" }, + { url = "https://files.pythonhosted.org/packages/f0/2b/5c8ad37807efaedb14b20f01f36462684468c80da5b74f4018fb4c1804b5/pyiceberg-0.11.1-cp313-cp313-win_amd64.whl", hash = "sha256:eb3a0a3e630ee89758eb96b39b456f4697732351fb0c080e9498ea578f9b71f9", size = 530923, upload-time = "2026-03-03T00:10:26.196Z" }, +] + +[[package]] +name = "pyiceberg-core" +version = "0.9.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9d/c2/dd7bf4754236d38fe19c1e024bb18197b7194b534f5b47d343e31839013b/pyiceberg_core-0.9.1.tar.gz", hash = "sha256:7e146e7a8a26f5f9cc6a3d09eb16071774d4e95eb5980a34484cdeaf18109df9", size = 694622, upload-time = "2026-05-06T22:45:11.609Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/99/3c980d5257301365daf9e23424ed6dd2db740e22640785a2eb9c9c6a4060/pyiceberg_core-0.9.1-cp310-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:adf72ef90b8714ced5e4a1be402441133f8864d2282b80933c1dacfe5ea9ea29", size = 18455006, upload-time = "2026-05-06T22:45:00.47Z" }, + { url = "https://files.pythonhosted.org/packages/ce/11/54ee9a3b55817ab2cd0012e03070bd67104ff42832c042414c9b0050831c/pyiceberg_core-0.9.1-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6067c7923b7d7c416122e87f670cc655e220e69f9442f5c0f477655217e67176", size = 8799496, upload-time = "2026-05-06T22:45:03.079Z" }, + { url = "https://files.pythonhosted.org/packages/4d/8f/65c46aad223907923d7f4b4457f924d34864daa225aafb8c3f13e4c72562/pyiceberg_core-0.9.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8961ed835d6cb5522e2afb25a15df5c74164542ba945a14391878596b61ba909", size = 10400874, upload-time = "2026-05-06T22:45:05.041Z" }, + { url = "https://files.pythonhosted.org/packages/3a/53/da34104bd3f0c844bf777a98c970bece3e220995b09e723fcbdcff785c0f/pyiceberg_core-0.9.1-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:c4738c314a43341533d02683318e8b8c1361df01bd2880bf1ea563207edc6165", size = 10873322, upload-time = "2026-05-06T22:45:07.154Z" }, + { url = "https://files.pythonhosted.org/packages/5b/a2/a455ec2b0ff531680006960784deea70c744a1f2142c866d3540bbcfae6e/pyiceberg_core-0.9.1-cp310-abi3-win_amd64.whl", hash = "sha256:52f99f6ed8697593acbb5890f2da3b17c699265add15cedd7a148821744677c0", size = 9925793, upload-time = "2026-05-06T22:45:09.633Z" }, +] + [[package]] name = "pyjwt" version = "2.13.0" @@ -2992,6 +3133,59 @@ crypto = [ { name = "cryptography" }, ] +[[package]] +name = "pyparsing" +version = "3.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/91/9c6ee907786a473bf81c5f53cf703ba0957b23ab84c264080fb5a450416f/pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc", size = 6851574, upload-time = "2026-01-21T03:57:59.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, +] + +[[package]] +name = "pyroaring" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/46/a50510d080f8cb089303ec0f7cd80736b2949ca3d148f48f1cc90c49e345/pyroaring-1.1.0.tar.gz", hash = "sha256:f02e4021397ae02a139defdc6813b9942ab163de90affddd4ce4efbac299f619", size = 200298, upload-time = "2026-04-24T21:29:25.212Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/75/1d39ecb04e6cd96d191eb8884864355051df80928dd5096a9dea43fbf63b/pyroaring-1.1.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:72f68a16b00b35481d9b3bfe897ecd8a1f7da69efd92ba5b17347ca11c21cb0d", size = 333363, upload-time = "2026-04-24T21:28:23.838Z" }, + { url = "https://files.pythonhosted.org/packages/20/3e/65cd0871e86d11c5c5cfd0f5abb0ca80eb2b6b5dbe5a2433f315a9ebd90c/pyroaring-1.1.0-cp313-cp313-macosx_14_0_universal2.whl", hash = "sha256:4c443e9f942b6089efe8c9b264576e9d116f90be28a315679375bba2d8a915d6", size = 710573, upload-time = "2026-04-24T21:28:24.884Z" }, + { url = "https://files.pythonhosted.org/packages/f9/a2/f8f23515f41414332e60cd86e4957e2a6838070b2ad5fe25e80f136de635/pyroaring-1.1.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:3beb40eb1220d1ce4fb3661bb019e9a21857e5bb294fe8c1c5016aeb6e82318c", size = 384880, upload-time = "2026-04-24T21:28:25.864Z" }, + { url = "https://files.pythonhosted.org/packages/b0/5b/82dc44b5074a1ff62e702d12611272d1711a60d5518dab23f94e1f7a9b3d/pyroaring-1.1.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f1f56004e8f1c1489bf279c25f1fa4764252cd9af5fb35675774268a4a615ba", size = 1999529, upload-time = "2026-04-24T21:28:26.859Z" }, + { url = "https://files.pythonhosted.org/packages/11/40/b07bac8cdc4b709a05f5c55bb52d4f684e5ea1fadfa0b6d9decf477a9d2a/pyroaring-1.1.0-cp313-cp313-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:13660386ea8905ee4d42c21a6275463e2dc7d31e0b5d65eec210aa7043ad96f4", size = 1842927, upload-time = "2026-04-24T21:28:28.056Z" }, + { url = "https://files.pythonhosted.org/packages/0d/60/c4b511965802dfc77978a9e16f2813f47fb3083db1822019ba1bb169c685/pyroaring-1.1.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0dfb6cf50fd8898179e460e699a6b8326ca508c627d083f7bf62f769fe1717d5", size = 2199538, upload-time = "2026-04-24T21:28:29.425Z" }, + { url = "https://files.pythonhosted.org/packages/e8/12/38f6b50b3f3f41a8b752d3e9efcf105b18eb2c66811831059f25613734ac/pyroaring-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:81ebbc0c880c8a10f13118632e5c0d59159ceada8b651bba18f2e6dc70efdeda", size = 2896904, upload-time = "2026-04-24T21:28:30.67Z" }, + { url = "https://files.pythonhosted.org/packages/5a/b6/b5436e4b93c6bf2bd3dd6ccb88cbdc64b12084151a43e2f5c94be50eb710/pyroaring-1.1.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:370d191b0d1b32bbd99452ef5f0485f22fcc4bf7404d33b821d0ce2459951152", size = 2733819, upload-time = "2026-04-24T21:28:31.882Z" }, + { url = "https://files.pythonhosted.org/packages/ab/8f/f392f268de9607a5c7a95aaed6b9c8a81f00c14d85c33855e9f492095478/pyroaring-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e8b3bfad0ae3ef0e67b40c193863dce8b7d79de545dadbe53c19acc3ace38f66", size = 3161730, upload-time = "2026-04-24T21:28:33.244Z" }, + { url = "https://files.pythonhosted.org/packages/9e/a1/03250fd4834b6a5c13e6600bca47ea20fda579f80bce3551d4985185d164/pyroaring-1.1.0-cp313-cp313-win32.whl", hash = "sha256:eead129046822cb0fd47c78740b81bdaffd0515c0bb0306a2318acf0f0540b58", size = 211194, upload-time = "2026-04-24T21:28:35.001Z" }, + { url = "https://files.pythonhosted.org/packages/70/63/d9b307462cddc82fe94a67d6810e5c802818690e131ba690c1de674d8558/pyroaring-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:90ab2f00c09eed5bd986a80c8641e2dc10e7aca1a2d892d89a44b396e39c08ea", size = 263110, upload-time = "2026-04-24T21:28:35.976Z" }, + { url = "https://files.pythonhosted.org/packages/d9/4a/aa6e9833a6ba9a630efdbec8783b63da6602f763b37a5b5fbc01d73a1af1/pyroaring-1.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:51dd2490a64ad4ed53c4fb58ef1ee3f84f6cbd97cdb47abd9065c9f714ab72ef", size = 216546, upload-time = "2026-04-24T21:28:37.065Z" }, + { url = "https://files.pythonhosted.org/packages/93/ab/2260fd567a2d5d957393b932ea940dc31146bd509c88164c1b786eee7836/pyroaring-1.1.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:5e337f8c5b3c2e0c27da83fc2cb702684a47eee907a960cfee964fcb5344515b", size = 335093, upload-time = "2026-04-24T21:28:38.325Z" }, + { url = "https://files.pythonhosted.org/packages/b2/6c/df82a832ff3760c7c7653b80d030fb43b18eb88bfa604e7de3e84457286e/pyroaring-1.1.0-cp314-cp314-macosx_14_0_universal2.whl", hash = "sha256:53acecba8f898e96b84d4139356e30719c70358177e270055901d3ec1cb0e34c", size = 712387, upload-time = "2026-04-24T21:28:39.404Z" }, + { url = "https://files.pythonhosted.org/packages/12/b9/a94d6b2d7a1be2fa5009ecfc345bacb2ee0b536020aeb23e92c6bb7e70f2/pyroaring-1.1.0-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:986efb3aec7655d69c14db2309a2072dbf181bdb906091fede83ad18e316cdaf", size = 385413, upload-time = "2026-04-24T21:28:40.563Z" }, + { url = "https://files.pythonhosted.org/packages/60/6a/3658eadbe28a5a2093c27857dd21441f1ea1cede2ddbe367df76e3018859/pyroaring-1.1.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92643c9dd303de8960c3dbed93a28b8d87da5ed0a7776568979f379d7bc8a885", size = 1995135, upload-time = "2026-04-24T21:28:41.931Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e2/4706ec770790d3433520eb0ea98fc662ccb1533164fd00b01f3413c3425c/pyroaring-1.1.0-cp314-cp314-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6a1d4c59d5b23c01d62f86d57ceefd0c0977de0425aafa7069f2d70563fed3b8", size = 1833652, upload-time = "2026-04-24T21:28:43.381Z" }, + { url = "https://files.pythonhosted.org/packages/2e/38/b8b861738e49fd4c4a54bebe257dced603999365629b4e10cb85fac940b0/pyroaring-1.1.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8ac1bc26223befbca986551521f37f4c1670dfe26fccb2f0fc2775e75be99c1", size = 2188218, upload-time = "2026-04-24T21:28:44.487Z" }, + { url = "https://files.pythonhosted.org/packages/01/8c/96afa9b5f509a5c607deaf30538edb3bdf026447a864cfe3f2c3d7484875/pyroaring-1.1.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b490f2d22df30affbfdcbe4f7896f321edb72a8dc0cbe5f38adec3de5b947c25", size = 2898243, upload-time = "2026-04-24T21:28:45.9Z" }, + { url = "https://files.pythonhosted.org/packages/f9/1e/86f8720525250fe742fc77ea5c2a2074a1ea830efe84a79112ce6fe113d3/pyroaring-1.1.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:56a67794188275f8897a8f1fa64d6313c48241bebbdef38833063e7281b29ef8", size = 2715091, upload-time = "2026-04-24T21:28:47.721Z" }, + { url = "https://files.pythonhosted.org/packages/9d/21/29e8f58c8af2ce016904a7e6aa61be675945224971cd70f3f698e584a23f/pyroaring-1.1.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d9f196007f0b15ea19c21732faacaea83cbf5946b6db4949b3b98cf871c93f0", size = 3149470, upload-time = "2026-04-24T21:28:48.918Z" }, + { url = "https://files.pythonhosted.org/packages/ab/e1/f67ef1c9de461a80707a2f2981320b1b30632720bac426a9bfd51e4744b6/pyroaring-1.1.0-cp314-cp314-win32.whl", hash = "sha256:abc0f0ce22464864fea208315d25e999e45cb5ee646ac1ca11d314a6a51dbe4a", size = 216552, upload-time = "2026-04-24T21:28:50.652Z" }, + { url = "https://files.pythonhosted.org/packages/e8/83/a8d9fee17e6eedf2a2281b2aabcdde86930408486381ec48d1f7d3404521/pyroaring-1.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:532ae6bb1d3431d9956ef07589dd5c8dd918301a83d937c7dc6e511b1364d76a", size = 270712, upload-time = "2026-04-24T21:28:51.817Z" }, + { url = "https://files.pythonhosted.org/packages/42/50/ab2bf3fe45e4c2952690d657321a8470558f92cf93cb197fe5d31f7110d2/pyroaring-1.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:d2706a89242a347be20805147d58a38f4f4d8f6846228c4ee8dfd3587113719c", size = 224783, upload-time = "2026-04-24T21:28:53.183Z" }, + { url = "https://files.pythonhosted.org/packages/d4/0c/9eb48ac698280170f184045814b7bd44829af37c1c6de79a4d7b5ea0c8b8/pyroaring-1.1.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:39eff7dd06c163c22d0a9f9fd72d27e671457bea8cdb71215382a10512539e1d", size = 345689, upload-time = "2026-04-24T21:28:54.494Z" }, + { url = "https://files.pythonhosted.org/packages/f6/61/66b18f8ed17e70f88a410dcfac21e5964c2ad01bd4d6a25024a87522c8a9/pyroaring-1.1.0-cp314-cp314t-macosx_14_0_universal2.whl", hash = "sha256:562fa04bbfd41144d1276ed79505007557c161371450d68a1d71fc83dc01d083", size = 732373, upload-time = "2026-04-24T21:28:55.525Z" }, + { url = "https://files.pythonhosted.org/packages/04/7a/976482874ea5e4476f9dd84e7d0274e480446b1b6ab45dfe301281814b3b/pyroaring-1.1.0-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:591e2ed4d60443dafd9075c1f72e9aaf359ccf5120e32a8c340c2b2ae3da45e7", size = 392985, upload-time = "2026-04-24T21:28:56.629Z" }, + { url = "https://files.pythonhosted.org/packages/ea/22/1eed09ff3aa792865dd52ef447cbe52dbc5901ed88bf1cf4513f7220150e/pyroaring-1.1.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:381eda673442c389993f8b0db2dbf5d02ea8ea9aac6ba736f64cc1ffb6c96885", size = 2045479, upload-time = "2026-04-24T21:28:58.008Z" }, + { url = "https://files.pythonhosted.org/packages/7d/47/de43464f9b28c445868e4bc8f0e6c6dcd51103bd9a757e3dcd9af25a4a69/pyroaring-1.1.0-cp314-cp314t-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d9127feb5356ba3a92bdffa04c1bf6bcbc8d436369f78badf441018c3029dd63", size = 1853013, upload-time = "2026-04-24T21:28:59.265Z" }, + { url = "https://files.pythonhosted.org/packages/03/17/29b128a580ec43905fb766b934e7dcb1095059e99e38e941edf50152207b/pyroaring-1.1.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:650db21c10f42ff2b09ef02c10a779a3d59d0c7512552f3844738b30adbcb8a5", size = 2222628, upload-time = "2026-04-24T21:29:00.792Z" }, + { url = "https://files.pythonhosted.org/packages/bd/d8/bb7d69978a5fcac95da48bedb114554d8345b50b77f042e8cd2a8277bb4b/pyroaring-1.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a5fbcb86e44f1c0c9c052917eee67a04cbac9de7392fb4bc77c140ff4a7e471", size = 2931231, upload-time = "2026-04-24T21:29:02.275Z" }, + { url = "https://files.pythonhosted.org/packages/0b/12/d44d144352a4586544313a97b2576f8f8673b98c02ae7fed77d38751cc1f/pyroaring-1.1.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:0f1d76ef29034017eb2cceebd5fa0504d6ced218ce6432f99da5adecbe038269", size = 2729546, upload-time = "2026-04-24T21:29:03.414Z" }, + { url = "https://files.pythonhosted.org/packages/0c/f9/0d01c6ba01c0d01609fddb1d46138a7ae95b7db386bac4afb0ff082d5c0e/pyroaring-1.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:19d0c81c865d63791fe20e5b38733b66f4f962e677ae7e8b3d3c4947ac6e752f", size = 3177123, upload-time = "2026-04-24T21:29:04.619Z" }, + { url = "https://files.pythonhosted.org/packages/43/6d/f991526fdef3cf7739f6db0cdf12b157e840e0ddd4a7e1c2a477da9072d6/pyroaring-1.1.0-cp314-cp314t-win32.whl", hash = "sha256:1fc112b9a9890f89cc645a16604783ed7fa25299f149b0ef7b45a5e2e3c1f31f", size = 241484, upload-time = "2026-04-24T21:29:06.114Z" }, + { url = "https://files.pythonhosted.org/packages/2e/6e/fb9876940acb50df355a473c087b9924e7b3368070403683941653b6fabc/pyroaring-1.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d92a0f4c7e6bb7deeafac68c79c92ef9340895fe825cf1a31078443753ab6756", size = 304537, upload-time = "2026-04-24T21:29:07.312Z" }, + { url = "https://files.pythonhosted.org/packages/8a/e0/39afe4bddbed6276c54e35e310aa345fbeb00f8890e96e7f48cdc2be9c66/pyroaring-1.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:99c42fe1449acfbf130da65e66b4d5b2726aba4497be359bae7672e38a15fc62", size = 234615, upload-time = "2026-04-24T21:29:08.751Z" }, +] + [[package]] name = "pyspark" version = "4.1.1" @@ -3014,15 +3208,15 @@ connect = [ [[package]] name = "pystow" -version = "0.8.14" +version = "0.8.15" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/40/43/40dd4fa249d388fcb3044e617e3b45c0457602672d6a9fc0b51aab191a62/pystow-0.8.14.tar.gz", hash = "sha256:09b2b2aa8f3177cc79689ccc40dac96724b03a614335a6100928f5ddb163f2b9", size = 53751, upload-time = "2026-05-20T14:53:59.694Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/7a/8f48a8a07d7bfd5c2bdca9c4925516025dc2abe02d3c200d97b4ce99cc2a/pystow-0.8.15.tar.gz", hash = "sha256:1513ab112cfe557bc84d5b30b111b2caca1c7b643e3ab266564977de2d121b10", size = 53839, upload-time = "2026-05-31T21:48:22.48Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/db/cb/4a2424c2736b4d7752f18324e44434b87f2158e42832a020a4b4e24501b3/pystow-0.8.14-py3-none-any.whl", hash = "sha256:8f087aaff834f4300017ea604ab633b1b1e77919e27107add430288be5ca57f9", size = 61041, upload-time = "2026-05-20T14:54:01.36Z" }, + { url = "https://files.pythonhosted.org/packages/07/e8/6b5bb1d699b69926d2358c56acb29dbd3280e86283fd36938b261edea997/pystow-0.8.15-py3-none-any.whl", hash = "sha256:f15c4c515ea602c044df7ed23562086a7cc57f2c9143e19e3fa8aecfa5cf15aa", size = 61110, upload-time = "2026-05-31T21:48:20.956Z" }, ] [[package]] @@ -3116,11 +3310,11 @@ wheels = [ [[package]] name = "python-multipart" -version = "0.0.29" +version = "0.0.30" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4e/fe/70bd71a6738b09a0bdf6480ca6436b167469ca4578b2a0efbe390b4b0e70/python_multipart-0.0.29.tar.gz", hash = "sha256:643e93849196645e2dbdd81a0f8829a23123ad7f797a84a364c6fb3563f18904", size = 45678, upload-time = "2026-05-17T17:29:47.654Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4b/82/c8cd43a6e0719bf5a3b034f6726dd701f75829c08944c83d4b95d02ed0e8/python_multipart-0.0.30.tar.gz", hash = "sha256:0edfe0475c1f46ddd3ff7785a626f6118af32bdcf359bb21260367313bb32118", size = 46316, upload-time = "2026-05-31T19:24:55.198Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8f/cb/769cfc37177252872a45a71f3fbdde9d51b471a3f3c14bfe95dde3407386/python_multipart-0.0.29-py3-none-any.whl", hash = "sha256:2ddcc971cef266225f54f552d8fa10bcfbb1f14446caec199060daac59ff2d69", size = 29640, upload-time = "2026-05-17T17:29:45.69Z" }, + { url = "https://files.pythonhosted.org/packages/1c/fd/0318007beb234790993d3ec5afd051d1dbceb733e81e3afe2b981ece3f37/python_multipart-0.0.30-py3-none-any.whl", hash = "sha256:830964def8c90607ac5daa00514e3987815865713ade8d20febc9177ac0c3c5b", size = 29730, upload-time = "2026-05-31T19:24:53.814Z" }, ] [[package]] @@ -3385,15 +3579,15 @@ wheels = [ [[package]] name = "rich" -version = "15.0.0" +version = "14.3.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markdown-it-py" }, { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e9/67/cae617f1351490c25a4b8ac3b8b63a4dda609295d8222bad12242dfdc629/rich-14.3.4.tar.gz", hash = "sha256:817e02727f2b25b40ef56f5aa2217f400c8489f79ca8f46ea2b70dd5e14558a9", size = 230524, upload-time = "2026-04-11T02:57:45.419Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, + { url = "https://files.pythonhosted.org/packages/b3/76/6d163cfac87b632216f71879e6b2cf17163f773ff59c00b5ff4900a80fa3/rich-14.3.4-py3-none-any.whl", hash = "sha256:07e7adb4690f68864777b1450859253bed81a99a31ac321ac1817b2313558952", size = 310480, upload-time = "2026-04-11T02:57:47.484Z" }, ] [[package]] @@ -3759,14 +3953,26 @@ wheels = [ [[package]] name = "starlette" -version = "1.2.0" +version = "1.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c5/bf/616a066c2760f6c2b1ae3437cc28149734d069fbb46511712beae118a68c/starlette-1.2.0.tar.gz", hash = "sha256:3c5a6b23fff42492914e93890bb80cbfea72dbf37de268eec06185d62a4ca553", size = 2668923, upload-time = "2026-05-28T11:42:50.568Z" } +sdist = { url = "https://files.pythonhosted.org/packages/25/44/ec35f1b6e83094b997da438a02c8c9b0ade2b1e84cfc48bd4656780760a6/starlette-1.2.1.tar.gz", hash = "sha256:9b9b5ebb992e67d6093741e63c2f59e4f6fff986f81163c087867bd7b924b3f6", size = 2701854, upload-time = "2026-05-31T01:07:51.847Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/54/196d0c1db10af76baa4f64894448505d60d3cdf70ef92cbb35f46a4e4c71/starlette-1.2.1-py3-none-any.whl", hash = "sha256:4de0082d08c8f6764a85a54cf1120d6939507a19905c7768acad2a9f875d2b89", size = 73350, upload-time = "2026-05-31T01:07:50.09Z" }, +] + +[[package]] +name = "strictyaml" +version = "1.7.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/08/efd28d49162ce89c2ad61a88bd80e11fb77bc9f6c145402589112d38f8af/strictyaml-1.7.3.tar.gz", hash = "sha256:22f854a5fcab42b5ddba8030a0e4be51ca89af0267961c8d6cfa86395586c407", size = 115206, upload-time = "2023-03-10T12:50:27.062Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/85/492183764d5d01d4514be3730fdb8e228a80605783099551c51627578b5d/starlette-1.2.0-py3-none-any.whl", hash = "sha256:36e0c76ac59157e75dc4b3bdeafba97fb04eaf1878045f15dbef666a6f092ed7", size = 73213, upload-time = "2026-05-28T11:42:48.801Z" }, + { url = "https://files.pythonhosted.org/packages/96/7c/a81ef5ef10978dd073a854e0fa93b5d8021d0594b639cc8f6453c3c78a1d/strictyaml-1.7.3-py3-none-any.whl", hash = "sha256:fb5c8a4edb43bebb765959e420f9b3978d7f1af88c80606c03fb420888f5d1c7", size = 123917, upload-time = "2023-03-10T12:50:17.242Z" }, ] [[package]] @@ -3918,7 +4124,7 @@ wheels = [ [[package]] name = "typer" -version = "0.26.3" +version = "0.26.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-doc" }, @@ -3926,9 +4132,9 @@ dependencies = [ { name = "rich" }, { name = "shellingham" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cc/15/f5fc7be23b7196bc065b282d9589a372392fb10860c80f9c1dd7eb008662/typer-0.26.3.tar.gz", hash = "sha256:3e2b9352f535e5303ef27806dadc2c8647687bdca5c902f03fec3fb88f46a46a", size = 198326, upload-time = "2026-05-28T20:30:50.984Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f4/8a/8dc5733b8939e7f1a71173091a6e27a1658345edbff548a0bf3f5bb26173/typer-0.26.6.tar.gz", hash = "sha256:cdbc160fe7e795b835fb6016419494a521a67bfb86b9476a1ccd0e7727d3ae5b", size = 201595, upload-time = "2026-06-02T13:47:50.536Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cd/cc/c6c5dea061e2740355bfeef22ac6a41751bd2f3903e83921295569bdcec4/typer-0.26.3-py3-none-any.whl", hash = "sha256:e70549ec5a403ca8a0bf0802ddd9f3c6ff7a14ccbb859b01b697baa943636f33", size = 122338, upload-time = "2026-05-28T20:30:49.816Z" }, + { url = "https://files.pythonhosted.org/packages/30/c7/519138260db5e2fe03a509bf9e8ef6af9a514d3565c8fa74fc4fededbae1/typer-0.26.6-py3-none-any.whl", hash = "sha256:49f96d9ee5730cef607bbe155042f40b41fa4c0d0dec04990d580837493805be", size = 122464, upload-time = "2026-06-02T13:47:51.768Z" }, ] [[package]] @@ -4079,15 +4285,15 @@ wheels = [ [[package]] name = "vcrpy" -version = "8.1.1" +version = "8.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyyaml" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b3/07/bcfd5ebd7cb308026ab78a353e091bd699593358be49197d39d004e5ad83/vcrpy-8.1.1.tar.gz", hash = "sha256:58e3053e33b423f3594031cb758c3f4d1df931307f1e67928e30cf352df7709f", size = 85770, upload-time = "2026-01-04T19:22:03.886Z" } +sdist = { url = "https://files.pythonhosted.org/packages/08/db/08183b845b0040bb877dad2bd7e4e0976fc232bb3476d7ee369c6c4f8b5a/vcrpy-8.2.1.tar.gz", hash = "sha256:d73a6e4eb6dae8148e659764b7a00e68cc51ba29ba9e6a85e1f0790ad96b97df", size = 90511, upload-time = "2026-06-16T13:20:52.906Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/d7/f79b05a5d728f8786876a7d75dfb0c5cae27e428081b2d60152fb52f155f/vcrpy-8.1.1-py3-none-any.whl", hash = "sha256:2d16f31ad56493efb6165182dd99767207031b0da3f68b18f975545ede8ac4b9", size = 42445, upload-time = "2026-01-04T19:22:02.532Z" }, + { url = "https://files.pythonhosted.org/packages/f8/7c/0e812ab83f5289404c674f3461ba783250b967d34b5ab034d361236ec042/vcrpy-8.2.1-py3-none-any.whl", hash = "sha256:7ce58c9e2792b246f79d6f4b3e9660676cc6f853be17e1547305b4437ab1ff85", size = 44925, upload-time = "2026-06-16T13:20:51.734Z" }, ] [[package]] @@ -4101,38 +4307,22 @@ wheels = [ [[package]] name = "websockets" -version = "16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/04/24/4b2031d72e840ce4c1ccb255f693b15c334757fc50023e4db9537080b8c4/websockets-16.0.tar.gz", hash = "sha256:5f6261a5e56e8d5c42a4497b364ea24d94d9563e8fbd44e78ac40879c60179b5", size = 179346, upload-time = "2026-01-10T09:23:47.181Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/9c/baa8456050d1c1b08dd0ec7346026668cbc6f145ab4e314d707bb845bf0d/websockets-16.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:878b336ac47938b474c8f982ac2f7266a540adc3fa4ad74ae96fea9823a02cc9", size = 177364, upload-time = "2026-01-10T09:22:59.333Z" }, - { url = "https://files.pythonhosted.org/packages/7e/0c/8811fc53e9bcff68fe7de2bcbe75116a8d959ac699a3200f4847a8925210/websockets-16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:52a0fec0e6c8d9a784c2c78276a48a2bdf099e4ccc2a4cad53b27718dbfd0230", size = 175039, upload-time = "2026-01-10T09:23:01.171Z" }, - { url = "https://files.pythonhosted.org/packages/aa/82/39a5f910cb99ec0b59e482971238c845af9220d3ab9fa76dd9162cda9d62/websockets-16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e6578ed5b6981005df1860a56e3617f14a6c307e6a71b4fff8c48fdc50f3ed2c", size = 175323, upload-time = "2026-01-10T09:23:02.341Z" }, - { url = "https://files.pythonhosted.org/packages/bd/28/0a25ee5342eb5d5f297d992a77e56892ecb65e7854c7898fb7d35e9b33bd/websockets-16.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:95724e638f0f9c350bb1c2b0a7ad0e83d9cc0c9259f3ea94e40d7b02a2179ae5", size = 184975, upload-time = "2026-01-10T09:23:03.756Z" }, - { url = "https://files.pythonhosted.org/packages/f9/66/27ea52741752f5107c2e41fda05e8395a682a1e11c4e592a809a90c6a506/websockets-16.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c0204dc62a89dc9d50d682412c10b3542d748260d743500a85c13cd1ee4bde82", size = 186203, upload-time = "2026-01-10T09:23:05.01Z" }, - { url = "https://files.pythonhosted.org/packages/37/e5/8e32857371406a757816a2b471939d51c463509be73fa538216ea52b792a/websockets-16.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:52ac480f44d32970d66763115edea932f1c5b1312de36df06d6b219f6741eed8", size = 185653, upload-time = "2026-01-10T09:23:06.301Z" }, - { url = "https://files.pythonhosted.org/packages/9b/67/f926bac29882894669368dc73f4da900fcdf47955d0a0185d60103df5737/websockets-16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6e5a82b677f8f6f59e8dfc34ec06ca6b5b48bc4fcda346acd093694cc2c24d8f", size = 184920, upload-time = "2026-01-10T09:23:07.492Z" }, - { url = "https://files.pythonhosted.org/packages/3c/a1/3d6ccdcd125b0a42a311bcd15a7f705d688f73b2a22d8cf1c0875d35d34a/websockets-16.0-cp313-cp313-win32.whl", hash = "sha256:abf050a199613f64c886ea10f38b47770a65154dc37181bfaff70c160f45315a", size = 178255, upload-time = "2026-01-10T09:23:09.245Z" }, - { url = "https://files.pythonhosted.org/packages/6b/ae/90366304d7c2ce80f9b826096a9e9048b4bb760e44d3b873bb272cba696b/websockets-16.0-cp313-cp313-win_amd64.whl", hash = "sha256:3425ac5cf448801335d6fdc7ae1eb22072055417a96cc6b31b3861f455fbc156", size = 178689, upload-time = "2026-01-10T09:23:10.483Z" }, - { url = "https://files.pythonhosted.org/packages/f3/1d/e88022630271f5bd349ed82417136281931e558d628dd52c4d8621b4a0b2/websockets-16.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8cc451a50f2aee53042ac52d2d053d08bf89bcb31ae799cb4487587661c038a0", size = 177406, upload-time = "2026-01-10T09:23:12.178Z" }, - { url = "https://files.pythonhosted.org/packages/f2/78/e63be1bf0724eeb4616efb1ae1c9044f7c3953b7957799abb5915bffd38e/websockets-16.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:daa3b6ff70a9241cf6c7fc9e949d41232d9d7d26fd3522b1ad2b4d62487e9904", size = 175085, upload-time = "2026-01-10T09:23:13.511Z" }, - { url = "https://files.pythonhosted.org/packages/bb/f4/d3c9220d818ee955ae390cf319a7c7a467beceb24f05ee7aaaa2414345ba/websockets-16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fd3cb4adb94a2a6e2b7c0d8d05cb94e6f1c81a0cf9dc2694fb65c7e8d94c42e4", size = 175328, upload-time = "2026-01-10T09:23:14.727Z" }, - { url = "https://files.pythonhosted.org/packages/63/bc/d3e208028de777087e6fb2b122051a6ff7bbcca0d6df9d9c2bf1dd869ae9/websockets-16.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:781caf5e8eee67f663126490c2f96f40906594cb86b408a703630f95550a8c3e", size = 185044, upload-time = "2026-01-10T09:23:15.939Z" }, - { url = "https://files.pythonhosted.org/packages/ad/6e/9a0927ac24bd33a0a9af834d89e0abc7cfd8e13bed17a86407a66773cc0e/websockets-16.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caab51a72c51973ca21fa8a18bd8165e1a0183f1ac7066a182ff27107b71e1a4", size = 186279, upload-time = "2026-01-10T09:23:17.148Z" }, - { url = "https://files.pythonhosted.org/packages/b9/ca/bf1c68440d7a868180e11be653c85959502efd3a709323230314fda6e0b3/websockets-16.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19c4dc84098e523fd63711e563077d39e90ec6702aff4b5d9e344a60cb3c0cb1", size = 185711, upload-time = "2026-01-10T09:23:18.372Z" }, - { url = "https://files.pythonhosted.org/packages/c4/f8/fdc34643a989561f217bb477cbc47a3a07212cbda91c0e4389c43c296ebf/websockets-16.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a5e18a238a2b2249c9a9235466b90e96ae4795672598a58772dd806edc7ac6d3", size = 184982, upload-time = "2026-01-10T09:23:19.652Z" }, - { url = "https://files.pythonhosted.org/packages/dd/d1/574fa27e233764dbac9c52730d63fcf2823b16f0856b3329fc6268d6ae4f/websockets-16.0-cp314-cp314-win32.whl", hash = "sha256:a069d734c4a043182729edd3e9f247c3b2a4035415a9172fd0f1b71658a320a8", size = 177915, upload-time = "2026-01-10T09:23:21.458Z" }, - { url = "https://files.pythonhosted.org/packages/8a/f1/ae6b937bf3126b5134ce1f482365fde31a357c784ac51852978768b5eff4/websockets-16.0-cp314-cp314-win_amd64.whl", hash = "sha256:c0ee0e63f23914732c6d7e0cce24915c48f3f1512ec1d079ed01fc629dab269d", size = 178381, upload-time = "2026-01-10T09:23:22.715Z" }, - { url = "https://files.pythonhosted.org/packages/06/9b/f791d1db48403e1f0a27577a6beb37afae94254a8c6f08be4a23e4930bc0/websockets-16.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:a35539cacc3febb22b8f4d4a99cc79b104226a756aa7400adc722e83b0d03244", size = 177737, upload-time = "2026-01-10T09:23:24.523Z" }, - { url = "https://files.pythonhosted.org/packages/bd/40/53ad02341fa33b3ce489023f635367a4ac98b73570102ad2cdd770dacc9a/websockets-16.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b784ca5de850f4ce93ec85d3269d24d4c82f22b7212023c974c401d4980ebc5e", size = 175268, upload-time = "2026-01-10T09:23:25.781Z" }, - { url = "https://files.pythonhosted.org/packages/74/9b/6158d4e459b984f949dcbbb0c5d270154c7618e11c01029b9bbd1bb4c4f9/websockets-16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:569d01a4e7fba956c5ae4fc988f0d4e187900f5497ce46339c996dbf24f17641", size = 175486, upload-time = "2026-01-10T09:23:27.033Z" }, - { url = "https://files.pythonhosted.org/packages/e5/2d/7583b30208b639c8090206f95073646c2c9ffd66f44df967981a64f849ad/websockets-16.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:50f23cdd8343b984957e4077839841146f67a3d31ab0d00e6b824e74c5b2f6e8", size = 185331, upload-time = "2026-01-10T09:23:28.259Z" }, - { url = "https://files.pythonhosted.org/packages/45/b0/cce3784eb519b7b5ad680d14b9673a31ab8dcb7aad8b64d81709d2430aa8/websockets-16.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:152284a83a00c59b759697b7f9e9cddf4e3c7861dd0d964b472b70f78f89e80e", size = 186501, upload-time = "2026-01-10T09:23:29.449Z" }, - { url = "https://files.pythonhosted.org/packages/19/60/b8ebe4c7e89fb5f6cdf080623c9d92789a53636950f7abacfc33fe2b3135/websockets-16.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bc59589ab64b0022385f429b94697348a6a234e8ce22544e3681b2e9331b5944", size = 186062, upload-time = "2026-01-10T09:23:31.368Z" }, - { url = "https://files.pythonhosted.org/packages/88/a8/a080593f89b0138b6cba1b28f8df5673b5506f72879322288b031337c0b8/websockets-16.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32da954ffa2814258030e5a57bc73a3635463238e797c7375dc8091327434206", size = 185356, upload-time = "2026-01-10T09:23:32.627Z" }, - { url = "https://files.pythonhosted.org/packages/c2/b6/b9afed2afadddaf5ebb2afa801abf4b0868f42f8539bfe4b071b5266c9fe/websockets-16.0-cp314-cp314t-win32.whl", hash = "sha256:5a4b4cc550cb665dd8a47f868c8d04c8230f857363ad3c9caf7a0c3bf8c61ca6", size = 178085, upload-time = "2026-01-10T09:23:33.816Z" }, - { url = "https://files.pythonhosted.org/packages/9f/3e/28135a24e384493fa804216b79a6a6759a38cc4ff59118787b9fb693df93/websockets-16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b14dc141ed6d2dde437cddb216004bcac6a1df0935d79656387bd41632ba0bbd", size = 178531, upload-time = "2026-01-10T09:23:35.016Z" }, - { url = "https://files.pythonhosted.org/packages/6f/28/258ebab549c2bf3e64d2b0217b973467394a9cea8c42f70418ca2c5d0d2e/websockets-16.0-py3-none-any.whl", hash = "sha256:1637db62fad1dc833276dded54215f2c7fa46912301a24bd94d45d46a011ceec", size = 171598, upload-time = "2026-01-10T09:23:45.395Z" }, +version = "15.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/e6/26d09fab466b7ca9c7737474c52be4f76a40301b08362eb2dbc19dcc16c1/websockets-15.0.1.tar.gz", hash = "sha256:82544de02076bafba038ce055ee6412d68da13ab47f0c60cab827346de828dee", size = 177016, upload-time = "2025-03-05T20:03:41.606Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/9f/51f0cf64471a9d2b4d0fc6c534f323b664e7095640c34562f5182e5a7195/websockets-15.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ee443ef070bb3b6ed74514f5efaa37a252af57c90eb33b956d35c8e9c10a1931", size = 175440, upload-time = "2025-03-05T20:02:36.695Z" }, + { url = "https://files.pythonhosted.org/packages/8a/05/aa116ec9943c718905997412c5989f7ed671bc0188ee2ba89520e8765d7b/websockets-15.0.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5a939de6b7b4e18ca683218320fc67ea886038265fd1ed30173f5ce3f8e85675", size = 173098, upload-time = "2025-03-05T20:02:37.985Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0b/33cef55ff24f2d92924923c99926dcce78e7bd922d649467f0eda8368923/websockets-15.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:746ee8dba912cd6fc889a8147168991d50ed70447bf18bcda7039f7d2e3d9151", size = 173329, upload-time = "2025-03-05T20:02:39.298Z" }, + { url = "https://files.pythonhosted.org/packages/31/1d/063b25dcc01faa8fada1469bdf769de3768b7044eac9d41f734fd7b6ad6d/websockets-15.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:595b6c3969023ecf9041b2936ac3827e4623bfa3ccf007575f04c5a6aa318c22", size = 183111, upload-time = "2025-03-05T20:02:40.595Z" }, + { url = "https://files.pythonhosted.org/packages/93/53/9a87ee494a51bf63e4ec9241c1ccc4f7c2f45fff85d5bde2ff74fcb68b9e/websockets-15.0.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c714d2fc58b5ca3e285461a4cc0c9a66bd0e24c5da9911e30158286c9b5be7f", size = 182054, upload-time = "2025-03-05T20:02:41.926Z" }, + { url = "https://files.pythonhosted.org/packages/ff/b2/83a6ddf56cdcbad4e3d841fcc55d6ba7d19aeb89c50f24dd7e859ec0805f/websockets-15.0.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f3c1e2ab208db911594ae5b4f79addeb3501604a165019dd221c0bdcabe4db8", size = 182496, upload-time = "2025-03-05T20:02:43.304Z" }, + { url = "https://files.pythonhosted.org/packages/98/41/e7038944ed0abf34c45aa4635ba28136f06052e08fc2168520bb8b25149f/websockets-15.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:229cf1d3ca6c1804400b0a9790dc66528e08a6a1feec0d5040e8b9eb14422375", size = 182829, upload-time = "2025-03-05T20:02:48.812Z" }, + { url = "https://files.pythonhosted.org/packages/e0/17/de15b6158680c7623c6ef0db361da965ab25d813ae54fcfeae2e5b9ef910/websockets-15.0.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:756c56e867a90fb00177d530dca4b097dd753cde348448a1012ed6c5131f8b7d", size = 182217, upload-time = "2025-03-05T20:02:50.14Z" }, + { url = "https://files.pythonhosted.org/packages/33/2b/1f168cb6041853eef0362fb9554c3824367c5560cbdaad89ac40f8c2edfc/websockets-15.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:558d023b3df0bffe50a04e710bc87742de35060580a293c2a984299ed83bc4e4", size = 182195, upload-time = "2025-03-05T20:02:51.561Z" }, + { url = "https://files.pythonhosted.org/packages/86/eb/20b6cdf273913d0ad05a6a14aed4b9a85591c18a987a3d47f20fa13dcc47/websockets-15.0.1-cp313-cp313-win32.whl", hash = "sha256:ba9e56e8ceeeedb2e080147ba85ffcd5cd0711b89576b83784d8605a7df455fa", size = 176393, upload-time = "2025-03-05T20:02:53.814Z" }, + { url = "https://files.pythonhosted.org/packages/1b/6c/c65773d6cab416a64d191d6ee8a8b1c68a09970ea6909d16965d26bfed1e/websockets-15.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:e09473f095a819042ecb2ab9465aee615bd9c2028e4ef7d933600a8401c79561", size = 176837, upload-time = "2025-03-05T20:02:55.237Z" }, + { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" }, ] [[package]]