diff --git a/.github/workflows/lint_test.yaml b/.github/workflows/lint_test.yaml new file mode 100644 index 0000000..f8c78e5 --- /dev/null +++ b/.github/workflows/lint_test.yaml @@ -0,0 +1,35 @@ +name: Lint archivist + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Install dependencies + run: uv sync --python 3.11 --group dev + + - name: Run lint + run: | + uv run ruff check + + - name: Run lint + run: | + uv run ruff format --check --diff + + - name: Run isort + run: | + uv run isort . --check --diff diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..2e0e5f7 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,24 @@ +name: Release archivist + +on: + release: + types: [published] + +jobs: + deploy: + runs-on: ubuntu-latest + environment: release + permissions: + id-token: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Build package + run: uv build --python 3.11 + + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..eed6642 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,36 @@ +name: Test archivist + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + test: + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Install dependencies + run: uv sync --python ${{ matrix.python-version }} --group dev + + - name: Run tests + run: | + uv run pytest + + - name: Upload coverage report + if: matrix.python-version == '3.11' + run: uv run coveralls + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index ba4c4f7..502f8b2 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ # Archivist +[![Coverage Status](https://coveralls.io/repos/github/simonsobs/archivist/badge.svg?branch=main)](https://coveralls.io/github/simonsobs/archivist?branch=main) + Tools for offline data storage compatible with the Librarian. diff --git a/pyproject.toml b/pyproject.toml index b662569..5cfd8e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,10 +7,11 @@ name = "archivist" dynamic = ["version"] description = "Archival data storage for the Librarian" readme = "README.md" -requires-python = ">=3.9" +requires-python = ">=3.11" +license = "MIT" +license-files = ["LICENSE"] classifiers = [ "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ] dependencies = [ @@ -36,6 +37,7 @@ write_to = "src/archivist/_version.py" dev = [ "coverage>=7.10.7", "coveralls>=4.0.1", + "httpx>=0.28.1", "isort>=6.1.0", "pre-commit>=4.3.0", "pytest>=8.4.2", @@ -113,3 +115,10 @@ mark-parentheses = false profile = "black" line_length = 120 skip = ["_version.py"] + +[tool.pytest.ini_options] +addopts = "--random-order --cov=archivist --cov-report=term-missing --cov-report=xml" + +[tool.coverage.run] +source = ["archivist"] +branch = true diff --git a/src/archivist/server.py b/src/archivist/server.py index 7679cb6..11e375b 100644 --- a/src/archivist/server.py +++ b/src/archivist/server.py @@ -38,7 +38,7 @@ def _status_worker_loop(): @asynccontextmanager -async def slack_post_at_startup_shutdown(app: FastAPI): +async def startup_shutdown_server(app: FastAPI): """ Lifespan event that posts to the slack hook once the FastAPI server starts up and shuts down. @@ -71,7 +71,7 @@ def main() -> FastAPI: title=server_settings.displayed_site_name, description=server_settings.displayed_site_description, openapi_url="/api/v2/openapi.json" if server_settings.debug else None, - lifespan=slack_post_at_startup_shutdown, + lifespan=startup_shutdown_server, ) logger.debug("Adding API router.") diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..e6a1a3b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,129 @@ +# Copyright (c) 2025-2026 Simons Observatory. +# Full license can be found in the top level "LICENSE" file. +from datetime import datetime, timezone + +import pytest + +import archivist.database as database +import archivist.queue as queue_module +import archivist.settings as settings_module +from archivist.settings import Settings +from archivist.storage.storage_disk import StorageDisk + + +@pytest.fixture(autouse=True) +def _reset_module_singletons(): + """ + Several modules cache lazily-created global state (the DB engine, + session maker, loaded settings, the in-memory status queue, and the + shared StorageDisk thread pool). Reset all of it before and after + every test so tests don't leak state into one another. + """ + + def _reset(): + if database._engine is not None: + database._engine.dispose() + database._engine = None + database._SessionMaker = None + settings_module._settings = None + queue_module._status_queue = None + StorageDisk._executor = None + + _reset() + yield + _reset() + + +@pytest.fixture +def archive_root(tmp_path): + path = tmp_path / "archive_root" + path.mkdir() + return path + + +@pytest.fixture +def local_root(tmp_path): + path = tmp_path / "local_root" + path.mkdir() + return path + + +@pytest.fixture +def settings(tmp_path, archive_root, local_root) -> Settings: + """A Settings instance backed by a throwaway on-disk sqlite database.""" + + return Settings( + database_driver="sqlite", + database=str(tmp_path / "archivist_test.db"), + archive_type="posix", + archive_root=str(archive_root), + local_root=str(local_root), + ) + + +@pytest.fixture +def config_path(settings, tmp_path): + """`settings` serialised to a JSON file, as the CLI/server would be given.""" + + path = tmp_path / "test_config.json" + path.write_text(settings.model_dump_json()) + return path + + +@pytest.fixture +def use_settings(monkeypatch, settings, config_path): + """ + Make `get_settings()` return `settings`, regardless of whether the + caller does `import archivist.settings` or + `from archivist.settings import get_settings` (the latter binds its + own reference, so patching the module attribute alone would miss it). + We instead drive this through the same env-var + cache mechanism + `get_settings()` itself uses. + """ + + monkeypatch.setenv("ARCHIVIST_CONFIG_PATH", str(config_path)) + monkeypatch.setattr(settings_module, "_settings", None) + + return settings_module.get_settings() + + +@pytest.fixture +def db_session(use_settings): + """A real database session against a freshly created schema.""" + + database.create_all() + session = database.get_session() + try: + yield session + finally: + session.close() + + +def make_manifest_entry(**overrides): + """Build a dict-form manifest entry, suitable for `ManifestEntry(**entry)`.""" + + entry = { + "name": "file.txt", + "create_time": datetime(2026, 1, 1, tzinfo=timezone.utc), + "size": 1024, + "checksum": "0" * 64, + "uploader": "test-uploader", + "source": "/source", + "instance_path": "/source/file.txt", + "instance_create_time": datetime(2026, 1, 1, tzinfo=timezone.utc), + "instance_available": True, + "outgoing_transfer_id": 0, + } + entry.update(overrides) + return entry + + +def make_manifest_request(json_safe=False, **overrides): + """A one-entry manifest request body. Set `json_safe=True` for a raw HTTP JSON body.""" + + entry = make_manifest_entry(**overrides) + if json_safe: + for key in ("create_time", "instance_create_time"): + entry[key] = entry[key].isoformat() + + return {"librarian_name": "test-librarian", "store_files": [entry]} diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..82ad234 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,66 @@ +# Copyright (c) 2025-2026 Simons Observatory. +# Full license can be found in the top level "LICENSE" file. + +""" +Tests for the FastAPI routes in archivist.api. + +These build a bare FastAPI app that includes the same routers as the real +server (`archivist.server.main`), but skip the lifespan handler -- which +starts busy-looping background worker threads -- since it isn't relevant +to exercising the HTTP endpoints themselves. +""" + +import pytest +from conftest import make_manifest_request +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from archivist.api import health_router +from archivist.api import router as api_router +from archivist.database import yield_session +from archivist.orm.archivequeue import ArchiveQueue +from archivist.settings import get_settings + + +@pytest.fixture +def client(db_session, use_settings): + app = FastAPI() + app.include_router(api_router) + app.include_router(health_router) + + def _yield_session(): + yield db_session + + app.dependency_overrides[get_settings] = lambda: use_settings + app.dependency_overrides[yield_session] = _yield_session + + return TestClient(app) + + +def test_health(client, use_settings): + response = client.get("/health") + + assert response.status_code == 200 + assert response.json() == {"status": "ok", "name": use_settings.displayed_site_name} + + +def test_archive_queues_the_manifest_and_returns_its_id(client, db_session): + """The route enqueues rather than archiving synchronously; the queued row is the contract.""" + + response = client.post("/api/v1/archive", json=make_manifest_request(json_safe=True)) + + assert response.status_code == 200 + manifest_id = response.json()["manifest_id"] + + item = db_session.query(ArchiveQueue).filter_by(manifest_id=manifest_id).one() + assert not item.consumed + assert not item.completed + + +def test_archive_rejects_manifests_over_the_size_limit(client, use_settings): + use_settings.maximal_size_bytes = 100 + + response = client.post("/api/v1/archive", json=make_manifest_request(json_safe=True, size=1_000_000)) + + assert response.status_code == 400 + assert "error" in response.json() diff --git a/tests/test_classes.py b/tests/test_classes.py deleted file mode 100644 index 2db294e..0000000 --- a/tests/test_classes.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2025-2026 Simons Observatory. -# Full license can be found in the top level "LICENSE" file. - -"""Test package imports.""" - -import archivist -from archivist import Archive, RegistryLibrarian, RegistrySqlite, StorageDisk, StorageHPSS -from archivist import _version as pkg_version - - -def test_construction(): - archive = Archive(paths=[]) - reg_lib = RegistryLibrarian() - reg_sql = RegistrySqlite() - store_disk = StorageDisk(".") - store_hpss = StorageHPSS() diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..f1a577e --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,86 @@ +# Copyright (c) 2025-2026 Simons Observatory. +# Full license can be found in the top level "LICENSE" file. + +""" +Tests for the archivist.__main__ click CLI. + +The only real logic here is turning a source path into a manifest; the rest +is click/requests plumbing, so that's all this covers. +""" + +import hashlib +import os +from unittest import mock + +import pytest +from click.testing import CliRunner + +import archivist.__main__ as cli_module +from archivist.__main__ import main + + +@pytest.fixture(autouse=True) +def _restore_config_path_env(): + """ + The CLI's `main()` callback mutates `os.environ` directly (not via + click/pydantic-settings helpers), so make sure it doesn't leak the path + to a tmp_path config file that gets torn down after this test into + later, unrelated tests. + """ + + original = os.environ.get("ARCHIVIST_CONFIG_PATH") + yield + if original is None: + os.environ.pop("ARCHIVIST_CONFIG_PATH", None) + else: + os.environ["ARCHIVIST_CONFIG_PATH"] = original + + +@pytest.fixture +def post(): + """Patch out the HTTP call to the server and hand back the mock.""" + + response = mock.Mock(status_code=200) + response.json.return_value = {"manifest_id": "abc-123"} + + with mock.patch.object(cli_module.requests, "post", return_value=response) as mock_post: + yield mock_post + + +@pytest.mark.parametrize("source_is_dir", [False, True], ids=["single-file", "directory"]) +def test_archive_builds_a_manifest_from_the_source_path(config_path, post, tmp_path, source_is_dir): + """A directory is walked recursively; a file is archived on its own.""" + + source_dir = tmp_path / "adir" + (source_dir / "sub").mkdir(parents=True) + (source_dir / "a.txt").write_bytes(b"a") + (source_dir / "sub" / "b.txt").write_bytes(b"b") + + source = source_dir if source_is_dir else source_dir / "a.txt" + expected_names = ["a.txt", "b.txt"] if source_is_dir else ["a.txt"] + + result = CliRunner().invoke(main, ["-c", str(config_path), "archive", "--source-path", str(source)]) + + assert result.exit_code == 0 + assert "Archive succeeded, manifest_id=abc-123" in result.output + + body = post.call_args.kwargs["json"] + assert body["librarian_name"] == "librarian" + assert sorted(entry["name"] for entry in body["store_files"]) == expected_names + + entry = next(entry for entry in body["store_files"] if entry["name"] == "a.txt") + assert entry["checksum"] == hashlib.sha256(b"a").hexdigest() + + +def test_start_server_hands_the_settings_to_uvicorn(config_path, settings): + with mock.patch("uvicorn.run") as mock_run: + result = CliRunner().invoke(main, ["-c", str(config_path), "start-server"]) + + assert result.exit_code == 0 + mock_run.assert_called_once_with( + "archivist.server:main", + host=settings.host, + port=settings.port, + log_level=settings.log_level.lower(), + factory=True, + ) diff --git a/tests/test_core.py b/tests/test_core.py new file mode 100644 index 0000000..fbec4b5 --- /dev/null +++ b/tests/test_core.py @@ -0,0 +1,88 @@ +# Copyright (c) 2025-2026 Simons Observatory. +# Full license can be found in the top level "LICENSE" file. +import hashlib +from pathlib import Path + +import pytest +from conftest import make_manifest_entry +from pydantic import ValidationError + +from archivist.core.archive import Archive +from archivist.core.models import ManifestEntry, ManifestRequest +from archivist.utils import _checksum + + +@pytest.mark.parametrize( + "data", [b"", b"hello world", b"x" * (1024 * 1024 + 17)], ids=["empty", "small", "multi-chunk"] +) +def test_checksum(tmp_path, data): + path = tmp_path / "file.bin" + path.write_bytes(data) + + assert _checksum(str(path)) == hashlib.sha256(data).hexdigest() + + +def test_manifest_request_accepts_many_or_no_entries(): + request = ManifestRequest( + librarian_name="lib", + store_files=[make_manifest_entry(), make_manifest_entry(name="other.txt")], + ) + + assert request.librarian_name == "lib" + assert len(request.store_files) == 2 + assert isinstance(request.store_files[0], ManifestEntry) + assert isinstance(request.store_files[1], ManifestEntry) + assert ManifestRequest(librarian_name="lib", store_files=[]).store_files == [] + + with pytest.raises(ValidationError): + ManifestRequest(store_files=[]) + + +def test_archive_construction(): + archive = Archive() + + assert archive.manifest is None + assert archive.manifest_id is None + assert archive.local_root is None + assert archive.archive_root is None + assert archive.checksum is None + + manifest = {"store_files": []} + archive = Archive( + manifest=manifest, + manifest_id="abc", + local_root="/local", + archive_root="/archive", + type="posix", + ) + + assert archive.manifest == manifest + assert archive.manifest_id == "abc" + assert archive.local_root == "/local" + assert archive.archive_root == "/archive" + assert "abc" in repr(archive) + assert "posix" in repr(archive) + + +def test_create_archive_posix(): + """The source/destination pairs are what a storage backend actually copies.""" + + manifest = { + "store_files": [ + make_manifest_entry(instance_path="/local/sub/file.txt"), + make_manifest_entry(instance_path="/local/b.txt"), + ] + } + archive = Archive(manifest=manifest, local_root="/local", archive_root="/archive", type="posix") + + pairs = archive.create_archive() + + assert pairs == [ + (Path("/local/sub/file.txt"), Path("/archive/sub/file.txt")), + (Path("/local/b.txt"), Path("/archive/b.txt")), + ] + + +@pytest.mark.parametrize("archive_type", ["hpss", "something-else"]) +def test_create_archive_returns_none_for_unsupported_types(archive_type): + assert Archive(manifest={"store_files": []}, type=archive_type).create_archive() is None diff --git a/tests/test_db.py b/tests/test_db.py new file mode 100644 index 0000000..dc00596 --- /dev/null +++ b/tests/test_db.py @@ -0,0 +1,22 @@ +# Copyright (c) 2025-2026 Simons Observatory. +# Full license can be found in the top level "LICENSE" file. +import time + +from archivist.orm.archivequeue import ArchiveQueue + + +def test_dequeue_takes_the_oldest_unconsumed_item_and_marks_it_consumed(db_session): + for manifest_id in ("first", "second"): + db_session.add(ArchiveQueue.new_item(manifest_id=manifest_id, manifest="{}", paths=["/a"], root="/root")) + db_session.commit() + time.sleep(0.01) + + dequeued = ArchiveQueue.dequeue(db_session) + + assert dequeued.manifest_id == "first" + assert dequeued.consumed + assert dequeued.consumed_time is not None + + # Consumed items are never handed out again, and an empty queue is not an error. + assert ArchiveQueue.dequeue(db_session).manifest_id == "second" + assert ArchiveQueue.dequeue(db_session) is None diff --git a/tests/test_server.py b/tests/test_server.py new file mode 100644 index 0000000..a691afb --- /dev/null +++ b/tests/test_server.py @@ -0,0 +1,77 @@ +# Copyright (c) 2025-2026 Simons Observatory. +# Full license can be found in the top level "LICENSE" file. +import asyncio +import threading +from unittest import mock + +import pytest + +import archivist.server as server_module +from archivist.settings import Settings + + +@pytest.fixture(autouse=True) +def _reset_stop_event(): + """`_archive_stop_event` is module-level shared state; keep it clean per test.""" + + server_module._archive_stop_event.clear() + yield + server_module._archive_stop_event.clear() + + +def test_main_builds_an_app_serving_the_routers(): + """uvicorn loads this factory by name, so a broken one fails only at runtime.""" + + with mock.patch.object(server_module, "server_settings", Settings(debug=True)): + app = server_module.main() + + assert {"/health", "/api/v1/archive"} <= set(app.openapi()["paths"].keys()) + + +@pytest.mark.parametrize( + "loop_name,patch_target", + [ + ("_archive_worker_loop", "archivist.tasks.archive.start_archive"), + ("_status_worker_loop", "archivist.tasks.archive.process_status_queue"), + ], +) +def test_worker_loops_survive_a_failed_iteration_and_stop_on_the_event(loop_name, patch_target): + """These busy-loop forever; an escaping exception would silently kill the worker.""" + + call_count = 0 + + def _fake(*args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1: + raise RuntimeError("boom") + if call_count >= 3: + server_module._archive_stop_event.set() + + with mock.patch(patch_target, side_effect=_fake): + thread = threading.Thread(target=getattr(server_module, loop_name)) + thread.start() + thread.join(timeout=5) + + assert not thread.is_alive() + assert call_count >= 3 + + +def test_lifespan_creates_the_schema_starts_workers_and_stops_them_on_exit(): + started = [] + + async def _run(): + async with server_module.startup_shutdown_server(mock.Mock()): + # Give the submitted worker threads a beat to run. + await asyncio.sleep(0.1) + + with ( + mock.patch.object(server_module, "_archive_worker_loop", lambda: started.append("archive")), + mock.patch.object(server_module, "_status_worker_loop", lambda: started.append("status")), + mock.patch("archivist.database.create_all") as mock_create_all, + ): + asyncio.run(_run()) + + mock_create_all.assert_called_once() + assert sorted(started) == ["archive", "status"] + assert server_module._archive_stop_event.is_set() diff --git a/tests/test_settings.py b/tests/test_settings.py new file mode 100644 index 0000000..b8c7643 --- /dev/null +++ b/tests/test_settings.py @@ -0,0 +1,79 @@ +# Copyright (c) 2025-2026 Simons Observatory. +# Full license can be found in the top level "LICENSE" file. +import json + +import pytest +from pydantic import ValidationError + +import archivist.settings as settings_module +from archivist.settings import Settings, get_settings + + +@pytest.fixture(autouse=True) +def _no_ambient_config(monkeypatch): + """These tests are about config discovery, so start from a clean environment.""" + + monkeypatch.delenv("ARCHIVIST_CONFIG_PATH", raising=False) + + +def test_defaults(): + settings = Settings() + + assert settings.name == "archivist_server" + assert settings.debug is False + assert settings.database_driver == "sqlite" + assert settings.archive_type == "posix" + assert settings.host == "0.0.0.0" + assert settings.port == 8080 + + +def test_sqlite_database_uri(): + uri = Settings(database_driver="sqlite", database="/tmp/foo.db").sqlalchemy_database_uri + + assert uri.drivername == "sqlite" + assert uri.database == "/tmp/foo.db" + + +def test_postgres_database_uri(): + uri = Settings( + database_driver="postgresql", + database_user="user", + database_password="pass", + database_host="localhost", + database_port=5432, + database="archivist", + ).sqlalchemy_database_uri + + assert uri.render_as_string(hide_password=False) == "postgresql://user:pass@localhost:5432/archivist" + + +def test_load_from_file(tmp_path): + config = tmp_path / "config.json" + config.write_text(json.dumps({"name": "my-archivist", "port": 9999})) + + settings = Settings.from_file(config) + + assert settings.name == "my-archivist" + assert settings.port == 9999 + + +def test_from_file_missing_raises(): + with pytest.raises(FileNotFoundError): + Settings.from_file("/nonexistent/path/config.json") + + +def test_get_settings_from_environment_path(monkeypatch, tmp_path): + config = tmp_path / "config.json" + config.write_text(json.dumps({"name": "from-env-config"})) + monkeypatch.setenv("ARCHIVIST_CONFIG_PATH", str(config)) + + assert get_settings().name == "from-env-config" + + +def test_get_settings_raises_on_an_invalid_config_file(monkeypatch, tmp_path): + config = tmp_path / "config.json" + config.write_text(json.dumps({"port": "not-a-port"})) + monkeypatch.setenv("ARCHIVIST_CONFIG_PATH", str(config)) + + with pytest.raises(ValidationError): + get_settings() diff --git a/tests/test_storage.py b/tests/test_storage.py new file mode 100644 index 0000000..7a2b8c0 --- /dev/null +++ b/tests/test_storage.py @@ -0,0 +1,78 @@ +# Copyright (c) 2025-2026 Simons Observatory. +# Full license can be found in the top level "LICENSE" file. +import pytest +from conftest import make_manifest_entry + +from archivist.core.archive import Archive +from archivist.storage import storage_factory +from archivist.storage.base import Storage +from archivist.storage.storage_disk import StorageDisk + + +def _archive_for(source, local_root, archive_root): + return Archive( + manifest={"store_files": [make_manifest_entry(instance_path=str(source))]}, + local_root=str(local_root), + archive_root=str(archive_root), + type="posix", + ) + + +def test_base_storage_notimplemented(): + storage = Storage() + + with pytest.raises(NotImplementedError): + storage.store(archive=None) + + with pytest.raises(NotImplementedError): + storage.extract(archive_name="a", storage_info={}, outdir="/tmp/out") + + +def test_store_returns_the_backend_result_verbatim(): + """`tasks.archive` enqueues whatever `store()` hands back, so it must not be wrapped.""" + + sentinel = object() + + class _RecordingStorage(Storage): + def _store(self, archive): + self.stored = archive + return sentinel + + storage = _RecordingStorage() + + assert storage.store(archive="my-archive") is sentinel + assert storage.stored == "my-archive" + + +def test_storage_factory_maps_configured_types(): + assert storage_factory["posix"] is StorageDisk + + +@pytest.mark.parametrize( + "relative_path", + ["file.txt", "sub/dir/nested.txt"], + ids=["top-level", "nested"], +) +def test_disk_store_archive(use_settings, local_root, archive_root, relative_path): + source = local_root / relative_path + source.parent.mkdir(parents=True, exist_ok=True) + source.write_text("content") + + storage = StorageDisk(settings=use_settings) + task = storage.store(_archive_for(source, local_root, archive_root)) + task.future.result(timeout=5) + + # `store()` returns the backend itself, carrying the archive and its future. + assert task is storage + assert (archive_root / relative_path).read_text() == "content" + + +def test_disk_store_archive_directory_trees(use_settings, local_root, archive_root): + source_dir = local_root / "adir" + source_dir.mkdir() + (source_dir / "inner.txt").write_text("inner-content") + + storage = StorageDisk(settings=use_settings) + storage.store(_archive_for(source_dir, local_root, archive_root)).future.result(timeout=5) + + assert (archive_root / "adir" / "inner.txt").read_text() == "inner-content" diff --git a/tests/test_tasks.py b/tests/test_tasks.py new file mode 100644 index 0000000..e61ca75 --- /dev/null +++ b/tests/test_tasks.py @@ -0,0 +1,135 @@ +# Copyright (c) 2025-2026 Simons Observatory. +# Full license can be found in the top level "LICENSE" file. +import queue as stdlib_queue +import time +from unittest import mock + +import pytest +from conftest import make_manifest_request + +from archivist.core.models import ManifestRequest +from archivist.orm.archivequeue import ArchiveQueue +from archivist.queue import Queue, get_status_queue +from archivist.tasks.archive import process_status_queue, start_archive + + +class _FakeStorageTask: + """Stands in for a storage backend's return value, without doing any I/O.""" + + def __init__(self, done, manifest_id="m1"): + self.future = mock.Mock(done=mock.Mock(return_value=done)) + self._archive = mock.Mock(manifest_id=manifest_id) + + +@pytest.fixture +def enqueue_manifest(db_session, archive_root): + """Queue a real archive job for `source_path` and return its manifest id.""" + + def _enqueue(source_path, manifest_id="m1"): + request = ManifestRequest(**make_manifest_request(instance_path=str(source_path))) + db_session.add( + ArchiveQueue.new_item( + manifest_id=manifest_id, + manifest=request.model_dump_json(), + paths=[str(source_path)], + root=str(archive_root), + ) + ) + db_session.commit() + return manifest_id + + return _enqueue + + +def _drain_until_completed(session, manifest_id, timeout=5): + """Poll the status worker until the queued row is closed out (or we give up).""" + + deadline = time.time() + timeout + while time.time() < deadline: + process_status_queue() + session.expire_all() + item = session.query(ArchiveQueue).filter_by(manifest_id=manifest_id).one() + if item.completed: + return item + time.sleep(0.05) + + return item + + +def test_queue_is_fifo(): + q = Queue() + assert q.size == 0 + + q.enqueue("first") + q.enqueue("second") + assert q.size == 2 + assert [q.dequeue(), q.dequeue()] == ["first", "second"] + q.task_done() + + with pytest.raises(stdlib_queue.Empty): + q.dequeue(block=False) + + +def test_start_archive_consumes_the_row_and_copies_the_file(db_session, enqueue_manifest, local_root, archive_root): + # An empty queue is the common case in the worker's busy-loop; it must be a no-op. + start_archive(librarian_name="test-librarian") + assert get_status_queue().size == 0 + + source = local_root / "file.txt" + source.write_text("archived-content") + enqueue_manifest(source) + + start_archive(librarian_name="test-librarian") + assert db_session.query(ArchiveQueue).filter_by(manifest_id="m1").one().consumed + + task = get_status_queue().dequeue(block=False) + task.future.result(timeout=5) + assert (archive_root / "file.txt").read_text() == "archived-content" + + +def test_process_status_queue_marks_rows_completed_or_failed(db_session, enqueue_manifest, local_root): + """Either way the row must end up `completed`; `failed` is what distinguishes them.""" + + source = local_root / "file.txt" + source.write_text("hello") + + enqueue_manifest(source, manifest_id="stored-ok") + start_archive(librarian_name="test-librarian") + item = _drain_until_completed(db_session, "stored-ok") + + assert item.completed + assert not item.failed + + enqueue_manifest(source, manifest_id="storing-raised") + start_archive(librarian_name="test-librarian") + with mock.patch.object(ArchiveQueue, "complete", side_effect=RuntimeError("boom")): + item = _drain_until_completed(db_session, "storing-raised") + + assert item.completed + assert item.failed + + +def test_process_status_queue_handles_tasks_it_cannot_close_out(db_session, use_settings): + """ + The status worker busy-loops on this, so nothing here may raise: a task + whose future is still running goes back on the queue, and one that can no + longer be matched to a row (or is broken outright) is dropped. + """ + + status_queue = get_status_queue() + + status_queue.enqueue(_FakeStorageTask(done=False)) + process_status_queue() + assert status_queue.size == 1 + status_queue.dequeue(block=False) + + status_queue.enqueue(_FakeStorageTask(done=True, manifest_id="no-such-manifest")) + process_status_queue() + assert status_queue.size == 0 + + broken = mock.Mock() + type(broken).future = mock.PropertyMock(side_effect=RuntimeError("boom")) + status_queue.enqueue(broken) + with mock.patch("archivist.tasks.archive.sleep"): # skip the error path's back-off + process_status_queue() + assert status_queue.size == 0