diff --git a/README.md b/README.md index e72ecc7..e56171b 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,8 @@ uv run --directory src/ pytest -m integration uv run --directory src/ pytest ``` +**Note:** Use `-vs --log-cli-level=DEBUG` pytest options to get detailed progress when running the tests. + **Note:** Integration tests require Docker and docker compose plugin, and will spin up real database containers. They take significantly longer than unit tests. ## Building Docs diff --git a/docker-compose.test.yaml b/docker-compose.test.yaml index da69e85..4bb3251 100644 --- a/docker-compose.test.yaml +++ b/docker-compose.test.yaml @@ -18,6 +18,8 @@ services: - backup backup: build: ./src + labels: + stack-back.restic.backup.options: --verbose --tag test-tag environment: - DOCKER_HOST=tcp://socket-proxy:2375 - RESTIC_REPOSITORY=/restic_data diff --git a/docs/guide/configuration.rst b/docs/guide/configuration.rst index dad4414..4e6a618 100644 --- a/docs/guide/configuration.rst +++ b/docs/guide/configuration.rst @@ -402,6 +402,37 @@ Exclude example achieving the same result as the example above. The ``exclude`` and ``include`` tag can be used together in more complex situations. +Restic Backup Options +~~~~~~~~~~~~~~~~~~~~~ + +Additional restic options can be passed to the ``rcb backup`` sub-command +by adding the ``stack-back.restic.backup.options`` +label to the backup service. The value of this label is appended as-is at +the end of the underlying ``restic backup`` command. + +The option defaults to ``--verbose``. + +Example: + +.. code:: yaml + + backup: + image: ghcr.io/lawndoc/stack-back:latest + labels: + stack-back.restic.backup.options: "--tag production --verbose" + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + + +With the above example configuration, the backup will be executed as: + +.. code:: bash + + restic backup --tag production --verbose + + +It applies to both volume and database backups. + mariadb ~~~~~~~ diff --git a/src/restic_compose_backup/cli.py b/src/restic_compose_backup/cli.py index e395e02..219e26e 100644 --- a/src/restic_compose_backup/cli.py +++ b/src/restic_compose_backup/cli.py @@ -5,6 +5,7 @@ from restic_compose_backup import ( alerts, backup_runner, + enums, log, restic, ) @@ -234,11 +235,22 @@ def start_backup_process(config, containers): if len(containers.stop_during_backup_containers) > 0: utils.stop_containers(containers.stop_during_backup_containers) + backup_args_label = containers.this_container.get_label( + enums.LABEL_RESTIC_BACKUP_OPTIONS + ) + restic_backup_options = ( + backup_args_label.split() if backup_args_label else ["--verbose"] + ) + # back up volumes if has_volumes: try: - logger.info("Backing up volumes") - vol_result = restic.backup_files(config.repository, source="/volumes") + logger.info("Backing up volumes with arguments: %s", restic_backup_options) + vol_result = restic.backup_files( + config.repository, + restic_backup_options=restic_backup_options, + source="/volumes", + ) logger.debug("Volume backup exit code: %s", vol_result) if vol_result != 0: logger.error("Volume backup exited with non-zero code: %s", vol_result) @@ -260,7 +272,7 @@ def start_backup_process(config, containers): instance.service_name, instance.project_name, ) - result = instance.backup() + result = instance.backup(restic_backup_options=restic_backup_options) logger.debug("Exit code: %s", result) if result != 0: logger.error("Backup command exited with non-zero code: %s", result) diff --git a/src/restic_compose_backup/containers.py b/src/restic_compose_backup/containers.py index 55ee256..65e1fe6 100644 --- a/src/restic_compose_backup/containers.py +++ b/src/restic_compose_backup/containers.py @@ -1,7 +1,6 @@ import logging from pathlib import Path import socket -from typing import List from restic_compose_backup import enums, utils from restic_compose_backup.config import config @@ -304,7 +303,7 @@ def ping(self) -> bool: """Check the availability of the service""" raise NotImplementedError("Base container class don't implement this") - def backup(self): + def backup(self, restic_backup_options: list[str]) -> int: """Back up this service""" raise NotImplementedError("Base container class don't implement this") diff --git a/src/restic_compose_backup/containers_db.py b/src/restic_compose_backup/containers_db.py index 4054c6c..e8521ed 100644 --- a/src/restic_compose_backup/containers_db.py +++ b/src/restic_compose_backup/containers_db.py @@ -56,7 +56,7 @@ def dump_command(self) -> list: "--force", ] - def backup(self): + def backup(self, restic_backup_options: list[str]) -> int: config = Config() creds = self.get_credentials() @@ -65,6 +65,7 @@ def backup(self): self.backup_destination_path(), self.id, self.dump_command(), + restic_backup_options=restic_backup_options, environment={"MYSQL_PWD": creds["password"]}, ) @@ -129,7 +130,7 @@ def dump_command(self) -> list: "--force", ] - def backup(self): + def backup(self, restic_backup_options: list[str]) -> int: config = Config() creds = self.get_credentials() @@ -138,6 +139,7 @@ def backup(self): self.backup_destination_path(), self.id, self.dump_command(), + restic_backup_options=restic_backup_options, environment={"MYSQL_PWD": creds["password"]}, ) @@ -192,7 +194,7 @@ def dump_command(self) -> list: creds["database"], ] - def backup(self): + def backup(self, restic_backup_options: list[str]) -> int: config = Config() creds = self.get_credentials() @@ -201,6 +203,7 @@ def backup(self): self.backup_destination_path(), self.id, self.dump_command(), + restic_backup_options=restic_backup_options, ) def backup_destination_path(self) -> str: diff --git a/src/restic_compose_backup/enums.py b/src/restic_compose_backup/enums.py index 788da14..4f3e4a7 100644 --- a/src/restic_compose_backup/enums.py +++ b/src/restic_compose_backup/enums.py @@ -9,3 +9,4 @@ LABEL_MARIADB_ENABLED = "stack-back.mariadb" LABEL_BACKUP_PROCESS = "stack-back.process" +LABEL_RESTIC_BACKUP_OPTIONS = "stack-back.restic.backup.options" diff --git a/src/restic_compose_backup/restic.py b/src/restic_compose_backup/restic.py index 77e9cb5..df150bf 100644 --- a/src/restic_compose_backup/restic.py +++ b/src/restic_compose_backup/restic.py @@ -25,15 +25,15 @@ def init_repo(repository: str): ) -def backup_files(repository: str, source="/volumes"): +def backup_files(repository: str, restic_backup_options: List[str], source="/volumes"): return commands.run( restic( repository, [ - "--verbose", "backup", source, - ], + ] + + restic_backup_options, ) ) @@ -43,6 +43,7 @@ def backup_from_stdin( filename: str, container_id: str, source_command: List[str], + restic_backup_options: List[str], environment: Union[dict, list] = None, ): """ @@ -56,7 +57,8 @@ def backup_from_stdin( "--stdin", "--stdin-filename", filename, - ], + ] + + restic_backup_options, ) client = utils.docker_client() diff --git a/src/tests/integration/conftest.py b/src/tests/integration/conftest.py index 7af4b44..09049eb 100644 --- a/src/tests/integration/conftest.py +++ b/src/tests/integration/conftest.py @@ -1,11 +1,14 @@ """Pytest fixtures for integration tests""" +import logging import subprocess import time import pytest import docker from pathlib import Path +logger = logging.getLogger(__name__) + @pytest.fixture(scope="session") def docker_client(): @@ -179,7 +182,9 @@ def run_rcb_command(backup_container): def _run_command(command: str): full_command = f"rcb {command}" exit_code, output = backup_container.exec_run(full_command) - return exit_code, output.decode() + decoded_output = output.decode() + logger.debug("Command '%s' output:\n%s", full_command, decoded_output) + return exit_code, decoded_output return _run_command diff --git a/src/tests/integration/test_volume_backups.py b/src/tests/integration/test_volume_backups.py index ff6188d..3af5b2e 100644 --- a/src/tests/integration/test_volume_backups.py +++ b/src/tests/integration/test_volume_backups.py @@ -1,11 +1,83 @@ """Integration tests for volume backups""" import time +import dataclasses +import re import pytest pytestmark = pytest.mark.integration +@dataclasses.dataclass +class Snapshot: + """Represents a restic snapshot""" + + id: str + time: str + host: str + tags: str + paths: str + + +def parse_snapshots(output: str) -> list[Snapshot]: + """Parse restic snapshots output into Snapshot objects""" + lines = output.split("\n") + + # Find the header line to determine column positions + header_idx = -1 + for i, line in enumerate(lines): + if line.startswith("ID"): + header_idx = i + break + + if header_idx == -1: + return [] + + header = lines[header_idx] + # Find column positions based on header + id_pos = header.index("ID") + time_pos = header.index("Time") if "Time" in header else -1 + host_pos = header.index("Host") if "Host" in header else -1 + tags_pos = header.index("Tags") if "Tags" in header else -1 + paths_pos = header.index("Paths") if "Paths" in header else -1 + + snapshots = [] + # Process lines after the separator line + for line in lines[header_idx + 2 :]: + if not line.strip() or line.startswith("-"): + continue + if re.match(r"^\d+ snapshots", line): + break + + # Extract values based on column positions + id_val = ( + line[id_pos:time_pos].strip() if time_pos > 0 else line[id_pos:].strip() + ) + time_val = ( + line[time_pos:host_pos].strip() if time_pos > 0 and host_pos > 0 else "" + ) + host_val = ( + line[host_pos:tags_pos].strip() if host_pos > 0 and tags_pos > 0 else "" + ) + tags_val = ( + line[tags_pos:paths_pos].strip() if tags_pos > 0 and paths_pos > 0 else "" + ) + paths_val = line[paths_pos:].strip() if paths_pos > 0 else "" + + if id_val: + snapshots.append( + Snapshot( + id=id_val, + time=time_val, + host=host_val, + tags=tags_val, + paths=paths_val, + ) + ) + + return snapshots + + def test_backup_status(run_rcb_command): """Test that the status command works""" exit_code, output = run_rcb_command("status") @@ -30,7 +102,12 @@ def test_backup_bind_mount(run_rcb_command, create_test_data, backup_container): # Check that snapshots were created exit_code, output = run_rcb_command("snapshots") assert exit_code == 0, f"Snapshots command failed: {output}" - assert len(output.strip().split("\n")) > 1, "No snapshots found" + + snapshots = parse_snapshots(output) + assert len(snapshots) == 4, f"Expected 4 snapshots, found\n{output}" + assert all("test-tag" in s.tags for s in snapshots), ( + f"Not all snapshots have 'test-tag':\n{output}" + ) def test_restore_bind_mount(