From f3e345b834f31b600375dd8179f5886097631228 Mon Sep 17 00:00:00 2001 From: elmotec <1107551+elmotec@users.noreply.github.com> Date: Sat, 24 Jan 2026 17:00:17 -0500 Subject: [PATCH 1/5] feat: pass custom options to restic backup add stack-back.restic.backup.options label to the stack-back container to take `restic backup` options. Defaults to `--verbose` for backward compatibility. Also: - add LABEL_RESTIC_BACKUP_OPTIONS as enum. - feature works for both volumes and database backup. - change backup(), backup_from_stdin() signature to take restic_backup_options as argument. - add --tag test-tag to test compose file and integration_test. - enhance integration_test parsing of snapshots (could use json, but less readable). - improve visibility of logs when integration tests run showing output of restic command in pytest log (with --log-cli-level=DEBUG). - documentation. --- README.md | 2 + docker-compose.test.yaml | 2 + docs/guide/configuration.rst | 20 ++++++ src/restic_compose_backup/cli.py | 16 ++++- src/restic_compose_backup/containers.py | 3 +- src/restic_compose_backup/containers_db.py | 9 ++- src/restic_compose_backup/enums.py | 1 + src/restic_compose_backup/restic.py | 8 +-- src/tests/integration/conftest.py | 7 ++- src/tests/integration/test_volume_backups.py | 66 +++++++++++++++++++- 10 files changed, 120 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index e72ecc7..e56171b 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,8 @@ uv run --directory src/ pytest -m integration uv run --directory src/ pytest ``` +**Note:** Use `-vs --log-cli-level=DEBUG` pytest options to get detailed progress when running the tests. + **Note:** Integration tests require Docker and docker compose plugin, and will spin up real database containers. They take significantly longer than unit tests. ## Building Docs diff --git a/docker-compose.test.yaml b/docker-compose.test.yaml index da69e85..02e1398 100644 --- a/docker-compose.test.yaml +++ b/docker-compose.test.yaml @@ -18,6 +18,8 @@ services: - backup backup: build: ./src + labels: + stack-back.restic.backup.options: "--tag test-tag" environment: - DOCKER_HOST=tcp://socket-proxy:2375 - RESTIC_REPOSITORY=/restic_data diff --git a/docs/guide/configuration.rst b/docs/guide/configuration.rst index dad4414..a136475 100644 --- a/docs/guide/configuration.rst +++ b/docs/guide/configuration.rst @@ -402,6 +402,26 @@ Exclude example achieving the same result as the example above. The ``exclude`` and ``include`` tag can be used together in more complex situations. +Restic Backup Options +~~~~~~~~~~~~~~~~~~~~~~ + +Additional restic backup options can be passed by adding the +``stack-back.restic.backup.options`` label to the backup service. +Defaults to ``--verbose``. + +Example: + +.. code:: yaml + + backup: + image: ghcr.io/lawndoc/stack-back:latest + labels: + stack-back.restic.backup.options: "--tag production --verbose" + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + +This applies to both volume and database backups. + mariadb ~~~~~~~ diff --git a/src/restic_compose_backup/cli.py b/src/restic_compose_backup/cli.py index e395e02..d9add1e 100644 --- a/src/restic_compose_backup/cli.py +++ b/src/restic_compose_backup/cli.py @@ -5,6 +5,7 @@ from restic_compose_backup import ( alerts, backup_runner, + enums, log, restic, ) @@ -234,11 +235,20 @@ def start_backup_process(config, containers): if len(containers.stop_during_backup_containers) > 0: utils.stop_containers(containers.stop_during_backup_containers) + restic_backup_options = ["--verbose"] + backup_args_label = containers.this_container.get_label(enums.LABEL_RESTIC_BACKUP_OPTIONS) + if backup_args_label: + restic_backup_options.extend(backup_args_label.split()) + # back up volumes if has_volumes: try: - logger.info("Backing up volumes") - vol_result = restic.backup_files(config.repository, source="/volumes") + logger.info("Backing up volumes with arguments: %s", restic_backup_options) + vol_result = restic.backup_files( + config.repository, + restic_backup_options=restic_backup_options, + source="/volumes", + ) logger.debug("Volume backup exit code: %s", vol_result) if vol_result != 0: logger.error("Volume backup exited with non-zero code: %s", vol_result) @@ -260,7 +270,7 @@ def start_backup_process(config, containers): instance.service_name, instance.project_name, ) - result = instance.backup() + result = instance.backup(restic_backup_options=restic_backup_options) logger.debug("Exit code: %s", result) if result != 0: logger.error("Backup command exited with non-zero code: %s", result) diff --git a/src/restic_compose_backup/containers.py b/src/restic_compose_backup/containers.py index 55ee256..65e1fe6 100644 --- a/src/restic_compose_backup/containers.py +++ b/src/restic_compose_backup/containers.py @@ -1,7 +1,6 @@ import logging from pathlib import Path import socket -from typing import List from restic_compose_backup import enums, utils from restic_compose_backup.config import config @@ -304,7 +303,7 @@ def ping(self) -> bool: """Check the availability of the service""" raise NotImplementedError("Base container class don't implement this") - def backup(self): + def backup(self, restic_backup_options: list[str]) -> int: """Back up this service""" raise NotImplementedError("Base container class don't implement this") diff --git a/src/restic_compose_backup/containers_db.py b/src/restic_compose_backup/containers_db.py index 4054c6c..e8521ed 100644 --- a/src/restic_compose_backup/containers_db.py +++ b/src/restic_compose_backup/containers_db.py @@ -56,7 +56,7 @@ def dump_command(self) -> list: "--force", ] - def backup(self): + def backup(self, restic_backup_options: list[str]) -> int: config = Config() creds = self.get_credentials() @@ -65,6 +65,7 @@ def backup(self): self.backup_destination_path(), self.id, self.dump_command(), + restic_backup_options=restic_backup_options, environment={"MYSQL_PWD": creds["password"]}, ) @@ -129,7 +130,7 @@ def dump_command(self) -> list: "--force", ] - def backup(self): + def backup(self, restic_backup_options: list[str]) -> int: config = Config() creds = self.get_credentials() @@ -138,6 +139,7 @@ def backup(self): self.backup_destination_path(), self.id, self.dump_command(), + restic_backup_options=restic_backup_options, environment={"MYSQL_PWD": creds["password"]}, ) @@ -192,7 +194,7 @@ def dump_command(self) -> list: creds["database"], ] - def backup(self): + def backup(self, restic_backup_options: list[str]) -> int: config = Config() creds = self.get_credentials() @@ -201,6 +203,7 @@ def backup(self): self.backup_destination_path(), self.id, self.dump_command(), + restic_backup_options=restic_backup_options, ) def backup_destination_path(self) -> str: diff --git a/src/restic_compose_backup/enums.py b/src/restic_compose_backup/enums.py index 788da14..4f3e4a7 100644 --- a/src/restic_compose_backup/enums.py +++ b/src/restic_compose_backup/enums.py @@ -9,3 +9,4 @@ LABEL_MARIADB_ENABLED = "stack-back.mariadb" LABEL_BACKUP_PROCESS = "stack-back.process" +LABEL_RESTIC_BACKUP_OPTIONS = "stack-back.restic.backup.options" diff --git a/src/restic_compose_backup/restic.py b/src/restic_compose_backup/restic.py index 77e9cb5..60bce5f 100644 --- a/src/restic_compose_backup/restic.py +++ b/src/restic_compose_backup/restic.py @@ -25,15 +25,14 @@ def init_repo(repository: str): ) -def backup_files(repository: str, source="/volumes"): +def backup_files(repository: str, restic_backup_options: List[str], source="/volumes"): return commands.run( restic( repository, [ - "--verbose", "backup", source, - ], + ] + restic_backup_options, ) ) @@ -43,6 +42,7 @@ def backup_from_stdin( filename: str, container_id: str, source_command: List[str], + restic_backup_options: List[str], environment: Union[dict, list] = None, ): """ @@ -56,7 +56,7 @@ def backup_from_stdin( "--stdin", "--stdin-filename", filename, - ], + ] + restic_backup_options, ) client = utils.docker_client() diff --git a/src/tests/integration/conftest.py b/src/tests/integration/conftest.py index 7af4b44..09049eb 100644 --- a/src/tests/integration/conftest.py +++ b/src/tests/integration/conftest.py @@ -1,11 +1,14 @@ """Pytest fixtures for integration tests""" +import logging import subprocess import time import pytest import docker from pathlib import Path +logger = logging.getLogger(__name__) + @pytest.fixture(scope="session") def docker_client(): @@ -179,7 +182,9 @@ def run_rcb_command(backup_container): def _run_command(command: str): full_command = f"rcb {command}" exit_code, output = backup_container.exec_run(full_command) - return exit_code, output.decode() + decoded_output = output.decode() + logger.debug("Command '%s' output:\n%s", full_command, decoded_output) + return exit_code, decoded_output return _run_command diff --git a/src/tests/integration/test_volume_backups.py b/src/tests/integration/test_volume_backups.py index ff6188d..7aa472a 100644 --- a/src/tests/integration/test_volume_backups.py +++ b/src/tests/integration/test_volume_backups.py @@ -1,11 +1,72 @@ """Integration tests for volume backups""" import time +import dataclasses +import re import pytest pytestmark = pytest.mark.integration +@dataclasses.dataclass +class Snapshot: + """Represents a restic snapshot""" + id: str + time: str + host: str + tags: str + paths: str + + +def parse_snapshots(output: str) -> list[Snapshot]: + """Parse restic snapshots output into Snapshot objects""" + lines = output.split("\n") + + # Find the header line to determine column positions + header_idx = -1 + for i, line in enumerate(lines): + if line.startswith("ID"): + header_idx = i + break + + if header_idx == -1: + return [] + + header = lines[header_idx] + # Find column positions based on header + id_pos = header.index("ID") + time_pos = header.index("Time") if "Time" in header else -1 + host_pos = header.index("Host") if "Host" in header else -1 + tags_pos = header.index("Tags") if "Tags" in header else -1 + paths_pos = header.index("Paths") if "Paths" in header else -1 + + snapshots = [] + # Process lines after the separator line + for line in lines[header_idx + 2:]: + if not line.strip() or line.startswith("-"): + continue + if re.match(r"^\d+ snapshots", line): + break + + # Extract values based on column positions + id_val = line[id_pos:time_pos].strip() if time_pos > 0 else line[id_pos:].strip() + time_val = line[time_pos:host_pos].strip() if time_pos > 0 and host_pos > 0 else "" + host_val = line[host_pos:tags_pos].strip() if host_pos > 0 and tags_pos > 0 else "" + tags_val = line[tags_pos:paths_pos].strip() if tags_pos > 0 and paths_pos > 0 else "" + paths_val = line[paths_pos:].strip() if paths_pos > 0 else "" + + if id_val: + snapshots.append(Snapshot( + id=id_val, + time=time_val, + host=host_val, + tags=tags_val, + paths=paths_val, + )) + + return snapshots + + def test_backup_status(run_rcb_command): """Test that the status command works""" exit_code, output = run_rcb_command("status") @@ -30,7 +91,10 @@ def test_backup_bind_mount(run_rcb_command, create_test_data, backup_container): # Check that snapshots were created exit_code, output = run_rcb_command("snapshots") assert exit_code == 0, f"Snapshots command failed: {output}" - assert len(output.strip().split("\n")) > 1, "No snapshots found" + + snapshots = parse_snapshots(output) + assert len(snapshots) == 4, f"Expected 4 snapshots, found\n{output}" + assert all("test-tag" in s.tags for s in snapshots), f"Not all snapshots have 'test-tag':\n{output}" def test_restore_bind_mount( From ab2fcfa5c3d322e06ea0a171d3ad50045801aa22 Mon Sep 17 00:00:00 2001 From: elmotec <1107551+elmotec@users.noreply.github.com> Date: Tue, 27 Jan 2026 18:38:12 -0500 Subject: [PATCH 2/5] replace restic backup options instead of extending Per https://github.com/lawndoc/stack-back/pull/101#discussion_r2724680650. It does feel less surprising. Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/restic_compose_backup/cli.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/restic_compose_backup/cli.py b/src/restic_compose_backup/cli.py index d9add1e..5244413 100644 --- a/src/restic_compose_backup/cli.py +++ b/src/restic_compose_backup/cli.py @@ -235,10 +235,8 @@ def start_backup_process(config, containers): if len(containers.stop_during_backup_containers) > 0: utils.stop_containers(containers.stop_during_backup_containers) - restic_backup_options = ["--verbose"] backup_args_label = containers.this_container.get_label(enums.LABEL_RESTIC_BACKUP_OPTIONS) - if backup_args_label: - restic_backup_options.extend(backup_args_label.split()) + restic_backup_options = backup_args_label.split() if backup_args_label else ["--verbose"] # back up volumes if has_volumes: From 4160c5453957597774b7d6a7a00e76b723558af7 Mon Sep 17 00:00:00 2001 From: elmotec <1107551+elmotec@users.noreply.github.com> Date: Sun, 8 Feb 2026 15:14:30 -0500 Subject: [PATCH 3/5] format: apply `uv run ruff format .` in src/ --- src/restic_compose_backup/cli.py | 8 ++- src/restic_compose_backup/restic.py | 6 ++- src/tests/integration/test_volume_backups.py | 55 ++++++++++++-------- 3 files changed, 44 insertions(+), 25 deletions(-) diff --git a/src/restic_compose_backup/cli.py b/src/restic_compose_backup/cli.py index 5244413..219e26e 100644 --- a/src/restic_compose_backup/cli.py +++ b/src/restic_compose_backup/cli.py @@ -235,8 +235,12 @@ def start_backup_process(config, containers): if len(containers.stop_during_backup_containers) > 0: utils.stop_containers(containers.stop_during_backup_containers) - backup_args_label = containers.this_container.get_label(enums.LABEL_RESTIC_BACKUP_OPTIONS) - restic_backup_options = backup_args_label.split() if backup_args_label else ["--verbose"] + backup_args_label = containers.this_container.get_label( + enums.LABEL_RESTIC_BACKUP_OPTIONS + ) + restic_backup_options = ( + backup_args_label.split() if backup_args_label else ["--verbose"] + ) # back up volumes if has_volumes: diff --git a/src/restic_compose_backup/restic.py b/src/restic_compose_backup/restic.py index 60bce5f..df150bf 100644 --- a/src/restic_compose_backup/restic.py +++ b/src/restic_compose_backup/restic.py @@ -32,7 +32,8 @@ def backup_files(repository: str, restic_backup_options: List[str], source="/vol [ "backup", source, - ] + restic_backup_options, + ] + + restic_backup_options, ) ) @@ -56,7 +57,8 @@ def backup_from_stdin( "--stdin", "--stdin-filename", filename, - ] + restic_backup_options, + ] + + restic_backup_options, ) client = utils.docker_client() diff --git a/src/tests/integration/test_volume_backups.py b/src/tests/integration/test_volume_backups.py index 7aa472a..3af5b2e 100644 --- a/src/tests/integration/test_volume_backups.py +++ b/src/tests/integration/test_volume_backups.py @@ -11,6 +11,7 @@ @dataclasses.dataclass class Snapshot: """Represents a restic snapshot""" + id: str time: str host: str @@ -21,17 +22,17 @@ class Snapshot: def parse_snapshots(output: str) -> list[Snapshot]: """Parse restic snapshots output into Snapshot objects""" lines = output.split("\n") - + # Find the header line to determine column positions header_idx = -1 for i, line in enumerate(lines): if line.startswith("ID"): header_idx = i break - + if header_idx == -1: return [] - + header = lines[header_idx] # Find column positions based on header id_pos = header.index("ID") @@ -39,31 +40,41 @@ def parse_snapshots(output: str) -> list[Snapshot]: host_pos = header.index("Host") if "Host" in header else -1 tags_pos = header.index("Tags") if "Tags" in header else -1 paths_pos = header.index("Paths") if "Paths" in header else -1 - + snapshots = [] # Process lines after the separator line - for line in lines[header_idx + 2:]: + for line in lines[header_idx + 2 :]: if not line.strip() or line.startswith("-"): continue if re.match(r"^\d+ snapshots", line): break - + # Extract values based on column positions - id_val = line[id_pos:time_pos].strip() if time_pos > 0 else line[id_pos:].strip() - time_val = line[time_pos:host_pos].strip() if time_pos > 0 and host_pos > 0 else "" - host_val = line[host_pos:tags_pos].strip() if host_pos > 0 and tags_pos > 0 else "" - tags_val = line[tags_pos:paths_pos].strip() if tags_pos > 0 and paths_pos > 0 else "" + id_val = ( + line[id_pos:time_pos].strip() if time_pos > 0 else line[id_pos:].strip() + ) + time_val = ( + line[time_pos:host_pos].strip() if time_pos > 0 and host_pos > 0 else "" + ) + host_val = ( + line[host_pos:tags_pos].strip() if host_pos > 0 and tags_pos > 0 else "" + ) + tags_val = ( + line[tags_pos:paths_pos].strip() if tags_pos > 0 and paths_pos > 0 else "" + ) paths_val = line[paths_pos:].strip() if paths_pos > 0 else "" - + if id_val: - snapshots.append(Snapshot( - id=id_val, - time=time_val, - host=host_val, - tags=tags_val, - paths=paths_val, - )) - + snapshots.append( + Snapshot( + id=id_val, + time=time_val, + host=host_val, + tags=tags_val, + paths=paths_val, + ) + ) + return snapshots @@ -91,10 +102,12 @@ def test_backup_bind_mount(run_rcb_command, create_test_data, backup_container): # Check that snapshots were created exit_code, output = run_rcb_command("snapshots") assert exit_code == 0, f"Snapshots command failed: {output}" - + snapshots = parse_snapshots(output) assert len(snapshots) == 4, f"Expected 4 snapshots, found\n{output}" - assert all("test-tag" in s.tags for s in snapshots), f"Not all snapshots have 'test-tag':\n{output}" + assert all("test-tag" in s.tags for s in snapshots), ( + f"Not all snapshots have 'test-tag':\n{output}" + ) def test_restore_bind_mount( From 57fdbdd854283b5f8eb45efe18767842e2b08a47 Mon Sep 17 00:00:00 2001 From: elmotec <1107551+elmotec@users.noreply.github.com> Date: Sun, 8 Feb 2026 15:21:38 -0500 Subject: [PATCH 4/5] fix: restored dropped --verbose option See https://github.com/lawndoc/stack-back/pull/101#discussion_r2776931174 --- docker-compose.test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.test.yaml b/docker-compose.test.yaml index 02e1398..4bb3251 100644 --- a/docker-compose.test.yaml +++ b/docker-compose.test.yaml @@ -19,7 +19,7 @@ services: backup: build: ./src labels: - stack-back.restic.backup.options: "--tag test-tag" + stack-back.restic.backup.options: --verbose --tag test-tag environment: - DOCKER_HOST=tcp://socket-proxy:2375 - RESTIC_REPOSITORY=/restic_data From a63e3327b44f5f6c7e55d2cae441c850b3f0042f Mon Sep 17 00:00:00 2001 From: elmotec <1107551+elmotec@users.noreply.github.com> Date: Tue, 17 Feb 2026 07:56:03 -0500 Subject: [PATCH 5/5] docs: make clear the options are appended after backup Per https://github.com/lawndoc/stack-back/pull/101#discussion_r2724680668 --- docs/guide/configuration.rst | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/docs/guide/configuration.rst b/docs/guide/configuration.rst index a136475..4e6a618 100644 --- a/docs/guide/configuration.rst +++ b/docs/guide/configuration.rst @@ -403,11 +403,14 @@ The ``exclude`` and ``include`` tag can be used together in more complex situations. Restic Backup Options -~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~ + +Additional restic options can be passed to the ``rcb backup`` sub-command +by adding the ``stack-back.restic.backup.options`` +label to the backup service. The value of this label is appended as-is at +the end of the underlying ``restic backup`` command. -Additional restic backup options can be passed by adding the -``stack-back.restic.backup.options`` label to the backup service. -Defaults to ``--verbose``. +The option defaults to ``--verbose``. Example: @@ -420,7 +423,15 @@ Example: volumes: - /var/run/docker.sock:/var/run/docker.sock:ro -This applies to both volume and database backups. + +With the above example configuration, the backup will be executed as: + +.. code:: bash + + restic backup --tag production --verbose + + +It applies to both volume and database backups. mariadb ~~~~~~~