From b1ae81729ddc974601d4828a96870c30e1085b21 Mon Sep 17 00:00:00 2001 From: muebau Date: Tue, 17 Mar 2026 09:22:53 +0100 Subject: [PATCH] feat: pre/post/error/finally lifecycle hooks and ordered backup (#108) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add user-defined hook commands that run around the backup process, following try/catch/finally semantics: - pre: runs before backup starts - post: runs only if pre hooks + backup both succeeded - error: runs only on failure - finally: always runs regardless of outcome Labels on any container: stack-back.hooks...cmd command to run stack-back.hooks...context service to exec into (optional) stack-back.hooks...on-error "abort" (default) | "continue" When context is omitted, the hook runs in the container where the label is defined: backup-container hooks run locally via sh -c, target- container hooks run via docker exec into that target container. An explicit context overrides this, exec-ing into the named service. For pre hooks (setup), backup-container hooks execute first, then target-container hooks. For post/error/finally hooks (teardown), the order is reversed: target-container hooks first, then backup-container hooks — giving symmetric stack-like semantics. The backup lifecycle in start_backup_process() is restructured so that stopped containers are always restarted before any post/error/finally hooks execute, ensuring hooks can exec into previously-stopped services. Also adds ordered backup execution via labels on the backup container: stack-back.ordered: true stack-back.order.: "service-name" Containers are backed up in the defined sequence. Containers not listed are appended after the ordered ones. Duplicate, unknown, and missing entries are handled with warnings. Hook and ordering labels are forwarded to the spawned backup process container so the inner process can read them. New modules: - hooks.py: hook parsing, collection, ordering, and execution Changed modules: - cli.py: lifecycle restructured with try/finally, hook integration, label forwarding, hook summary in status output - containers.py: ordered backup via _parse_backup_order() - enums.py: LABEL_ORDERED, LABEL_ORDER_PREFIX constants - fixtures.py: forward "env" field in test container definitions New test files: - test_hooks.py: hook parsing, collection ordering, execution, context resolution, abort/continue, backup ordering - test_backup_lifecycle.py: operation sequencing with mocked hooks - test_end_to_end_lifecycle.py: full pipeline with real Container and Hook objects, mocked only at the I/O boundary --- src/restic_compose_backup/cli.py | 839 ++++++++------- src/restic_compose_backup/containers.py | 1071 ++++++++++--------- src/restic_compose_backup/enums.py | 25 +- src/restic_compose_backup/hooks.py | 265 +++++ src/tests/unit/fixtures.py | 121 +-- src/tests/unit/test_backup_lifecycle.py | 291 +++++ src/tests/unit/test_end_to_end_lifecycle.py | 938 ++++++++++++++++ src/tests/unit/test_hooks.py | 559 ++++++++++ 8 files changed, 3170 insertions(+), 939 deletions(-) create mode 100644 src/restic_compose_backup/hooks.py create mode 100644 src/tests/unit/test_backup_lifecycle.py create mode 100644 src/tests/unit/test_end_to_end_lifecycle.py create mode 100644 src/tests/unit/test_hooks.py diff --git a/src/restic_compose_backup/cli.py b/src/restic_compose_backup/cli.py index e395e02..f4412da 100644 --- a/src/restic_compose_backup/cli.py +++ b/src/restic_compose_backup/cli.py @@ -1,374 +1,465 @@ -import argparse -import os -import logging - -from restic_compose_backup import ( - alerts, - backup_runner, - log, - restic, -) -from restic_compose_backup.config import Config -from restic_compose_backup.containers import RunningContainers -from restic_compose_backup import cron, utils - -logger = logging.getLogger(__name__) - - -def main(): - """CLI entrypoint""" - args = parse_args() - config = Config() - log.setup(level=args.log_level or config.log_level) - containers = RunningContainers() - - # Ensure log level is propagated to parent container if overridden - if args.log_level: - containers.this_container.set_config_env("LOG_LEVEL", args.log_level) - - if args.action == "status": - status(config, containers) - - elif args.action == "snapshots": - snapshots(config, containers) - - elif args.action == "backup": - backup(config, containers) - - elif args.action == "start-backup-process": - start_backup_process(config, containers) - - elif args.action == "maintenance": - maintenance(config, containers) - - elif args.action == "cleanup": - cleanup(config, containers) - - elif args.action == "alert": - alert(config, containers) - - elif args.action == "version": - import restic_compose_backup - - print(restic_compose_backup.__version__) - - elif args.action == "crontab": - crontab(config) - - elif args.action == "dump-env": - dump_env() - - # Random test stuff here - elif args.action == "test": - nodes = utils.get_swarm_nodes() - print("Swarm nodes:") - for node in nodes: - addr = node.attrs["Status"]["Addr"] - state = node.attrs["Status"]["State"] - print(" - {} {} {}".format(node.id, addr, state)) - - -def status(config, containers): - """Outputs the backup config for the compose setup""" - logger.info("Status for compose project '%s'", containers.project_name) - logger.info("Repository: '%s'", config.repository) - logger.info("Backup currently running?: %s", containers.backup_process_running) - logger.info( - "Include project name in backup path?: %s", - utils.is_true(config.include_project_name), - ) - logger.debug( - "Exclude bind mounts from backups?: %s", - utils.is_true(config.exclude_bind_mounts), - ) - logger.debug( - "Include all compose projects?: %s", - utils.is_true(config.include_all_compose_projects), - ) - logger.debug( - f"Use cache for integrity check?: {utils.is_true(config.check_with_cache)}" - ) - logger.info("Checking docker availability") - - utils.list_containers() - - if containers.stale_backup_process_containers: - utils.remove_containers(containers.stale_backup_process_containers) - - logger.info("Contacting repository") - if not restic.is_initialized(config.repository): - logger.info("Repository is not initialized. Attempting to initialize it.") - result = restic.init_repo(config.repository) - if result == 0: - logger.info("Successfully initialized repository: %s", config.repository) - else: - logger.error("Failed to initialize repository") - - logger.info("%s Detected Config %s", "-" * 25, "-" * 25) - - # Start making snapshots - backup_containers = containers.containers_for_backup() - for container in backup_containers: - logger.info("service: %s", container.service_name) - - if container.volume_backup_enabled: - logger.info(f" - stop during backup: {container.stop_during_backup}") - for mount in container.filter_mounts(): - logger.info( - " - volume: %s -> %s", - mount.source, - container.get_volume_backup_destination(mount, "/volumes"), - ) - - if container.database_backup_enabled: - instance = container.instance - ping = instance.ping() - logger.info( - " - %s (is_ready=%s) -> %s", - instance.container_type, - ping, - instance.backup_destination_path(), - ) - if not ping: - logger.error( - "Database '%s' in service %s cannot be reached", - instance.container_type, - container.service_name, - ) - - if len(backup_containers) == 0: - logger.info("No containers in the project has 'stack-back.*' label") - - logger.info("-" * 67) - - -def backup(config, containers: RunningContainers): - """Request a backup to start""" - # Make sure we don't spawn multiple backup processes - if containers.backup_process_running: - alerts.send( - subject="Backup process container already running", - body=( - "A backup process container is already running. \n" - f"Id: {containers.backup_process_container.id}\n" - f"Name: {containers.backup_process_container.name}\n" - ), - alert_type="ERROR", - ) - raise RuntimeError("Backup process already running") - - # Map all volumes from the backup container into the backup process container - volumes = containers.this_container.volumes - - # Map volumes from other containers we are backing up - mounts = containers.generate_backup_mounts("/volumes") - volumes.update(mounts) - - logger.debug( - "Starting backup container with image %s", containers.this_container.image - ) - try: - result = backup_runner.run( - image=containers.this_container.image, - command="rcb start-backup-process", - volumes=volumes, - environment=containers.this_container.environment, - source_container_id=containers.this_container.id, - labels={ - containers.backup_process_label: "True", - "com.docker.compose.project": containers.project_name, - }, - ) - except Exception as ex: - logger.exception(ex) - alerts.send( - subject="Exception during backup", - body=str(ex), - alert_type="ERROR", - ) - return - - logger.info("Backup container exit code: %s", result) - - # Alert the user if something went wrong - if result != 0: - alerts.send( - subject="Backup process exited with non-zero code", - body=open("backup.log").read(), - alert_type="ERROR", - ) - - -def start_backup_process(config, containers): - """The actual backup process running inside the spawned container""" - if not utils.is_true(os.environ.get("BACKUP_PROCESS_CONTAINER")): - logger.error( - "Cannot run backup process in this container. Use backup command instead. " - "This will spawn a new container with the necessary mounts." - ) - alerts.send( - subject="Cannot run backup process in this container", - body=( - "Cannot run backup process in this container. Use backup command instead. " - "This will spawn a new container with the necessary mounts." - ), - ) - exit(1) - - status(config, containers) - errors = False - - # Did we actually get any volumes mounted? - try: - has_volumes = os.stat("/volumes") is not None - except FileNotFoundError: - logger.warning("Found no volumes to back up") - has_volumes = False - - # Warn if there is nothing to do - if len(containers.containers_for_backup()) == 0 and not has_volumes: - logger.error("No containers for backup found") - exit(1) - - # stop containers labeled to stop during backup - if len(containers.stop_during_backup_containers) > 0: - utils.stop_containers(containers.stop_during_backup_containers) - - # back up volumes - if has_volumes: - try: - logger.info("Backing up volumes") - vol_result = restic.backup_files(config.repository, source="/volumes") - logger.debug("Volume backup exit code: %s", vol_result) - if vol_result != 0: - logger.error("Volume backup exited with non-zero code: %s", vol_result) - errors = True - except Exception as ex: - logger.error("Exception raised during volume backup") - logger.exception(ex) - errors = True - - # back up databases - logger.info("Backing up databases") - for container in containers.containers_for_backup(): - if container.database_backup_enabled: - try: - instance = container.instance - logger.debug( - "Backing up %s in service %s from project %s", - instance.container_type, - instance.service_name, - instance.project_name, - ) - result = instance.backup() - logger.debug("Exit code: %s", result) - if result != 0: - logger.error("Backup command exited with non-zero code: %s", result) - errors = True - except Exception as ex: - logger.exception(ex) - errors = True - - # restart stopped containers after backup - if len(containers.stop_during_backup_containers) > 0: - utils.start_containers(containers.stop_during_backup_containers) - - if errors: - logger.error("Exit code: %s", errors) - exit(1) - - # Only run maintenance tasks if maintenance is not scheduled - if not config.maintenance_schedule: - maintenance(config, containers) - - logger.info("Backup completed") - - -def maintenance(config, containers): - """Run maintenance tasks""" - logger.info("Running maintenance tasks") - result = cleanup(config, containers) - if result != 0: - logger.error("Cleanup exit code: %s", result) - exit(1) - - logger.info("Checking the repository for errors") - check_with_cache = utils.is_true(config.check_with_cache) - result = restic.check(config.repository, with_cache=check_with_cache) - if result != 0: - logger.error("Check exit code: %s", result) - exit(1) - - -def cleanup(config, containers): - """Run forget / prune to minimize storage space""" - logger.info("Forget outdated snapshots") - forget_result = restic.forget( - config.repository, - config.keep_daily, - config.keep_weekly, - config.keep_monthly, - config.keep_yearly, - ) - logger.info("Prune stale data freeing storage space") - prune_result = restic.prune(config.repository) - return forget_result and prune_result - - -def snapshots(config, containers): - """Display restic snapshots""" - stdout, stderr = restic.snapshots(config.repository, last=True) - for line in stdout.decode().split("\n"): - print(line) - - -def alert(config, containers): - """Test alerts""" - logger.info("Testing alerts") - alerts.send( - subject="{}: Test Alert".format(containers.project_name), - body="Test message", - ) - - -def crontab(config): - """Generate the crontab""" - print(cron.generate_crontab(config)) - - -def dump_env(): - """Dump all environment variables to a file that can be sourced from cron""" - print("# This file was generated by stack-back") - for key, value in os.environ.items(): - print("export {}='{}'".format(key, value)) - - -def parse_args(): - parser = argparse.ArgumentParser(prog="restic_compose_backup") - parser.add_argument( - "action", - choices=[ - "status", - "snapshots", - "backup", - "start-backup-process", - "maintenance", - "alert", - "cleanup", - "version", - "crontab", - "dump-env", - "test", - ], - ) - parser.add_argument( - "--log-level", - default=None, - choices=list(log.LOG_LEVELS.keys()), - help="Log level", - ) - return parser.parse_args() - - -if __name__ == "__main__": - main() +import argparse +import os +import logging + +from restic_compose_backup import ( + alerts, + backup_runner, + hooks, + log, + restic, +) +from restic_compose_backup.config import Config +from restic_compose_backup.containers import RunningContainers +from restic_compose_backup import cron, enums, utils + +logger = logging.getLogger(__name__) + + +def main(): + """CLI entrypoint""" + args = parse_args() + config = Config() + log.setup(level=args.log_level or config.log_level) + containers = RunningContainers() + + # Ensure log level is propagated to parent container if overridden + if args.log_level: + containers.this_container.set_config_env("LOG_LEVEL", args.log_level) + + if args.action == "status": + status(config, containers) + + elif args.action == "snapshots": + snapshots(config, containers) + + elif args.action == "backup": + backup(config, containers) + + elif args.action == "start-backup-process": + start_backup_process(config, containers) + + elif args.action == "maintenance": + maintenance(config, containers) + + elif args.action == "cleanup": + cleanup(config, containers) + + elif args.action == "alert": + alert(config, containers) + + elif args.action == "version": + import restic_compose_backup + + print(restic_compose_backup.__version__) + + elif args.action == "crontab": + crontab(config) + + elif args.action == "dump-env": + dump_env() + + # Random test stuff here + elif args.action == "test": + nodes = utils.get_swarm_nodes() + print("Swarm nodes:") + for node in nodes: + addr = node.attrs["Status"]["Addr"] + state = node.attrs["Status"]["State"] + print(" - {} {} {}".format(node.id, addr, state)) + + +def _log_hook_summary(hook): + """Log a single hook's configuration in a readable format.""" + summary = f"[{hook.order}] {hook.cmd}" + if hook.context: + summary += f" | context: {hook.context}" + if hook.on_error != hooks.ON_ERROR_ABORT: + summary += f" | on-error: {hook.on_error}" + summary += f" | from: {hook.source_service_name}" + logger.info(" %s", summary) + + +def status(config, containers): + """Outputs the backup config for the compose setup""" + logger.info("Status for compose project '%s'", containers.project_name) + logger.info("Repository: '%s'", config.repository) + logger.info("Backup currently running?: %s", containers.backup_process_running) + logger.info( + "Include project name in backup path?: %s", + utils.is_true(config.include_project_name), + ) + logger.debug( + "Exclude bind mounts from backups?: %s", + utils.is_true(config.exclude_bind_mounts), + ) + logger.debug( + "Include all compose projects?: %s", + utils.is_true(config.include_all_compose_projects), + ) + logger.debug( + f"Use cache for integrity check?: {utils.is_true(config.check_with_cache)}" + ) + logger.info("Checking docker availability") + + utils.list_containers() + + if containers.stale_backup_process_containers: + utils.remove_containers(containers.stale_backup_process_containers) + + logger.info("Contacting repository") + if not restic.is_initialized(config.repository): + logger.info("Repository is not initialized. Attempting to initialize it.") + result = restic.init_repo(config.repository) + if result == 0: + logger.info("Successfully initialized repository: %s", config.repository) + else: + logger.error("Failed to initialize repository") + + logger.info("%s Detected Config %s", "-" * 25, "-" * 25) + + # Display backup configuration for each container + backup_containers = containers.containers_for_backup() + for container in backup_containers: + logger.info("service: %s", container.service_name) + + if container.volume_backup_enabled: + logger.info(f" - stop during backup: {container.stop_during_backup}") + for mount in container.filter_mounts(): + logger.info( + " - volume: %s -> %s", + mount.source, + container.get_volume_backup_destination(mount, "/volumes"), + ) + + if container.database_backup_enabled: + instance = container.instance + ping = instance.ping() + logger.info( + " - %s (is_ready=%s) -> %s", + instance.container_type, + ping, + instance.backup_destination_path(), + ) + if not ping: + logger.error( + "Database '%s' in service %s cannot be reached", + instance.container_type, + container.service_name, + ) + + if len(backup_containers) == 0: + logger.info("No containers in the project has 'stack-back.*' label") + + # Display backup order if configured + if utils.is_true( + containers.this_container.get_label(enums.LABEL_ORDERED) + ): + logger.info("%s Backup Order %s", "-" * 23, "-" * 23) + for position, container in enumerate(backup_containers, 1): + logger.info(" %d. %s", position, container.service_name) + + # Display hooks + has_hooks = False + for stage in hooks.HOOK_STAGES: + stage_hooks = hooks.collect_hooks( + stage, containers.this_container, backup_containers, + ) + if stage_hooks: + if not has_hooks: + logger.info( + "%s Hooks %s", "-" * 27, "-" * 27, + ) + has_hooks = True + logger.info(" %s:", stage) + for hook in stage_hooks: + _log_hook_summary(hook) + + logger.info("-" * 67) + + +def backup(config, containers: RunningContainers): + """Request a backup to start""" + # Make sure we don't spawn multiple backup processes + if containers.backup_process_running: + alerts.send( + subject="Backup process container already running", + body=( + "A backup process container is already running. \n" + f"Id: {containers.backup_process_container.id}\n" + f"Name: {containers.backup_process_container.name}\n" + ), + alert_type="ERROR", + ) + raise RuntimeError("Backup process already running") + + # Map all volumes from the backup container into the backup process container + volumes = containers.this_container.volumes + + # Map volumes from other containers we are backing up + mounts = containers.generate_backup_mounts("/volumes") + volumes.update(mounts) + + logger.debug( + "Starting backup container with image %s", containers.this_container.image + ) + + # Build labels for the backup process container, forwarding hook and + # ordering labels so the spawned container can read them. + process_labels = { + containers.backup_process_label: "True", + "com.docker.compose.project": containers.project_name, + } + for key, value in containers.this_container._labels.items(): + if ( + key.startswith("stack-back.hooks.") + or key.startswith("stack-back.order.") + or key == enums.LABEL_ORDERED + ): + process_labels[key] = value + + try: + result = backup_runner.run( + image=containers.this_container.image, + command="rcb start-backup-process", + volumes=volumes, + environment=containers.this_container.environment, + source_container_id=containers.this_container.id, + labels=process_labels, + ) + except Exception as ex: + logger.exception(ex) + alerts.send( + subject="Exception during backup", + body=str(ex), + alert_type="ERROR", + ) + return + + logger.info("Backup container exit code: %s", result) + + # Alert the user if something went wrong + if result != 0: + alerts.send( + subject="Backup process exited with non-zero code", + body=open("backup.log").read(), + alert_type="ERROR", + ) + + +def start_backup_process(config, containers): + """The actual backup process running inside the spawned container""" + if not utils.is_true(os.environ.get("BACKUP_PROCESS_CONTAINER")): + logger.error( + "Cannot run backup process in this container. Use backup command instead. " + "This will spawn a new container with the necessary mounts." + ) + alerts.send( + subject="Cannot run backup process in this container", + body=( + "Cannot run backup process in this container. Use backup command instead. " + "This will spawn a new container with the necessary mounts." + ), + ) + exit(1) + + status(config, containers) + errors = False + + # Did we actually get any volumes mounted? + try: + has_volumes = os.stat("/volumes") is not None + except FileNotFoundError: + logger.warning("Found no volumes to back up") + has_volumes = False + + backup_containers = containers.containers_for_backup() + + # Warn if there is nothing to do + if len(backup_containers) == 0 and not has_volumes: + logger.error("No containers for backup found") + exit(1) + + # Collect lifecycle hooks from backup container + target containers + pre_hooks = hooks.collect_hooks( + "pre", containers.this_container, backup_containers, + ) + post_hooks = hooks.collect_hooks( + "post", containers.this_container, backup_containers, + ) + error_hooks = hooks.collect_hooks( + "error", containers.this_container, backup_containers, + ) + finally_hooks = hooks.collect_hooks( + "finally", containers.this_container, backup_containers, + ) + + containers_stopped = False + + try: + # --- pre hooks --- + if not hooks.execute_hooks(pre_hooks, containers): + errors = True + + if not errors: + # stop containers labeled to stop during backup + if len(containers.stop_during_backup_containers) > 0: + utils.stop_containers(containers.stop_during_backup_containers) + containers_stopped = True + + # back up volumes + if has_volumes: + try: + logger.info("Backing up volumes") + vol_result = restic.backup_files(config.repository, source="/volumes") + logger.debug("Volume backup exit code: %s", vol_result) + if vol_result != 0: + logger.error("Volume backup exited with non-zero code: %s", vol_result) + errors = True + except Exception as ex: + logger.error("Exception raised during volume backup") + logger.exception(ex) + errors = True + + # back up databases + logger.info("Backing up databases") + for container in backup_containers: + if container.database_backup_enabled: + try: + instance = container.instance + logger.debug( + "Backing up %s in service %s from project %s", + instance.container_type, + instance.service_name, + instance.project_name, + ) + result = instance.backup() + logger.debug("Exit code: %s", result) + if result != 0: + logger.error("Backup command exited with non-zero code: %s", result) + errors = True + except Exception as ex: + logger.exception(ex) + errors = True + + except Exception as ex: + logger.exception(ex) + errors = True + + finally: + # Always restart stopped containers first, so that all subsequent + # hooks (post, error, finally) can exec into them. + if containers_stopped: + utils.start_containers(containers.stop_during_backup_containers) + + # --- post hooks (only when everything succeeded) --- + if not errors: + if not hooks.execute_hooks(post_hooks, containers): + errors = True + + # --- error hooks (only on failure) --- + if errors: + hooks.execute_hooks(error_hooks, containers) + + # --- finally hooks (always) --- + hooks.execute_hooks(finally_hooks, containers) + + if errors: + logger.error("Exit code: %s", 1) + exit(1) + + # Only run maintenance tasks if maintenance is not scheduled + if not config.maintenance_schedule: + maintenance(config, containers) + + logger.info("Backup completed") + + +def maintenance(config, containers): + """Run maintenance tasks""" + logger.info("Running maintenance tasks") + result = cleanup(config, containers) + if result != 0: + logger.error("Cleanup exit code: %s", result) + exit(1) + + logger.info("Checking the repository for errors") + check_with_cache = utils.is_true(config.check_with_cache) + result = restic.check(config.repository, with_cache=check_with_cache) + if result != 0: + logger.error("Check exit code: %s", result) + exit(1) + + +def cleanup(config, containers): + """Run forget / prune to minimize storage space""" + logger.info("Forget outdated snapshots") + forget_result = restic.forget( + config.repository, + config.keep_daily, + config.keep_weekly, + config.keep_monthly, + config.keep_yearly, + ) + logger.info("Prune stale data freeing storage space") + prune_result = restic.prune(config.repository) + return forget_result and prune_result + + +def snapshots(config, containers): + """Display restic snapshots""" + stdout, stderr = restic.snapshots(config.repository, last=True) + for line in stdout.decode().split("\n"): + print(line) + + +def alert(config, containers): + """Test alerts""" + logger.info("Testing alerts") + alerts.send( + subject="{}: Test Alert".format(containers.project_name), + body="Test message", + ) + + +def crontab(config): + """Generate the crontab""" + print(cron.generate_crontab(config)) + + +def dump_env(): + """Dump all environment variables to a file that can be sourced from cron""" + print("# This file was generated by stack-back") + for key, value in os.environ.items(): + print("export {}='{}'".format(key, value)) + + +def parse_args(): + parser = argparse.ArgumentParser(prog="restic_compose_backup") + parser.add_argument( + "action", + choices=[ + "status", + "snapshots", + "backup", + "start-backup-process", + "maintenance", + "alert", + "cleanup", + "version", + "crontab", + "dump-env", + "test", + ], + ) + parser.add_argument( + "--log-level", + default=None, + choices=list(log.LOG_LEVELS.keys()), + help="Log level", + ) + return parser.parse_args() + + +if __name__ == "__main__": + main() diff --git a/src/restic_compose_backup/containers.py b/src/restic_compose_backup/containers.py index 55ee256..e26477c 100644 --- a/src/restic_compose_backup/containers.py +++ b/src/restic_compose_backup/containers.py @@ -1,494 +1,577 @@ -import logging -from pathlib import Path -import socket -from typing import List - -from restic_compose_backup import enums, utils -from restic_compose_backup.config import config - -logger = logging.getLogger(__name__) - -VOLUME_TYPE_BIND = "bind" -VOLUME_TYPE_VOLUME = "volume" - - -class Container: - """Represents a docker container""" - - container_type = None - - def __init__(self, data: dict): - self._data = data - self._state = data.get("State") - self._config = data.get("Config") - self._mounts = [Mount(mnt, container=self) for mnt in data.get("Mounts")] - - if not self._state: - raise ValueError("Container meta missing State") - if self._config is None: - raise ValueError("Container meta missing Config") - - self._labels = self._config.get("Labels") - if self._labels is None: - raise ValueError("Container meta missing Config->Labels") - - self._include = self._parse_pattern(self.get_label(enums.LABEL_VOLUMES_INCLUDE)) - self._exclude = self._parse_pattern(self.get_label(enums.LABEL_VOLUMES_EXCLUDE)) - - @property - def instance(self) -> "Container": - """Container: Get a service specific subclass instance""" - # TODO: Do this smarter in the future (simple registry) - if self.database_backup_enabled: - from restic_compose_backup import containers_db - - if self.mariadb_backup_enabled: - return containers_db.MariadbContainer(self._data) - if self.mysql_backup_enabled: - return containers_db.MysqlContainer(self._data) - if self.postgresql_backup_enabled: - return containers_db.PostgresContainer(self._data) - else: - return self - - @property - def id(self) -> str: - """str: The id of the container""" - return self._data.get("Id") - - @property - def image(self) -> str: - """Image name""" - return self.get_config("Image") - - @property - def name(self) -> str: - """Container name""" - return self._data["Name"].replace("/", "") - - @property - def service_name(self) -> str: - """Name of the container/service""" - return self.get_label( - "com.docker.compose.service", default="" - ) or self.get_label("com.docker.swarm.service.name", default="") - - @property - def backup_process_label(self) -> str: - """str: The unique backup process label for this project""" - return f"{enums.LABEL_BACKUP_PROCESS}-{self.project_name}" - - @property - def project_name(self) -> str: - """str: Name of the compose setup""" - return self.get_label("com.docker.compose.project", default="") - - @property - def stack_name(self) -> str: - """str: Name of the stack is present""" - return self.get_label("com.docker.stack.namespace") - - @property - def is_oneoff(self) -> bool: - """Was this container started with run command?""" - return self.get_label("com.docker.compose.oneoff", default="False") == "True" - - @property - def environment(self) -> list: - """All configured env vars for the container as a list""" - return self.get_config("Env") - - def remove(self): - self._data.remove() - - def get_config_env(self, name) -> str: - """Get a config environment variable by name""" - # convert to dict and fetch env var by name - data = {i[0 : i.find("=")]: i[i.find("=") + 1 :] for i in self.environment} - return data.get(name) - - def set_config_env(self, name, value): - """Set an environment variable""" - env = self.environment - new_value = f"{name}={value}" - for i, entry in enumerate(env): - if f"{name}=" in entry: - env[i] = new_value - break - else: - env.append(new_value) - - @property - def volumes(self) -> dict: - """ - Return volumes for the container in the following format: - {'/home/user1/': {'bind': '/mnt/vol2', 'mode': 'rw'},} - """ - volumes = {} - for mount in self._mounts: - volumes[mount.source] = { - "bind": mount.destination, - "mode": "rw", - } - - return volumes - - @property - def backup_enabled(self) -> bool: - """Is backup enabled for this container?""" - return any( - [ - self.volume_backup_enabled, - self.database_backup_enabled, - ] - ) - - @property - def volume_backup_enabled(self) -> bool: - """bool: If the ``stack-back.volumes`` label is set""" - explicitly_enabled = utils.is_true(self.get_label(enums.LABEL_VOLUMES_ENABLED)) - explicitly_disabled = utils.is_false( - self.get_label(enums.LABEL_VOLUMES_ENABLED) - ) - automatically_enabled = utils.is_true(config.auto_backup_all) - return explicitly_enabled or (automatically_enabled and not explicitly_disabled) - - @property - def database_backup_enabled(self) -> bool: - """bool: Is database backup enabled in any shape or form?""" - return any( - [ - self.mysql_backup_enabled, - self.mariadb_backup_enabled, - self.postgresql_backup_enabled, - ] - ) - - @property - def mysql_backup_enabled(self) -> bool: - """bool: If the ``stack-back.mysql`` label is set""" - explicity_enabled = utils.is_true(self.get_label(enums.LABEL_MYSQL_ENABLED)) - explicity_disabled = utils.is_false(self.get_label(enums.LABEL_MYSQL_ENABLED)) - automatically_enabled = utils.is_true( - config.auto_backup_all - ) and self.image.startswith("mysql") - return explicity_enabled or (automatically_enabled and not explicity_disabled) - - @property - def mariadb_backup_enabled(self) -> bool: - """bool: If the ``stack-back.mariadb`` label is set""" - explicity_enabled = utils.is_true(self.get_label(enums.LABEL_MARIADB_ENABLED)) - explicity_disabled = utils.is_false(self.get_label(enums.LABEL_MARIADB_ENABLED)) - automatically_enabled = utils.is_true( - config.auto_backup_all - ) and self.image.startswith("mariadb") - return explicity_enabled or (automatically_enabled and not explicity_disabled) - - @property - def postgresql_backup_enabled(self) -> bool: - """bool: If the ``stack-back.postgres`` label is set""" - explicity_enabled = utils.is_true(self.get_label(enums.LABEL_POSTGRES_ENABLED)) - explicity_disabled = utils.is_false( - self.get_label(enums.LABEL_POSTGRES_ENABLED) - ) - automatically_enabled = utils.is_true( - config.auto_backup_all - ) and self.image.startswith("postgres") - return explicity_enabled or (automatically_enabled and not explicity_disabled) - - @property - def stop_during_backup(self) -> bool: - """bool: If the ``stack-back.volumes.stop-during-backup`` label is set""" - return ( - utils.is_true(self.get_label(enums.LABEL_STOP_DURING_BACKUP)) - and not self.database_backup_enabled - ) - - @property - def is_backup_process_container(self) -> bool: - """Is this container the running backup process?""" - return self.get_label(self.backup_process_label) == "True" - - @property - def is_running(self) -> bool: - """bool: Is the container running?""" - return self._state.get("Running", False) - - def get_config(self, name, default=None): - """Get value from config dict""" - return self._config.get(name, default) - - def get_label(self, name, default=None): - """Get a label by name""" - return self._labels.get(name, None) - - def filter_mounts(self): - """Get all mounts for this container matching include/exclude filters""" - filtered = [] - database_mounts = [ - "/var/lib/mysql", - "/var/lib/mariadb", - "/var/lib/postgresql/data", - ] - - # If exclude_bind_mounts is true, only volume mounts are kept in the list of mounts - exclude_bind_mounts = utils.is_true(config.exclude_bind_mounts) - mounts = list( - filter( - lambda m: not exclude_bind_mounts or m.type == "volume", self._mounts - ) - ) - - if not self.volume_backup_enabled: - return filtered - - if self._include: - for mount in mounts: - for pattern in self._include: - if pattern in mount.source: - break - else: - continue - - filtered.append(mount) - - elif self._exclude: - for mount in mounts: - for pattern in self._exclude: - if pattern in mount.source: - break - else: - filtered.append(mount) - else: - for mount in mounts: - if ( - self.database_backup_enabled - and mount.destination in database_mounts - ): - continue - filtered.append(mount) - - return filtered - - def volumes_for_backup(self, source_prefix="/volumes", mode="ro"): - """Get volumes configured for backup""" - mounts = self.filter_mounts() - volumes = {} - for mount in mounts: - volumes[mount.source] = { - "bind": self.get_volume_backup_destination(mount, source_prefix), - "mode": mode, - } - - return volumes - - def get_volume_backup_destination(self, mount, source_prefix) -> str: - """Get the destination path for backups of the given mount""" - destination = Path(source_prefix) - - if utils.is_true(config.include_project_name): - project_name = self.project_name - if project_name != "": - destination /= project_name - - destination /= self.service_name - destination /= Path(utils.strip_root(mount.destination)) - - return str(destination) - - def get_credentials(self) -> dict: - """dict: get credentials for the service""" - raise NotImplementedError("Base container class don't implement this") - - def ping(self) -> bool: - """Check the availability of the service""" - raise NotImplementedError("Base container class don't implement this") - - def backup(self): - """Back up this service""" - raise NotImplementedError("Base container class don't implement this") - - def backup_destination_path(self) -> str: - """Return the path backups will be saved at""" - raise NotImplementedError("Base container class don't implement this") - - def dump_command(self) -> list: - """list: create a dump command restic and use to send data through stdin""" - raise NotImplementedError("Base container class don't implement this") - - def _parse_pattern(self, value: str) -> List[str]: - """list: Safely parse include/exclude pattern from user""" - if not value: - return None - - if type(value) is not str: - return None - - value = value.strip() - if len(value) == 0: - return None - - return value.split(",") - - def __eq__(self, other): - """Compare container by id""" - if other is None: - return False - - if not isinstance(other, Container): - return False - - return self.id == other.id - - def __repr__(self): - return str(self) - - def __str__(self): - return "".format(self.name) - - -class Mount: - """Represents a volume mount (volume or bind)""" - - def __init__(self, data, container=None): - self._data = data - self._container = container - - @property - def container(self) -> Container: - """The container this mount belongs to""" - return self._container - - @property - def type(self) -> str: - """bind/volume""" - return self._data.get("Type") - - @property - def name(self) -> str: - """Name of the mount""" - return self._data.get("Name") - - @property - def source(self) -> str: - """Source of the mount. Volume name or path""" - return self._data.get("Source") - - @property - def destination(self) -> str: - """Destination path for the volume mount in the container""" - return self._data.get("Destination") - - def __repr__(self) -> str: - return str(self) - - def __str__(self) -> str: - return str(self._data) - - def __hash__(self): - """Uniqueness for a volume""" - if self.type == VOLUME_TYPE_VOLUME: - return hash(self.name) - elif self.type == VOLUME_TYPE_BIND: - return hash(self.source) - else: - raise ValueError("Unknown volume type: {}".format(self.type)) - - -class RunningContainers: - def __init__(self): - all_containers = utils.list_containers() - self.containers = [] - self.this_container = None - self.backup_process_container = None - self.stale_backup_process_containers = [] - self.stop_during_backup_containers = [] - - # Find the container we are running in. - # If we don't have this information we cannot continue - for container_data in all_containers: - if container_data.get("Id").startswith(socket.gethostname()): - self.this_container = Container(container_data) - - if not self.this_container: - raise ValueError("Cannot find metadata for backup container") - - # Gather relevant containers - for container_data in all_containers: - container = Container(container_data) - - # Gather stale backup process containers - if ( - self.this_container.image == container.image - and not container.is_running - and container.is_backup_process_container - ): - self.stale_backup_process_containers.append(container) - - # We only care about running containers after this point - if not container.is_running: - continue - - # If not swarm mode we need to filter in compose project - if ( - not config.swarm_mode - and not config.include_all_compose_projects - and container.project_name != self.this_container.project_name - ): - continue - - # Gather stop during backup containers - if container.stop_during_backup: - self.stop_during_backup_containers.append(container) - - # Detect running backup process container - if container.is_backup_process_container: - self.backup_process_container = container - - # Containers started manually are not included - if container.is_oneoff: - continue - - # Do not include the stack-back and backup process containers - if "stack-back" in container.image: - continue - - self.containers.append(container) - - @property - def project_name(self) -> str: - """str: Name of the compose project""" - return self.this_container.project_name - - @property - def backup_process_label(self) -> str: - """str: The backup process label for this project""" - return self.this_container.backup_process_label - - @property - def backup_process_running(self) -> bool: - """Is the backup process container running?""" - return self.backup_process_container is not None - - def containers_for_backup(self) -> list[Container]: - """Obtain all containers with backup enabled""" - return [container for container in self.containers if container.backup_enabled] - - def generate_backup_mounts(self, dest_prefix="/volumes") -> dict: - """Generate mounts for backup for the entire compose setup""" - mounts = {} - for container in self.containers_for_backup(): - if container.volume_backup_enabled: - mounts.update( - container.volumes_for_backup(source_prefix=dest_prefix, mode="ro") - ) - - return mounts - - def get_service(self, name) -> Container: - """Container: Get a service by name""" - for container in self.containers: - if container.service_name == name: - return container - - return None +import logging +from pathlib import Path +import socket +from typing import List + +from restic_compose_backup import enums, utils +from restic_compose_backup.config import config + +logger = logging.getLogger(__name__) + +VOLUME_TYPE_BIND = "bind" +VOLUME_TYPE_VOLUME = "volume" + + +class Container: + """Represents a docker container""" + + container_type = None + + def __init__(self, data: dict): + self._data = data + self._state = data.get("State") + self._config = data.get("Config") + self._mounts = [Mount(mnt, container=self) for mnt in data.get("Mounts")] + + if not self._state: + raise ValueError("Container meta missing State") + if self._config is None: + raise ValueError("Container meta missing Config") + + self._labels = self._config.get("Labels") + if self._labels is None: + raise ValueError("Container meta missing Config->Labels") + + self._include = self._parse_pattern(self.get_label(enums.LABEL_VOLUMES_INCLUDE)) + self._exclude = self._parse_pattern(self.get_label(enums.LABEL_VOLUMES_EXCLUDE)) + + @property + def instance(self) -> "Container": + """Container: Get a service specific subclass instance""" + # TODO: Do this smarter in the future (simple registry) + if self.database_backup_enabled: + from restic_compose_backup import containers_db + + if self.mariadb_backup_enabled: + return containers_db.MariadbContainer(self._data) + if self.mysql_backup_enabled: + return containers_db.MysqlContainer(self._data) + if self.postgresql_backup_enabled: + return containers_db.PostgresContainer(self._data) + else: + return self + + @property + def id(self) -> str: + """str: The id of the container""" + return self._data.get("Id") + + @property + def image(self) -> str: + """Image name""" + return self.get_config("Image") + + @property + def name(self) -> str: + """Container name""" + return self._data["Name"].replace("/", "") + + @property + def service_name(self) -> str: + """Name of the container/service""" + return self.get_label( + "com.docker.compose.service", default="" + ) or self.get_label("com.docker.swarm.service.name", default="") + + @property + def backup_process_label(self) -> str: + """str: The unique backup process label for this project""" + return f"{enums.LABEL_BACKUP_PROCESS}-{self.project_name}" + + @property + def project_name(self) -> str: + """str: Name of the compose setup""" + return self.get_label("com.docker.compose.project", default="") + + @property + def stack_name(self) -> str: + """str: Name of the stack is present""" + return self.get_label("com.docker.stack.namespace") + + @property + def is_oneoff(self) -> bool: + """Was this container started with run command?""" + return self.get_label("com.docker.compose.oneoff", default="False") == "True" + + @property + def environment(self) -> list: + """All configured env vars for the container as a list""" + return self.get_config("Env") + + def remove(self): + self._data.remove() + + def get_config_env(self, name) -> str: + """Get a config environment variable by name""" + # convert to dict and fetch env var by name + data = {i[0 : i.find("=")]: i[i.find("=") + 1 :] for i in self.environment} + return data.get(name) + + def set_config_env(self, name, value): + """Set an environment variable""" + env = self.environment + new_value = f"{name}={value}" + for i, entry in enumerate(env): + if f"{name}=" in entry: + env[i] = new_value + break + else: + env.append(new_value) + + @property + def volumes(self) -> dict: + """ + Return volumes for the container in the following format: + {'/home/user1/': {'bind': '/mnt/vol2', 'mode': 'rw'},} + """ + volumes = {} + for mount in self._mounts: + volumes[mount.source] = { + "bind": mount.destination, + "mode": "rw", + } + + return volumes + + @property + def backup_enabled(self) -> bool: + """Is backup enabled for this container?""" + return any( + [ + self.volume_backup_enabled, + self.database_backup_enabled, + ] + ) + + @property + def volume_backup_enabled(self) -> bool: + """bool: If the ``stack-back.volumes`` label is set""" + explicitly_enabled = utils.is_true(self.get_label(enums.LABEL_VOLUMES_ENABLED)) + explicitly_disabled = utils.is_false( + self.get_label(enums.LABEL_VOLUMES_ENABLED) + ) + automatically_enabled = utils.is_true(config.auto_backup_all) + return explicitly_enabled or (automatically_enabled and not explicitly_disabled) + + @property + def database_backup_enabled(self) -> bool: + """bool: Is database backup enabled in any shape or form?""" + return any( + [ + self.mysql_backup_enabled, + self.mariadb_backup_enabled, + self.postgresql_backup_enabled, + ] + ) + + @property + def mysql_backup_enabled(self) -> bool: + """bool: If the ``stack-back.mysql`` label is set""" + explicity_enabled = utils.is_true(self.get_label(enums.LABEL_MYSQL_ENABLED)) + explicity_disabled = utils.is_false(self.get_label(enums.LABEL_MYSQL_ENABLED)) + automatically_enabled = utils.is_true( + config.auto_backup_all + ) and self.image.startswith("mysql") + return explicity_enabled or (automatically_enabled and not explicity_disabled) + + @property + def mariadb_backup_enabled(self) -> bool: + """bool: If the ``stack-back.mariadb`` label is set""" + explicity_enabled = utils.is_true(self.get_label(enums.LABEL_MARIADB_ENABLED)) + explicity_disabled = utils.is_false(self.get_label(enums.LABEL_MARIADB_ENABLED)) + automatically_enabled = utils.is_true( + config.auto_backup_all + ) and self.image.startswith("mariadb") + return explicity_enabled or (automatically_enabled and not explicity_disabled) + + @property + def postgresql_backup_enabled(self) -> bool: + """bool: If the ``stack-back.postgres`` label is set""" + explicity_enabled = utils.is_true(self.get_label(enums.LABEL_POSTGRES_ENABLED)) + explicity_disabled = utils.is_false( + self.get_label(enums.LABEL_POSTGRES_ENABLED) + ) + automatically_enabled = utils.is_true( + config.auto_backup_all + ) and self.image.startswith("postgres") + return explicity_enabled or (automatically_enabled and not explicity_disabled) + + @property + def stop_during_backup(self) -> bool: + """bool: If the ``stack-back.volumes.stop-during-backup`` label is set""" + return ( + utils.is_true(self.get_label(enums.LABEL_STOP_DURING_BACKUP)) + and not self.database_backup_enabled + ) + + @property + def is_backup_process_container(self) -> bool: + """Is this container the running backup process?""" + return self.get_label(self.backup_process_label) == "True" + + @property + def is_running(self) -> bool: + """bool: Is the container running?""" + return self._state.get("Running", False) + + def get_config(self, name, default=None): + """Get value from config dict""" + return self._config.get(name, default) + + def get_label(self, name, default=None): + """Get a label by name""" + return self._labels.get(name, None) + + def filter_mounts(self): + """Get all mounts for this container matching include/exclude filters""" + filtered = [] + database_mounts = [ + "/var/lib/mysql", + "/var/lib/mariadb", + "/var/lib/postgresql/data", + ] + + # If exclude_bind_mounts is true, only volume mounts are kept in the list of mounts + exclude_bind_mounts = utils.is_true(config.exclude_bind_mounts) + mounts = list( + filter( + lambda m: not exclude_bind_mounts or m.type == "volume", self._mounts + ) + ) + + if not self.volume_backup_enabled: + return filtered + + if self._include: + for mount in mounts: + for pattern in self._include: + if pattern in mount.source: + break + else: + continue + + filtered.append(mount) + + elif self._exclude: + for mount in mounts: + for pattern in self._exclude: + if pattern in mount.source: + break + else: + filtered.append(mount) + else: + for mount in mounts: + if ( + self.database_backup_enabled + and mount.destination in database_mounts + ): + continue + filtered.append(mount) + + return filtered + + def volumes_for_backup(self, source_prefix="/volumes", mode="ro"): + """Get volumes configured for backup""" + mounts = self.filter_mounts() + volumes = {} + for mount in mounts: + volumes[mount.source] = { + "bind": self.get_volume_backup_destination(mount, source_prefix), + "mode": mode, + } + + return volumes + + def get_volume_backup_destination(self, mount, source_prefix) -> str: + """Get the destination path for backups of the given mount""" + destination = Path(source_prefix) + + if utils.is_true(config.include_project_name): + project_name = self.project_name + if project_name != "": + destination /= project_name + + destination /= self.service_name + destination /= Path(utils.strip_root(mount.destination)) + + return str(destination) + + def get_credentials(self) -> dict: + """dict: get credentials for the service""" + raise NotImplementedError("Base container class don't implement this") + + def ping(self) -> bool: + """Check the availability of the service""" + raise NotImplementedError("Base container class don't implement this") + + def backup(self): + """Back up this service""" + raise NotImplementedError("Base container class don't implement this") + + def backup_destination_path(self) -> str: + """Return the path backups will be saved at""" + raise NotImplementedError("Base container class don't implement this") + + def dump_command(self) -> list: + """list: create a dump command restic and use to send data through stdin""" + raise NotImplementedError("Base container class don't implement this") + + def _parse_pattern(self, value: str) -> List[str]: + """list: Safely parse include/exclude pattern from user""" + if not value: + return None + + if type(value) is not str: + return None + + value = value.strip() + if len(value) == 0: + return None + + return value.split(",") + + def __eq__(self, other): + """Compare container by id""" + if other is None: + return False + + if not isinstance(other, Container): + return False + + return self.id == other.id + + def __repr__(self): + return str(self) + + def __str__(self): + return "".format(self.name) + + +class Mount: + """Represents a volume mount (volume or bind)""" + + def __init__(self, data, container=None): + self._data = data + self._container = container + + @property + def container(self) -> Container: + """The container this mount belongs to""" + return self._container + + @property + def type(self) -> str: + """bind/volume""" + return self._data.get("Type") + + @property + def name(self) -> str: + """Name of the mount""" + return self._data.get("Name") + + @property + def source(self) -> str: + """Source of the mount. Volume name or path""" + return self._data.get("Source") + + @property + def destination(self) -> str: + """Destination path for the volume mount in the container""" + return self._data.get("Destination") + + def __repr__(self) -> str: + return str(self) + + def __str__(self) -> str: + return str(self._data) + + def __hash__(self): + """Uniqueness for a volume""" + if self.type == VOLUME_TYPE_VOLUME: + return hash(self.name) + elif self.type == VOLUME_TYPE_BIND: + return hash(self.source) + else: + raise ValueError("Unknown volume type: {}".format(self.type)) + + +class RunningContainers: + def __init__(self): + all_containers = utils.list_containers() + self.containers = [] + self.this_container = None + self.backup_process_container = None + self.stale_backup_process_containers = [] + self.stop_during_backup_containers = [] + + # Find the container we are running in. + # If we don't have this information we cannot continue + for container_data in all_containers: + if container_data.get("Id").startswith(socket.gethostname()): + self.this_container = Container(container_data) + + if not self.this_container: + raise ValueError("Cannot find metadata for backup container") + + # Gather relevant containers + for container_data in all_containers: + container = Container(container_data) + + # Gather stale backup process containers + if ( + self.this_container.image == container.image + and not container.is_running + and container.is_backup_process_container + ): + self.stale_backup_process_containers.append(container) + + # We only care about running containers after this point + if not container.is_running: + continue + + # If not swarm mode we need to filter in compose project + if ( + not config.swarm_mode + and not config.include_all_compose_projects + and container.project_name != self.this_container.project_name + ): + continue + + # Gather stop during backup containers + if container.stop_during_backup: + self.stop_during_backup_containers.append(container) + + # Detect running backup process container + if container.is_backup_process_container: + self.backup_process_container = container + + # Containers started manually are not included + if container.is_oneoff: + continue + + # Do not include the stack-back and backup process containers + if "stack-back" in container.image: + continue + + self.containers.append(container) + + @property + def project_name(self) -> str: + """str: Name of the compose project""" + return self.this_container.project_name + + @property + def backup_process_label(self) -> str: + """str: The backup process label for this project""" + return self.this_container.backup_process_label + + @property + def backup_process_running(self) -> bool: + """Is the backup process container running?""" + return self.backup_process_container is not None + + def containers_for_backup(self) -> list[Container]: + """Obtain all containers with backup enabled. + + When ``stack-back.ordered`` is set on the backup container, the + returned list respects the ``stack-back.order.`` sequence. + Containers not mentioned in the order list are appended afterwards. + """ + enabled = [ + container for container in self.containers + if container.backup_enabled + ] + + order = self._parse_backup_order() + if not order: + return enabled + + containers_by_service = { + container.service_name: container + for container in enabled + } + ordered: list[Container] = [] + seen: set[str] = set() + + for service_name in order: + if service_name in seen: + logger.warning( + "Duplicate service '%s' in backup order, skipping", + service_name, + ) + continue + seen.add(service_name) + + if service_name in containers_by_service: + ordered.append(containers_by_service[service_name]) + else: + logger.warning( + "Ordered service '%s' not found in backup containers", + service_name, + ) + + # Append containers not mentioned in the order list + for container in enabled: + if container.service_name not in seen: + logger.warning( + "Service '%s' not in backup order, appending after ordered services", + container.service_name, + ) + ordered.append(container) + + return ordered + + def _parse_backup_order(self) -> list[str]: + """Parse ``stack-back.order.`` labels from the backup container. + + Returns an ordered list of service names, or an empty list when + ordering is disabled or no order labels are found. + """ + if not self.this_container: + return [] + + if not utils.is_true(self.this_container.get_label(enums.LABEL_ORDERED)): + return [] + + order_map: dict[int, str] = {} + for label, value in self.this_container._labels.items(): + if not label.startswith(enums.LABEL_ORDER_PREFIX): + continue + suffix = label[len(enums.LABEL_ORDER_PREFIX):] + try: + order_num = int(suffix) + except ValueError: + logger.warning("Invalid order number in label '%s'", label) + continue + order_map[order_num] = value + + if not order_map: + logger.warning( + "stack-back.ordered is set but no order labels found, " + "falling back to default order" + ) + return [] + + return [ + order_map[position] + for position in sorted(order_map.keys()) + ] + + def generate_backup_mounts(self, dest_prefix="/volumes") -> dict: + """Generate mounts for backup for the entire compose setup""" + mounts = {} + for container in self.containers_for_backup(): + if container.volume_backup_enabled: + mounts.update( + container.volumes_for_backup(source_prefix=dest_prefix, mode="ro") + ) + + return mounts + + def get_service(self, name) -> Container: + """Container: Get a service by name""" + for container in self.containers: + if container.service_name == name: + return container + + return None diff --git a/src/restic_compose_backup/enums.py b/src/restic_compose_backup/enums.py index 788da14..2ce0ab3 100644 --- a/src/restic_compose_backup/enums.py +++ b/src/restic_compose_backup/enums.py @@ -1,11 +1,14 @@ -# Labels -LABEL_VOLUMES_ENABLED = "stack-back.volumes" -LABEL_VOLUMES_INCLUDE = "stack-back.volumes.include" -LABEL_VOLUMES_EXCLUDE = "stack-back.volumes.exclude" -LABEL_STOP_DURING_BACKUP = "stack-back.volumes.stop-during-backup" - -LABEL_MYSQL_ENABLED = "stack-back.mysql" -LABEL_POSTGRES_ENABLED = "stack-back.postgres" -LABEL_MARIADB_ENABLED = "stack-back.mariadb" - -LABEL_BACKUP_PROCESS = "stack-back.process" +# Labels +LABEL_VOLUMES_ENABLED = "stack-back.volumes" +LABEL_VOLUMES_INCLUDE = "stack-back.volumes.include" +LABEL_VOLUMES_EXCLUDE = "stack-back.volumes.exclude" +LABEL_STOP_DURING_BACKUP = "stack-back.volumes.stop-during-backup" + +LABEL_MYSQL_ENABLED = "stack-back.mysql" +LABEL_POSTGRES_ENABLED = "stack-back.postgres" +LABEL_MARIADB_ENABLED = "stack-back.mariadb" + +LABEL_BACKUP_PROCESS = "stack-back.process" + +LABEL_ORDERED = "stack-back.ordered" +LABEL_ORDER_PREFIX = "stack-back.order." \ No newline at end of file diff --git a/src/restic_compose_backup/hooks.py b/src/restic_compose_backup/hooks.py new file mode 100644 index 0000000..3995d34 --- /dev/null +++ b/src/restic_compose_backup/hooks.py @@ -0,0 +1,265 @@ +"""Backup hooks — pre, post, error, and finally lifecycle hooks. + +Hooks allow running user-defined commands before and after the backup process. +They follow try/catch/finally semantics: + +- **pre**: runs before backup starts +- **post**: runs only if all pre hooks and the backup itself succeeded +- **error**: runs only if a hook or the backup failed +- **finally**: always runs, regardless of outcome + +Hooks can be defined on the backup container (global) or on target containers +(per-service). For **pre** hooks, backup-container hooks execute first +(setup). For **post/error/finally** hooks, the order is reversed: +target-container hooks execute first, then backup-container hooks (teardown). + +Label format:: + + stack-back.hooks...cmd command string + stack-back.hooks...context service name to exec into (optional) + stack-back.hooks...on-error "abort" (default) | "continue" + +When ``context`` is omitted, the hook runs in the container where the label +is defined: backup-container hooks run locally (``commands.run``), target- +container hooks run via ``docker exec`` into that target container. An +explicit ``context`` overrides this and execs into the named service +regardless of where the label is defined. +""" + +import logging +from dataclasses import dataclass +from typing import List, Optional + +from restic_compose_backup import commands + +logger = logging.getLogger(__name__) + +HOOK_STAGES = ("pre", "post", "error", "finally") +LABEL_PREFIX = "stack-back.hooks." +ON_ERROR_ABORT = "abort" +ON_ERROR_CONTINUE = "continue" + + +@dataclass +class Hook: + """A single hook command to execute at a lifecycle stage.""" + + stage: str + order: int + cmd: str + # None → runs in the container where the label is defined + context: Optional[str] = None + on_error: str = ON_ERROR_ABORT + source_container_id: Optional[str] = None # set by collect_hooks() + source_service_name: Optional[str] = None # set by collect_hooks() + + +def parse_hooks_from_labels(labels: dict, stage: str) -> List[Hook]: + """Parse hook definitions from container labels for a given stage. + + Labels follow the pattern:: + + stack-back.hooks...cmd + stack-back.hooks...context (optional) + stack-back.hooks...on-error (optional, default: abort) + + Returns hooks sorted by order number. Entries without a ``cmd`` property + are skipped with a warning. + """ + prefix = f"{LABEL_PREFIX}{stage}." + hook_data: dict[int, dict[str, str]] = {} + + for label, value in labels.items(): + if not label.startswith(prefix): + continue + + # Extract "." from the label suffix after the prefix + label_suffix = label[len(prefix):] + parts = label_suffix.split(".", 1) + if len(parts) != 2: + logger.warning( + "Malformed hook label '%s', expected .", label + ) + continue + + try: + order_num = int(parts[0]) + except ValueError: + logger.warning("Invalid hook order number in label '%s'", label) + continue + + property_name = parts[1] + if order_num not in hook_data: + hook_data[order_num] = {} + hook_data[order_num][property_name] = value + + hooks: List[Hook] = [] + for order_num in sorted(hook_data.keys()): + data = hook_data[order_num] + cmd = data.get("cmd") + if not cmd: + logger.warning("Hook %s.%d has no 'cmd', skipping", stage, order_num) + continue + + on_error = data.get("on-error", ON_ERROR_ABORT) + if on_error not in (ON_ERROR_ABORT, ON_ERROR_CONTINUE): + logger.warning( + "Invalid on-error value '%s' for hook %s.%d, defaulting to '%s'", + on_error, + stage, + order_num, + ON_ERROR_ABORT, + ) + on_error = ON_ERROR_ABORT + + hooks.append( + Hook( + stage=stage, + order=order_num, + cmd=cmd, + context=data.get("context"), + on_error=on_error, + ) + ) + + return hooks + + +def collect_hooks(stage, backup_container, target_containers): + """Collect all hooks for *stage* from the backup container and targets. + + For **pre** hooks the backup-container hooks run first (set up global + state before per-service preparation). + + For **post / error / finally** hooks the order is reversed: target- + container hooks run first, then backup-container hooks (tear down + per-service state before global cleanup). This gives symmetric + setup / teardown semantics:: + + pre: backup-container → target containers (setup) + post: target containers → backup-container (teardown) + error: target containers → backup-container (teardown) + finally: target containers → backup-container (teardown) + + Source container metadata (id, service name) is attached to every hook + so that context-free hooks know where to execute. + """ + backup_hooks: List[Hook] = [] + for hook in parse_hooks_from_labels(backup_container._labels, stage): + hook.source_container_id = backup_container.id + hook.source_service_name = backup_container.service_name + backup_hooks.append(hook) + + target_hooks: List[Hook] = [] + for container in target_containers: + for hook in parse_hooks_from_labels(container._labels, stage): + hook.source_container_id = container.id + hook.source_service_name = container.service_name + target_hooks.append(hook) + + if stage == "pre": + # Setup: global first, then per-service + return backup_hooks + target_hooks + else: + # Teardown (post / error / finally): per-service first, then global + return target_hooks + backup_hooks + + +def execute_hooks(hooks, containers): + """Execute *hooks* in order, resolving each hook's target context. + + Returns ``True`` when every hook either succeeds (exit code 0) or + fails with ``on_error="continue"``. Returns ``False`` as soon as + any hook fails with ``on_error="abort"`` (the default); remaining + hooks in the list are skipped. + + An empty *hooks* list is always considered successful. + """ + if not hooks: + return True + + for hook in hooks: + context_info = f" (context: {hook.context})" if hook.context else "" + logger.info( + "Running %s hook [%d]: %s%s", + hook.stage, + hook.order, + hook.cmd, + context_info, + ) + + try: + exit_code = _execute_hook(hook, containers) + except Exception as ex: + logger.error("Hook execution error: %s", ex) + exit_code = 1 + + if exit_code != 0: + if hook.on_error == ON_ERROR_ABORT: + logger.error( + "Aborting: %s hook [%d] failed with exit code %d (on-error=%s)", + hook.stage, + hook.order, + exit_code, + ON_ERROR_ABORT, + ) + return False + else: + logger.warning( + "Continuing despite %s hook [%d] failure, exit code %d (on-error=%s)", + hook.stage, + hook.order, + exit_code, + ON_ERROR_CONTINUE, + ) + + return True + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + + +def _execute_hook(hook, containers): + """Resolve target context and execute a single hook. + + Returns the command's exit code (0 = success). + """ + this_container = containers.this_container + + if hook.context: + # Explicit context — look up the named service + target = containers.get_service(hook.context) + + # It may be the backup container itself + if target is None and this_container.service_name == hook.context: + return _run_local(hook.cmd) + + if target is None: + logger.error( + "Hook context container '%s' not found, failing hook: %s", + hook.context, + hook.cmd, + ) + return 1 + + return _run_in_container(target.id, hook.cmd) + else: + # No context — run in the source container + if hook.source_container_id == this_container.id: + return _run_local(hook.cmd) + else: + return _run_in_container(hook.source_container_id, hook.cmd) + + +def _run_local(cmd): + """Run a hook command locally in the backup process container.""" + logger.debug("Running hook locally: %s", cmd) + return commands.run(["sh", "-c", cmd]) + + +def _run_in_container(container_id, cmd): + """Run a hook command inside another container via docker exec.""" + logger.debug("Running hook in container %s: %s", container_id, cmd) + return commands.docker_exec(container_id, ["sh", "-c", cmd]) diff --git a/src/tests/unit/fixtures.py b/src/tests/unit/fixtures.py index b3dd0ce..92a6b75 100644 --- a/src/tests/unit/fixtures.py +++ b/src/tests/unit/fixtures.py @@ -1,60 +1,61 @@ -"""Generate test fixtures""" - -from datetime import datetime -import hashlib -import string -import random - - -def generate_sha256(): - """Generate a unique sha256""" - h = hashlib.sha256() - h.update(str(datetime.now().timestamp()).encode()) - return h.hexdigest() - - -def containers(project="default", containers=[]): - """ - Args: - project (str): Name of the compose project - containers (dict): - { - 'containers: [ - 'id': 'something' - 'service': 'service_name', - 'image': 'image:tag', - 'mounts: [{ - 'Source': '/home/user/stuff', - 'Destination': '/srv/stuff', - 'Type': 'bind' / 'volume' - }], - ] - } - """ - - def wrapper(*args, **kwargs): - return [ - { - "Id": container.get("id", generate_sha256()), - "Name": container.get("service") - + "_" - + "".join(random.choice(string.ascii_lowercase) for i in range(16)), - "Config": { - "Image": container.get("image", "image:latest"), - "Labels": { - "com.docker.compose.oneoff": "False", - "com.docker.compose.project": project, - "com.docker.compose.service": container["service"], - **container.get("labels", {}), - }, - }, - "Mounts": container.get("mounts", []), - "State": { - "Status": "running", - "Running": True, - }, - } - for container in containers - ] - - return wrapper +"""Generate test fixtures""" + +from datetime import datetime +import hashlib +import string +import random + + +def generate_sha256(): + """Generate a unique sha256""" + h = hashlib.sha256() + h.update(str(datetime.now().timestamp()).encode()) + return h.hexdigest() + + +def containers(project="default", containers=[]): + """ + Args: + project (str): Name of the compose project + containers (dict): + { + 'containers: [ + 'id': 'something' + 'service': 'service_name', + 'image': 'image:tag', + 'mounts: [{ + 'Source': '/home/user/stuff', + 'Destination': '/srv/stuff', + 'Type': 'bind' / 'volume' + }], + ] + } + """ + + def wrapper(*args, **kwargs): + return [ + { + "Id": container.get("id", generate_sha256()), + "Name": container.get("service") + + "_" + + "".join(random.choice(string.ascii_lowercase) for i in range(16)), + "Config": { + "Image": container.get("image", "image:latest"), + "Env": container.get("env", []), + "Labels": { + "com.docker.compose.oneoff": "False", + "com.docker.compose.project": project, + "com.docker.compose.service": container["service"], + **container.get("labels", {}), + }, + }, + "Mounts": container.get("mounts", []), + "State": { + "Status": "running", + "Running": True, + }, + } + for container in containers + ] + + return wrapper diff --git a/src/tests/unit/test_backup_lifecycle.py b/src/tests/unit/test_backup_lifecycle.py new file mode 100644 index 0000000..b487e3f --- /dev/null +++ b/src/tests/unit/test_backup_lifecycle.py @@ -0,0 +1,291 @@ +"""Unit tests for backup lifecycle orchestration in start_backup_process(). + +These tests verify the correct ordering of operations during a backup run: + + pre hooks → stop containers → backup → restart containers → + post hooks (on success) / error hooks (on failure) → finally hooks + +The ordering is critical because: +- Pre hooks may exec into containers (e.g., enable maintenance mode) +- Containers may be stopped during backup for filesystem consistency +- Post/error/finally hooks need containers running to exec into them + +A previous bug had post hooks executing before stopped containers were +restarted. These tests use a call_log pattern to assert the exact +sequence of operations and would have caught that bug. +""" + +import os +import unittest +from unittest import mock + +import pytest + +from restic_compose_backup import hooks +from restic_compose_backup.cli import start_backup_process + +pytestmark = pytest.mark.unit + + +class BackupLifecycleTests(unittest.TestCase): + """Tests for operation ordering in start_backup_process(). + + Uses a shared call_log list that records the order of operations. + Each mocked dependency appends its name when called. Tests then + assert the exact sequence (or relative ordering) of entries. + """ + + def setUp(self): + self.call_log = [] + + # Patch all external dependencies of start_backup_process + self.mock_status = mock.patch( + "restic_compose_backup.cli.status" + ).start() + self.mock_collect = mock.patch( + "restic_compose_backup.cli.hooks.collect_hooks" + ).start() + self.mock_execute = mock.patch( + "restic_compose_backup.cli.hooks.execute_hooks" + ).start() + self.mock_stop = mock.patch( + "restic_compose_backup.cli.utils.stop_containers" + ).start() + self.mock_start = mock.patch( + "restic_compose_backup.cli.utils.start_containers" + ).start() + self.mock_backup = mock.patch( + "restic_compose_backup.cli.restic.backup_files" + ).start() + self.mock_stat = mock.patch("os.stat").start() + mock.patch.dict( + os.environ, {"BACKUP_PROCESS_CONTAINER": "true"} + ).start() + + # Defaults: volumes exist, all operations succeed + self.mock_stat.return_value = True + self.mock_collect.side_effect = self._collect_one_hook_per_stage + self.mock_execute.side_effect = self._execute_and_record() + self.mock_backup.side_effect = self._record("backup_volumes", 0) + self.mock_stop.side_effect = self._record("stop_containers") + self.mock_start.side_effect = self._record("start_containers") + + def tearDown(self): + mock.patch.stopall() + + # ----- helpers ----- + + def _collect_one_hook_per_stage(self, stage, backup_container, targets): + """Return a single hook for each stage so every stage appears in the log.""" + return [hooks.Hook(stage=stage, order=1, cmd=f"echo {stage}")] + + def _execute_and_record(self, failing_stage=None): + """Side effect for execute_hooks that records the stage name. + + If *failing_stage* matches, the hook returns False (abort). + """ + def side_effect(hook_list, containers): + if not hook_list: + return True + stage = hook_list[0].stage + self.call_log.append(f"hooks:{stage}") + return stage != failing_stage + return side_effect + + def _record(self, name, return_value=None): + """Create a side effect that appends *name* to the call log.""" + def side_effect(*args, **kwargs): + self.call_log.append(name) + return return_value + return side_effect + + def _create_mock_containers(self, has_stop_during_backup=True, + has_database=False): + """Build a mock RunningContainers for lifecycle testing.""" + containers = mock.MagicMock() + containers.this_container._labels = {} + + backup_target = mock.MagicMock() + backup_target.database_backup_enabled = has_database + if has_database: + backup_target.instance.backup.side_effect = ( + self._record("backup_database", 0) + ) + backup_target.instance.container_type = "mysql" + backup_target.instance.service_name = "db" + backup_target.instance.project_name = "test" + containers.containers_for_backup.return_value = [backup_target] + + if has_stop_during_backup: + containers.stop_during_backup_containers = [mock.MagicMock()] + else: + containers.stop_during_backup_containers = [] + + return containers + + def _create_mock_config(self): + """Build a mock Config that skips maintenance.""" + config = mock.MagicMock() + config.maintenance_schedule = "0 3 * * *" # non-empty skips maintenance + return config + + # ----- lifecycle ordering tests ----- + + def test_successful_backup_lifecycle_order(self): + """Happy path: pre → stop → backup → restart → post → finally.""" + config = self._create_mock_config() + containers = self._create_mock_containers() + + start_backup_process(config, containers) + + self.assertEqual(self.call_log, [ + "hooks:pre", + "stop_containers", + "backup_volumes", + "start_containers", + "hooks:post", + "hooks:finally", + ]) + + def test_post_hooks_run_after_container_restart(self): + """Post hooks must execute after stopped containers are restarted. + + This test would have caught the original bug where post hooks + ran inside the try block before the finally block restarted + containers. + """ + config = self._create_mock_config() + containers = self._create_mock_containers() + + start_backup_process(config, containers) + + restart_position = self.call_log.index("start_containers") + post_position = self.call_log.index("hooks:post") + finally_position = self.call_log.index("hooks:finally") + + self.assertGreater( + post_position, restart_position, + "Post hooks must run after containers are restarted", + ) + self.assertGreater( + finally_position, restart_position, + "Finally hooks must run after containers are restarted", + ) + + def test_error_hooks_run_after_container_restart(self): + """On backup failure, error hooks must still run after restart.""" + config = self._create_mock_config() + containers = self._create_mock_containers() + self.mock_backup.side_effect = self._record("backup_volumes", 1) + + with self.assertRaises(SystemExit): + start_backup_process(config, containers) + + restart_position = self.call_log.index("start_containers") + error_position = self.call_log.index("hooks:error") + + self.assertGreater( + error_position, restart_position, + "Error hooks must run after containers are restarted", + ) + + def test_pre_hook_failure_skips_backup_and_stop(self): + """When a pre hook aborts, no backup runs and containers are not stopped.""" + config = self._create_mock_config() + containers = self._create_mock_containers() + self.mock_execute.side_effect = ( + self._execute_and_record(failing_stage="pre") + ) + + with self.assertRaises(SystemExit): + start_backup_process(config, containers) + + self.assertIn("hooks:pre", self.call_log) + self.assertNotIn("stop_containers", self.call_log) + self.assertNotIn("backup_volumes", self.call_log) + self.assertNotIn("hooks:post", self.call_log) + self.assertIn("hooks:error", self.call_log) + self.assertIn("hooks:finally", self.call_log) + + def test_backup_failure_triggers_error_not_post(self): + """On backup failure: error hooks run, post hooks do not.""" + config = self._create_mock_config() + containers = self._create_mock_containers() + self.mock_backup.side_effect = self._record("backup_volumes", 1) + + with self.assertRaises(SystemExit): + start_backup_process(config, containers) + + self.assertIn("hooks:pre", self.call_log) + self.assertIn("backup_volumes", self.call_log) + self.assertNotIn("hooks:post", self.call_log) + self.assertIn("hooks:error", self.call_log) + self.assertIn("hooks:finally", self.call_log) + + def test_post_hook_failure_triggers_error_hooks(self): + """When post hooks fail, error hooks still run.""" + config = self._create_mock_config() + containers = self._create_mock_containers() + self.mock_execute.side_effect = ( + self._execute_and_record(failing_stage="post") + ) + + with self.assertRaises(SystemExit): + start_backup_process(config, containers) + + self.assertEqual(self.call_log, [ + "hooks:pre", + "stop_containers", + "backup_volumes", + "start_containers", + "hooks:post", + "hooks:error", + "hooks:finally", + ]) + + def test_containers_always_restarted_on_failure(self): + """Stopped containers are restarted even when backup fails.""" + config = self._create_mock_config() + containers = self._create_mock_containers() + self.mock_backup.side_effect = self._record("backup_volumes", 1) + + with self.assertRaises(SystemExit): + start_backup_process(config, containers) + + self.assertIn("stop_containers", self.call_log) + self.assertIn("start_containers", self.call_log) + + def test_no_stop_when_no_stop_during_backup_containers(self): + """Without stop-during-backup containers, stop/start are skipped.""" + config = self._create_mock_config() + containers = self._create_mock_containers(has_stop_during_backup=False) + + start_backup_process(config, containers) + + self.assertEqual(self.call_log, [ + "hooks:pre", + "backup_volumes", + "hooks:post", + "hooks:finally", + ]) + + def test_database_backup_runs_between_volumes_and_restart(self): + """Database dumps run after volume backup but before container restart.""" + config = self._create_mock_config() + containers = self._create_mock_containers(has_database=True) + + start_backup_process(config, containers) + + volume_position = self.call_log.index("backup_volumes") + database_position = self.call_log.index("backup_database") + restart_position = self.call_log.index("start_containers") + + self.assertGreater( + database_position, volume_position, + "Database backup must run after volume backup", + ) + self.assertGreater( + restart_position, database_position, + "Container restart must run after database backup", + ) + diff --git a/src/tests/unit/test_end_to_end_lifecycle.py b/src/tests/unit/test_end_to_end_lifecycle.py new file mode 100644 index 0000000..bd0633c --- /dev/null +++ b/src/tests/unit/test_end_to_end_lifecycle.py @@ -0,0 +1,938 @@ +"""End-to-end backup lifecycle tests with real container and hook objects. + +Unlike test_backup_lifecycle.py (which mocks hooks.execute_hooks entirely) +and test_hooks.py (which tests hooks in isolation), these tests exercise +the FULL start_backup_process() flow with real Container objects and real +hook logic. Only the I/O boundary is mocked: + + commands.run → records local shell commands (hooks + restic) + commands.docker_exec → records docker exec calls (hooks into containers) + utils.stop_containers → records container stop operations + utils.start_containers → records container start operations + restic.backup_from_stdin → records database dump operations + +Everything above that boundary runs for real: + + hooks.parse_hooks_from_labels → label parsing from real Container._labels + hooks.collect_hooks → symmetric ordering (setup / teardown) + hooks.execute_hooks → iteration, abort / continue logic + hooks._execute_hook → context resolution (local vs. docker exec) + hooks._run_local → delegates to commands.run (mocked) + hooks._run_in_container → delegates to commands.docker_exec (mocked) + restic.backup_files → delegates to commands.run (mocked) + RunningContainers → container discovery from fixture data + containers_for_backup → ordered backup logic + +This catches bugs that neither test file alone would find: + +- Hook context resolution errors (which container gets exec'd into) +- Symmetric hook ordering (pre: global→targets, post: targets→global) +- Post/error hooks running after container restart (not before) +- Label parsing through to actual command execution +- Interaction between ordered backup and hook ordering +""" + +import os +import unittest +from unittest import mock + +import pytest + +from restic_compose_backup.containers import RunningContainers +from restic_compose_backup.cli import start_backup_process +from . import fixtures +from .conftest import BaseTestCase + +pytestmark = pytest.mark.unit + +list_containers_func = "restic_compose_backup.utils.list_containers" + +# Predictable container IDs for target containers. +# These must NOT start with the mocked hostname (backup_hash[:8]). +WEB_ID = "w" * 64 +DB_ID = "d" * 64 +ALPHA_ID = "a" * 64 +BETA_ID = "b" * 64 + + +class EndToEndBackupTests(BaseTestCase): + """End-to-end lifecycle tests using real Container/Hook objects. + + The call_log records every I/O operation in a readable string format:: + + "local: echo global-pre" hook running locally (commands.run) + "exec@web: echo web-pre" hook docker-exec'd into "web" + "stop_containers" containers stopped + "start_containers" containers restarted + "restic_backup_files" volume backup (restic via commands.run) + "backup_from_stdin@db" database dump (restic.backup_from_stdin) + + Tests assert the exact call_log sequence (or relative ordering) to + verify the full orchestration pipeline. + """ + + def setUp(self): + self.call_log = [] + self.restic_exit_code = 0 + self.failing_local_hooks = {} # {cmd_string: exit_code} + self.id_to_service = {} + + # Track patches explicitly — mock.patch.stopall() would also kill + # the hostname patcher from BaseTestCase.setUpClass. + self._patches = [ + mock.patch( + "restic_compose_backup.commands.run", + side_effect=self._mock_commands_run, + ), + mock.patch( + "restic_compose_backup.commands.docker_exec", + side_effect=self._mock_docker_exec, + ), + mock.patch( + "restic_compose_backup.cli.utils.stop_containers", + side_effect=self._mock_stop, + ), + mock.patch( + "restic_compose_backup.cli.utils.start_containers", + side_effect=self._mock_start, + ), + mock.patch( + "restic_compose_backup.restic.backup_from_stdin", + side_effect=self._mock_backup_from_stdin, + ), + mock.patch("restic_compose_backup.cli.status"), + mock.patch("os.stat", return_value=True), + mock.patch.dict( + os.environ, {"BACKUP_PROCESS_CONTAINER": "true"} + ), + ] + for patch in self._patches: + patch.start() + + def tearDown(self): + for patch in reversed(self._patches): + patch.stop() + + # ------------------------------------------------------------------ # + # I/O mock implementations # + # ------------------------------------------------------------------ # + + def _mock_commands_run(self, cmd): + """Distinguish restic commands from local hook commands. + + Both go through commands.run but produce different call_log entries: + - restic: ``["restic", "-r", "test", "--verbose", "backup", ...]`` + - hook: ``["sh", "-c", "echo something"]`` + """ + if cmd and cmd[0] == "restic": + self.call_log.append("restic_backup_files") + return self.restic_exit_code + hook_cmd = cmd[2] if len(cmd) >= 3 else " ".join(cmd) + self.call_log.append(f"local: {hook_cmd}") + return self.failing_local_hooks.get(hook_cmd, 0) + + def _mock_docker_exec(self, container_id, cmd, **kwargs): + """Record which container was exec'd into, with the command.""" + service = self.id_to_service.get( + container_id, f"unknown({container_id[:12]})" + ) + hook_cmd = cmd[2] if len(cmd) >= 3 else " ".join(cmd) + self.call_log.append(f"exec@{service}: {hook_cmd}") + return 0 + + def _mock_stop(self, containers): + self.call_log.append("stop_containers") + + def _mock_start(self, containers): + self.call_log.append("start_containers") + + def _mock_backup_from_stdin(self, repository, filename, container_id, + source_command, environment=None): + service = self.id_to_service.get( + container_id, f"unknown({container_id[:12]})" + ) + self.call_log.append(f"backup_from_stdin@{service}") + return 0 + + # ------------------------------------------------------------------ # + # Helpers # + # ------------------------------------------------------------------ # + + def _build_running_containers(self, container_defs): + """Create real RunningContainers from fixture definitions. + + Also populates ``self.id_to_service`` so call_log entries + show readable service names instead of raw container IDs. + """ + with mock.patch( + list_containers_func, + fixtures.containers(containers=container_defs), + ): + running = RunningContainers() + + self.id_to_service = { + running.this_container.id: running.this_container.service_name, + } + for container in running.containers: + self.id_to_service[container.id] = container.service_name + + return running + + def _make_config(self): + """Create a mock Config that skips maintenance.""" + cfg = mock.MagicMock() + cfg.repository = "test" + cfg.maintenance_schedule = "0 3 * * *" + return cfg + + # ------------------------------------------------------------------ # + # Scenario 1: Full happy path # + # ------------------------------------------------------------------ # + + def test_full_happy_path_with_hooks_on_backup_and_target(self): + """Complete successful backup with hooks on both backup and target. + + Verifies the exact sequence:: + + pre(backup) → pre(web) → stop → restic backup → start → + post(web) → post(backup) → finally(backup) + + This tests symmetric ordering, context resolution, and restart + happening before post/finally hooks. + """ + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "echo global-pre", + "stack-back.hooks.post.1.cmd": "echo global-post", + "stack-back.hooks.finally.1.cmd": "echo global-finally", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": { + "stack-back.volumes": "true", + "stack-back.volumes.stop-during-backup": "true", + "stack-back.hooks.pre.1.cmd": "echo web-pre", + "stack-back.hooks.post.1.cmd": "echo web-post", + }, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + self.assertEqual(self.call_log, [ + "local: echo global-pre", # pre: backup first (setup) + "exec@web: echo web-pre", # pre: target second + "stop_containers", # stop web + "restic_backup_files", # backup volumes + "start_containers", # restart web + "exec@web: echo web-post", # post: target first (teardown) + "local: echo global-post", # post: backup second + "local: echo global-finally", # finally: backup + ]) + + # ------------------------------------------------------------------ # + # Scenario 2: Post hooks must run after container restart # + # ------------------------------------------------------------------ # + + def test_post_hooks_always_run_after_container_restart(self): + """Post and finally hooks must execute AFTER stopped containers restart. + + This is the bug found during review: post hooks were originally + in the try block, running before the finally block restarted + containers. + """ + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.post.1.cmd": "echo global-post", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": { + "stack-back.volumes": "true", + "stack-back.volumes.stop-during-backup": "true", + "stack-back.hooks.post.1.cmd": "echo web-post", + }, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + restart_idx = self.call_log.index("start_containers") + post_web_idx = self.call_log.index("exec@web: echo web-post") + post_global_idx = self.call_log.index("local: echo global-post") + + self.assertGreater( + post_web_idx, restart_idx, + "Web post hook must run after containers are restarted", + ) + self.assertGreater( + post_global_idx, restart_idx, + "Global post hook must run after containers are restarted", + ) + + # ------------------------------------------------------------------ # + # Scenario 3: Context resolution — all cases # + # ------------------------------------------------------------------ # + + def test_backup_hook_without_context_runs_locally(self): + """A backup container hook without explicit context runs locally. + + Current behavior: backup-container hooks with no context run via + commands.run() (local subprocess) rather than docker exec. This + is functionally equivalent inside the backup process container + but uses a different code path than target-container hooks. + """ + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "echo local-test", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": {"stack-back.volumes": "true"}, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + self.assertIn("local: echo local-test", self.call_log) + self.assertNotIn("exec@backup: echo local-test", self.call_log) + + def test_target_hook_without_context_execs_into_own_container(self): + """A target container hook without explicit context execs into itself. + + The hook is defined on 'web', so it must docker-exec into the web + container — not run locally in the backup process container. + """ + container_defs = self.createContainers() + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": { + "stack-back.volumes": "true", + "stack-back.hooks.pre.1.cmd": "echo self-exec", + }, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + self.assertIn("exec@web: echo self-exec", self.call_log) + self.assertNotIn("local: echo self-exec", self.call_log) + + def test_context_override_backup_hook_execs_into_target(self): + """A backup container hook with explicit context execs into the named service.""" + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "echo ctx-override", + "stack-back.hooks.pre.1.context": "web", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": {"stack-back.volumes": "true"}, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + self.assertIn("exec@web: echo ctx-override", self.call_log) + self.assertNotIn("local: echo ctx-override", self.call_log) + + def test_cross_container_context_web_hook_execs_into_db(self): + """A hook defined on 'web' with context='db' execs into the db container.""" + container_defs = self.createContainers() + container_defs += [ + { + "id": WEB_ID, + "service": "web", + "labels": { + "stack-back.volumes": "true", + "stack-back.hooks.pre.1.cmd": "echo cross-ctx", + "stack-back.hooks.pre.1.context": "db", + }, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }, + { + "id": DB_ID, + "service": "db", + "image": "mysql:8", + "labels": {"stack-back.mysql": "true"}, + "env": ["MYSQL_ROOT_PASSWORD=secret"], + "mounts": [ + {"Source": "mysql_data", "Destination": "/var/lib/mysql", "Type": "volume"}, + ], + }, + ] + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + self.assertIn("exec@db: echo cross-ctx", self.call_log) + self.assertNotIn("exec@web: echo cross-ctx", self.call_log) + self.assertNotIn("local: echo cross-ctx", self.call_log) + + def test_unknown_hook_context_triggers_failure(self): + """A hook referencing a non-existent service triggers the error path.""" + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "echo bad-ctx", + "stack-back.hooks.pre.1.context": "nonexistent", + "stack-back.hooks.error.1.cmd": "echo global-error", + "stack-back.hooks.finally.1.cmd": "echo global-finally", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": {"stack-back.volumes": "true"}, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + + with self.assertRaises(SystemExit): + start_backup_process(self._make_config(), containers) + + # Bad context fails the hook → no backup → error + finally + self.assertNotIn("restic_backup_files", self.call_log) + self.assertIn("local: echo global-error", self.call_log) + self.assertIn("local: echo global-finally", self.call_log) + + # ------------------------------------------------------------------ # + # Scenario 4: Backup failure — error hooks, not post hooks # + # ------------------------------------------------------------------ # + + def test_backup_failure_fires_error_hooks_not_post(self): + """When volume backup fails, error hooks fire; post hooks do not. + + Also verifies symmetric teardown ordering: target error hooks + run before backup-container error hooks. + """ + self.restic_exit_code = 1 + + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.post.1.cmd": "echo global-post", + "stack-back.hooks.error.1.cmd": "echo global-error", + "stack-back.hooks.finally.1.cmd": "echo global-finally", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": { + "stack-back.volumes": "true", + "stack-back.volumes.stop-during-backup": "true", + "stack-back.hooks.error.1.cmd": "echo web-error", + }, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + + with self.assertRaises(SystemExit): + start_backup_process(self._make_config(), containers) + + self.assertEqual(self.call_log, [ + "stop_containers", + "restic_backup_files", # fails (exit code 1) + "start_containers", # restart before hooks + "exec@web: echo web-error", # error: target first (teardown) + "local: echo global-error", # error: backup second + "local: echo global-finally", # finally always + ]) + self.assertNotIn("local: echo global-post", self.call_log) + + def test_error_hooks_run_after_container_restart(self): + """On failure, error hooks must run after containers are restarted.""" + self.restic_exit_code = 1 + + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.error.1.cmd": "echo global-error", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": { + "stack-back.volumes": "true", + "stack-back.volumes.stop-during-backup": "true", + "stack-back.hooks.error.1.cmd": "echo web-error", + }, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + + with self.assertRaises(SystemExit): + start_backup_process(self._make_config(), containers) + + restart_idx = self.call_log.index("start_containers") + web_error_idx = self.call_log.index("exec@web: echo web-error") + global_error_idx = self.call_log.index("local: echo global-error") + + self.assertGreater( + web_error_idx, restart_idx, + "Web error hook must run after restart", + ) + self.assertGreater( + global_error_idx, restart_idx, + "Global error hook must run after restart", + ) + + # ------------------------------------------------------------------ # + # Scenario 5: Pre hook failure — skips everything # + # ------------------------------------------------------------------ # + + def test_pre_hook_failure_skips_backup_and_stop(self): + """When a pre hook aborts, no stop/backup happens; error+finally fire.""" + self.failing_local_hooks["fail-pre"] = 1 + + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "fail-pre", + "stack-back.hooks.error.1.cmd": "echo global-error", + "stack-back.hooks.finally.1.cmd": "echo global-finally", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": { + "stack-back.volumes": "true", + "stack-back.volumes.stop-during-backup": "true", + "stack-back.hooks.pre.1.cmd": "echo web-pre", + }, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + + with self.assertRaises(SystemExit): + start_backup_process(self._make_config(), containers) + + self.assertEqual(self.call_log, [ + "local: fail-pre", # pre hook fails → abort + # No web pre hook (aborted after first failure) + # No stop_containers + # No restic_backup_files + "local: echo global-error", # error fires + "local: echo global-finally", # finally fires + ]) + + # ------------------------------------------------------------------ # + # Scenario 6: Database backup in lifecycle # + # ------------------------------------------------------------------ # + + def test_database_backup_in_lifecycle(self): + """Database dump runs after volume backup, before restart. + + Uses a real MysqlContainer instance (via Container.instance) with + mocked restic.backup_from_stdin at the I/O boundary. + """ + container_defs = self.createContainers() + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": { + "stack-back.volumes": "true", + "stack-back.volumes.stop-during-backup": "true", + }, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + container_defs.append({ + "id": DB_ID, + "service": "db", + "image": "mysql:8", + "labels": {"stack-back.mysql": "true"}, + "env": ["MYSQL_ROOT_PASSWORD=secret"], + "mounts": [ + {"Source": "mysql_data", "Destination": "/var/lib/mysql", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + volume_idx = self.call_log.index("restic_backup_files") + db_idx = self.call_log.index("backup_from_stdin@db") + restart_idx = self.call_log.index("start_containers") + + self.assertGreater( + db_idx, volume_idx, + "DB dump must run after volume backup", + ) + self.assertGreater( + restart_idx, db_idx, + "Restart must run after DB dump", + ) + + # ------------------------------------------------------------------ # + # Scenario 7: Symmetric ordering with multiple targets # + # ------------------------------------------------------------------ # + + def test_symmetric_ordering_multiple_targets(self): + """Pre hooks go backup→alpha→beta; post hooks go alpha→beta→backup. + + Verifies that the backup container is correctly placed first for + setup (pre) and last for teardown (post). Target containers + maintain their discovery order in both stages. + """ + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "echo global-pre", + "stack-back.hooks.post.1.cmd": "echo global-post", + } + container_defs += [ + { + "id": ALPHA_ID, + "service": "alpha", + "labels": { + "stack-back.volumes": "true", + "stack-back.hooks.pre.1.cmd": "echo alpha-pre", + "stack-back.hooks.post.1.cmd": "echo alpha-post", + }, + "mounts": [ + {"Source": "a_data", "Destination": "/a", "Type": "volume"}, + ], + }, + { + "id": BETA_ID, + "service": "beta", + "labels": { + "stack-back.volumes": "true", + "stack-back.hooks.pre.1.cmd": "echo beta-pre", + "stack-back.hooks.post.1.cmd": "echo beta-post", + }, + "mounts": [ + {"Source": "b_data", "Destination": "/b", "Type": "volume"}, + ], + }, + ] + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + # Pre hooks: backup → alpha → beta (setup order) + pre_global = self.call_log.index("local: echo global-pre") + pre_alpha = self.call_log.index("exec@alpha: echo alpha-pre") + pre_beta = self.call_log.index("exec@beta: echo beta-pre") + self.assertLess(pre_global, pre_alpha) + self.assertLess(pre_alpha, pre_beta) + + # Post hooks: alpha → beta → backup (teardown order) + post_alpha = self.call_log.index("exec@alpha: echo alpha-post") + post_beta = self.call_log.index("exec@beta: echo beta-post") + post_global = self.call_log.index("local: echo global-post") + self.assertLess(post_alpha, post_global) + self.assertLess(post_beta, post_global) + + # ------------------------------------------------------------------ # + # Scenario 8: Ordered backup affects hook ordering # + # ------------------------------------------------------------------ # + + def test_ordered_backup_affects_hook_order(self): + """When ordered backup is configured, hooks follow container order. + + Order configured as beta→alpha means beta hooks run before alpha. + """ + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.ordered": "true", + "stack-back.order.1": "beta", + "stack-back.order.2": "alpha", + } + container_defs += [ + { + "id": ALPHA_ID, + "service": "alpha", + "labels": { + "stack-back.volumes": "true", + "stack-back.hooks.pre.1.cmd": "echo alpha-pre", + "stack-back.hooks.post.1.cmd": "echo alpha-post", + }, + "mounts": [ + {"Source": "a_data", "Destination": "/a", "Type": "volume"}, + ], + }, + { + "id": BETA_ID, + "service": "beta", + "labels": { + "stack-back.volumes": "true", + "stack-back.hooks.pre.1.cmd": "echo beta-pre", + "stack-back.hooks.post.1.cmd": "echo beta-post", + }, + "mounts": [ + {"Source": "b_data", "Destination": "/b", "Type": "volume"}, + ], + }, + ] + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + # Pre hooks follow configured order: beta before alpha + pre_beta = self.call_log.index("exec@beta: echo beta-pre") + pre_alpha = self.call_log.index("exec@alpha: echo alpha-pre") + self.assertLess( + pre_beta, pre_alpha, + "Ordered backup: beta pre should run before alpha pre", + ) + + # Post hooks follow configured order within targets: beta before alpha + post_beta = self.call_log.index("exec@beta: echo beta-post") + post_alpha = self.call_log.index("exec@alpha: echo alpha-post") + self.assertLess( + post_beta, post_alpha, + "Ordered backup post: targets maintain configured order", + ) + + # ------------------------------------------------------------------ # + # Scenario 9: Minimal scenario — no hooks, no stops # + # ------------------------------------------------------------------ # + + def test_no_hooks_no_stop_clean_backup(self): + """Minimal scenario: volume backup with no hooks and no stopped containers.""" + container_defs = self.createContainers() + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": {"stack-back.volumes": "true"}, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + self.assertEqual(self.call_log, ["restic_backup_files"]) + + # ------------------------------------------------------------------ # + # Scenario 10: on-error=continue allows backup to proceed # + # ------------------------------------------------------------------ # + + def test_continue_on_error_does_not_abort_pipeline(self): + """A pre hook with on-error=continue allows the backup to proceed.""" + self.failing_local_hooks["soft-fail"] = 1 + + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "soft-fail", + "stack-back.hooks.pre.1.on-error": "continue", + "stack-back.hooks.pre.2.cmd": "echo second-pre", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": {"stack-back.volumes": "true"}, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + # Both pre hooks ran despite first failure + self.assertIn("local: soft-fail", self.call_log) + self.assertIn("local: echo second-pre", self.call_log) + # Backup proceeded + self.assertIn("restic_backup_files", self.call_log) + + # ------------------------------------------------------------------ # + # Scenario 11: Post hook failure triggers error hooks # + # ------------------------------------------------------------------ # + + def test_post_hook_failure_triggers_error_hooks(self): + """When a post hook fails, error hooks still run.""" + self.failing_local_hooks["echo fail-post"] = 1 + + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.post.1.cmd": "echo fail-post", + "stack-back.hooks.error.1.cmd": "echo global-error", + "stack-back.hooks.finally.1.cmd": "echo global-finally", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": {"stack-back.volumes": "true"}, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + + with self.assertRaises(SystemExit): + start_backup_process(self._make_config(), containers) + + self.assertEqual(self.call_log, [ + "restic_backup_files", + "local: echo fail-post", # post hook fails + "local: echo global-error", # error fires + "local: echo global-finally", # finally fires + ]) + + # ------------------------------------------------------------------ # + # Scenario 12: Hooks on all four stages # + # ------------------------------------------------------------------ # + + def test_all_four_stages_with_failure(self): + """All hook stages fire in correct order during a backup failure. + + pre → (backup fails) → error → finally + No post hooks because backup failed. + """ + self.restic_exit_code = 1 + + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "echo pre", + "stack-back.hooks.post.1.cmd": "echo post", + "stack-back.hooks.error.1.cmd": "echo error", + "stack-back.hooks.finally.1.cmd": "echo finally", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": {"stack-back.volumes": "true"}, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + + with self.assertRaises(SystemExit): + start_backup_process(self._make_config(), containers) + + self.assertEqual(self.call_log, [ + "local: echo pre", + "restic_backup_files", # fails + "local: echo error", # error fires + "local: echo finally", # finally fires + ]) + self.assertNotIn("local: echo post", self.call_log) + + # ------------------------------------------------------------------ # + # Scenario 13: Multiple hooks per stage in correct order # + # ------------------------------------------------------------------ # + + def test_multiple_hooks_per_stage_execute_in_order(self): + """Multiple hooks on one container execute in numerical order.""" + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.pre.3.cmd": "echo third", + "stack-back.hooks.pre.1.cmd": "echo first", + "stack-back.hooks.pre.2.cmd": "echo second", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": {"stack-back.volumes": "true"}, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + first_idx = self.call_log.index("local: echo first") + second_idx = self.call_log.index("local: echo second") + third_idx = self.call_log.index("local: echo third") + + self.assertLess(first_idx, second_idx) + self.assertLess(second_idx, third_idx) + # All before backup + backup_idx = self.call_log.index("restic_backup_files") + self.assertLess(third_idx, backup_idx) + + # ------------------------------------------------------------------ # + # Scenario 14: Database with hooks — full lifecycle # + # ------------------------------------------------------------------ # + + def test_database_with_hooks_full_lifecycle(self): + """Full lifecycle with a web container (stopped) and DB container (hooks). + + Verifies that DB hooks fire via docker exec into the DB container + and that DB dump happens after volume backup. + """ + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "echo global-pre", + "stack-back.hooks.finally.1.cmd": "echo global-finally", + } + container_defs += [ + { + "id": WEB_ID, + "service": "web", + "labels": { + "stack-back.volumes": "true", + "stack-back.volumes.stop-during-backup": "true", + }, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }, + { + "id": DB_ID, + "service": "db", + "image": "mysql:8", + "labels": { + "stack-back.mysql": "true", + "stack-back.hooks.pre.1.cmd": "echo db-pre", + "stack-back.hooks.post.1.cmd": "echo db-post", + }, + "env": ["MYSQL_ROOT_PASSWORD=secret"], + "mounts": [ + {"Source": "mysql_data", "Destination": "/var/lib/mysql", "Type": "volume"}, + ], + }, + ] + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + self.assertEqual(self.call_log, [ + "local: echo global-pre", # pre: backup first + "exec@db: echo db-pre", # pre: db target + "stop_containers", # stop web (db NOT stopped) + "restic_backup_files", # backup volumes + "backup_from_stdin@db", # dump database + "start_containers", # restart web + "exec@db: echo db-post", # post: target first (teardown) + "local: echo global-finally", # finally: backup + ]) + + diff --git a/src/tests/unit/test_hooks.py b/src/tests/unit/test_hooks.py new file mode 100644 index 0000000..dc3dd85 --- /dev/null +++ b/src/tests/unit/test_hooks.py @@ -0,0 +1,559 @@ +"""Unit tests for backup hooks and ordered backup execution""" + +import unittest +from unittest import mock + +import pytest + +from restic_compose_backup import hooks +from restic_compose_backup.containers import RunningContainers +from . import fixtures +from .conftest import BaseTestCase + +pytestmark = pytest.mark.unit + +list_containers_func = "restic_compose_backup.utils.list_containers" + + +class HookParsingTests(BaseTestCase): + """Tests for parse_hooks_from_labels()""" + + def test_parse_basic_hooks(self): + labels = { + "stack-back.hooks.pre.1.cmd": "echo before", + "stack-back.hooks.pre.1.context": "web", + "stack-back.hooks.pre.2.cmd": "echo also before", + } + result = hooks.parse_hooks_from_labels(labels, "pre") + self.assertEqual(len(result), 2) + self.assertEqual(result[0].cmd, "echo before") + self.assertEqual(result[0].context, "web") + self.assertEqual(result[0].order, 1) + self.assertEqual(result[0].on_error, "abort") + self.assertEqual(result[1].cmd, "echo also before") + self.assertIsNone(result[1].context) + self.assertEqual(result[1].order, 2) + + def test_parse_on_error_continue(self): + labels = { + "stack-back.hooks.pre.1.cmd": "echo test", + "stack-back.hooks.pre.1.on-error": "continue", + } + result = hooks.parse_hooks_from_labels(labels, "pre") + self.assertEqual(len(result), 1) + self.assertEqual(result[0].on_error, "continue") + + def test_parse_invalid_on_error_defaults_to_abort(self): + labels = { + "stack-back.hooks.pre.1.cmd": "echo test", + "stack-back.hooks.pre.1.on-error": "invalid", + } + result = hooks.parse_hooks_from_labels(labels, "pre") + self.assertEqual(result[0].on_error, "abort") + + def test_parse_non_contiguous_order(self): + labels = { + "stack-back.hooks.pre.1.cmd": "echo first", + "stack-back.hooks.pre.5.cmd": "echo third", + "stack-back.hooks.pre.3.cmd": "echo second", + } + result = hooks.parse_hooks_from_labels(labels, "pre") + self.assertEqual(len(result), 3) + self.assertEqual( + [hook.order for hook in result], [1, 3, 5] + ) + + def test_parse_missing_cmd_skipped(self): + labels = { + "stack-back.hooks.pre.1.context": "web", + "stack-back.hooks.pre.2.cmd": "echo valid", + } + result = hooks.parse_hooks_from_labels(labels, "pre") + self.assertEqual(len(result), 1) + self.assertEqual(result[0].order, 2) + + def test_parse_stages_isolated(self): + labels = { + "stack-back.hooks.pre.1.cmd": "echo pre", + "stack-back.hooks.post.1.cmd": "echo post", + "stack-back.hooks.error.1.cmd": "echo error", + "stack-back.hooks.finally.1.cmd": "echo finally", + } + for stage in hooks.HOOK_STAGES: + result = hooks.parse_hooks_from_labels(labels, stage) + self.assertEqual(len(result), 1) + self.assertEqual(result[0].cmd, f"echo {stage}") + + def test_parse_empty_labels(self): + result = hooks.parse_hooks_from_labels({"other": "value"}, "pre") + self.assertEqual(len(result), 0) + + def test_parse_invalid_order_number(self): + labels = { + "stack-back.hooks.pre.abc.cmd": "echo bad", + "stack-back.hooks.pre.1.cmd": "echo good", + } + result = hooks.parse_hooks_from_labels(labels, "pre") + self.assertEqual(len(result), 1) + self.assertEqual(result[0].order, 1) + + +class HookCollectionTests(BaseTestCase): + """Tests for collect_hooks()""" + + def test_backup_hooks_before_target_hooks(self): + containers = self.createContainers() + containers[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "echo backup-pre", + "stack-back.hooks.post.1.cmd": "echo backup-post", + } + containers += [ + { + "service": "web", + "labels": { + "stack-back.volumes": True, + "stack-back.hooks.pre.1.cmd": "echo web-pre", + "stack-back.hooks.post.1.cmd": "echo web-post", + }, + "mounts": [ + {"Source": "data", "Destination": "/data", "Type": "volume"}, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + + backup_list = running.containers_for_backup() + + # pre: backup container hooks first (setup) + pre = hooks.collect_hooks("pre", running.this_container, backup_list) + self.assertEqual(len(pre), 2) + self.assertEqual(pre[0].cmd, "echo backup-pre") + self.assertEqual(pre[0].source_service_name, "backup") + self.assertEqual(pre[1].cmd, "echo web-pre") + self.assertEqual(pre[1].source_service_name, "web") + + # post: target container hooks first (teardown) + post = hooks.collect_hooks("post", running.this_container, backup_list) + self.assertEqual(len(post), 2) + self.assertEqual(post[0].cmd, "echo web-post") + self.assertEqual(post[0].source_service_name, "web") + self.assertEqual(post[1].cmd, "echo backup-post") + self.assertEqual(post[1].source_service_name, "backup") + + def test_no_hooks_returns_empty(self): + containers = self.createContainers() + containers += [ + { + "service": "web", + "labels": {"stack-back.volumes": True}, + "mounts": [ + {"Source": "d", "Destination": "/d", "Type": "volume"}, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + + self.assertEqual( + len( + hooks.collect_hooks( + "pre", running.this_container, running.containers_for_backup() + ) + ), + 0, + ) + + def test_multiple_targets_in_order(self): + containers = self.createContainers() + containers += [ + { + "service": "alpha", + "labels": { + "stack-back.volumes": True, + "stack-back.hooks.pre.1.cmd": "echo alpha", + }, + "mounts": [ + {"Source": "a", "Destination": "/a", "Type": "volume"}, + ], + }, + { + "service": "beta", + "labels": { + "stack-back.volumes": True, + "stack-back.hooks.pre.1.cmd": "echo beta", + }, + "mounts": [ + {"Source": "b", "Destination": "/b", "Type": "volume"}, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + + pre = hooks.collect_hooks( + "pre", running.this_container, running.containers_for_backup() + ) + self.assertEqual(pre[0].source_service_name, "alpha") + self.assertEqual(pre[1].source_service_name, "beta") + + +class HookExecutionTests(BaseTestCase): + """Tests for execute_hooks() and context resolution""" + + @mock.patch("restic_compose_backup.hooks._run_local") + @mock.patch("restic_compose_backup.hooks._run_in_container") + def test_empty_hooks_succeeds(self, mock_remote, mock_local): + containers = self.createContainers() + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + self.assertTrue(hooks.execute_hooks([], running)) + mock_local.assert_not_called() + mock_remote.assert_not_called() + + @mock.patch("restic_compose_backup.hooks._run_local", return_value=0) + def test_backup_hook_runs_locally(self, mock_local): + containers = self.createContainers() + containers[0]["labels"] = {"stack-back.hooks.pre.1.cmd": "echo test"} + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + pre = hooks.collect_hooks( + "pre", running.this_container, running.containers_for_backup() + ) + self.assertTrue(hooks.execute_hooks(pre, running)) + mock_local.assert_called_once_with("echo test") + + @mock.patch("restic_compose_backup.hooks._run_in_container", return_value=0) + def test_target_hook_uses_docker_exec(self, mock_remote): + containers = self.createContainers() + containers += [ + { + "service": "web", + "labels": { + "stack-back.volumes": True, + "stack-back.hooks.pre.1.cmd": "echo web", + }, + "mounts": [ + {"Source": "d", "Destination": "/d", "Type": "volume"}, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + web = running.get_service("web") + pre = hooks.collect_hooks( + "pre", running.this_container, running.containers_for_backup() + ) + self.assertTrue(hooks.execute_hooks(pre, running)) + mock_remote.assert_called_once_with(web.id, "echo web") + + @mock.patch("restic_compose_backup.hooks._run_in_container", return_value=0) + def test_explicit_context_overrides_source(self, mock_remote): + containers = self.createContainers() + containers[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "echo ctx", + "stack-back.hooks.pre.1.context": "web", + } + containers += [ + { + "service": "web", + "labels": {"stack-back.volumes": True}, + "mounts": [ + {"Source": "d", "Destination": "/d", "Type": "volume"}, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + web = running.get_service("web") + pre = hooks.collect_hooks( + "pre", running.this_container, running.containers_for_backup() + ) + self.assertTrue(hooks.execute_hooks(pre, running)) + mock_remote.assert_called_once_with(web.id, "echo ctx") + + @mock.patch("restic_compose_backup.hooks._run_local", return_value=1) + def test_abort_on_failure(self, mock_local): + containers = self.createContainers() + containers[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "fail", + "stack-back.hooks.pre.2.cmd": "echo skip", + } + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + pre = hooks.collect_hooks( + "pre", running.this_container, running.containers_for_backup() + ) + self.assertFalse(hooks.execute_hooks(pre, running)) + mock_local.assert_called_once_with("fail") + + @mock.patch("restic_compose_backup.hooks._run_local", side_effect=[1, 0]) + def test_continue_on_failure(self, mock_local): + containers = self.createContainers() + containers[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "fail", + "stack-back.hooks.pre.1.on-error": "continue", + "stack-back.hooks.pre.2.cmd": "echo ok", + } + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + pre = hooks.collect_hooks( + "pre", running.this_container, running.containers_for_backup() + ) + self.assertTrue(hooks.execute_hooks(pre, running)) + self.assertEqual(mock_local.call_count, 2) + + @mock.patch("restic_compose_backup.hooks._run_in_container") + def test_unknown_context_fails(self, mock_remote): + containers = self.createContainers() + containers[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "echo x", + "stack-back.hooks.pre.1.context": "nonexistent", + } + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + pre = hooks.collect_hooks( + "pre", running.this_container, running.containers_for_backup() + ) + self.assertFalse(hooks.execute_hooks(pre, running)) + mock_remote.assert_not_called() + + @mock.patch( + "restic_compose_backup.hooks._run_local", side_effect=Exception("boom") + ) + def test_exception_treated_as_failure(self, mock_local): + containers = self.createContainers() + containers[0]["labels"] = {"stack-back.hooks.pre.1.cmd": "explode"} + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + pre = hooks.collect_hooks( + "pre", running.this_container, running.containers_for_backup() + ) + self.assertFalse(hooks.execute_hooks(pre, running)) + + +class BackupOrderingTests(BaseTestCase): + """Tests for ordered backup execution""" + + def test_default_unordered(self): + containers = self.createContainers() + containers += [ + { + "service": "web", + "labels": {"stack-back.volumes": True}, + "mounts": [ + {"Source": "d", "Destination": "/d", "Type": "volume"}, + ], + }, + { + "service": "db", + "labels": {"stack-back.mysql": True}, + "mounts": [ + { + "Source": "m", + "Destination": "/var/lib/mysql", + "Type": "volume", + }, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + self.assertEqual(len(running.containers_for_backup()), 2) + + def test_ordered_containers(self): + containers = self.createContainers() + containers[0]["labels"] = { + "stack-back.ordered": "true", + "stack-back.order.1": "db", + "stack-back.order.2": "web", + } + containers += [ + { + "service": "web", + "labels": {"stack-back.volumes": True}, + "mounts": [ + {"Source": "d", "Destination": "/d", "Type": "volume"}, + ], + }, + { + "service": "db", + "labels": {"stack-back.mysql": True}, + "mounts": [ + { + "Source": "m", + "Destination": "/var/lib/mysql", + "Type": "volume", + }, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + backup = running.containers_for_backup() + self.assertEqual(backup[0].service_name, "db") + self.assertEqual(backup[1].service_name, "web") + + def test_ordered_with_gaps(self): + containers = self.createContainers() + containers[0]["labels"] = { + "stack-back.ordered": "true", + "stack-back.order.1": "db", + "stack-back.order.5": "web", + } + containers += [ + { + "service": "web", + "labels": {"stack-back.volumes": True}, + "mounts": [ + {"Source": "d", "Destination": "/d", "Type": "volume"}, + ], + }, + { + "service": "db", + "labels": {"stack-back.mysql": True}, + "mounts": [ + { + "Source": "m", + "Destination": "/var/lib/mysql", + "Type": "volume", + }, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + backup = running.containers_for_backup() + self.assertEqual(backup[0].service_name, "db") + self.assertEqual(backup[1].service_name, "web") + + def test_ordered_unknown_service_skipped(self): + containers = self.createContainers() + containers[0]["labels"] = { + "stack-back.ordered": "true", + "stack-back.order.1": "nonexistent", + "stack-back.order.2": "web", + } + containers += [ + { + "service": "web", + "labels": {"stack-back.volumes": True}, + "mounts": [ + {"Source": "d", "Destination": "/d", "Type": "volume"}, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + backup = running.containers_for_backup() + self.assertEqual(len(backup), 1) + self.assertEqual(backup[0].service_name, "web") + + def test_unordered_container_appended(self): + containers = self.createContainers() + containers[0]["labels"] = { + "stack-back.ordered": "true", + "stack-back.order.1": "db", + } + containers += [ + { + "service": "web", + "labels": {"stack-back.volumes": True}, + "mounts": [ + {"Source": "d", "Destination": "/d", "Type": "volume"}, + ], + }, + { + "service": "db", + "labels": {"stack-back.mysql": True}, + "mounts": [ + { + "Source": "m", + "Destination": "/var/lib/mysql", + "Type": "volume", + }, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + backup = running.containers_for_backup() + self.assertEqual(len(backup), 2) + self.assertEqual(backup[0].service_name, "db") + self.assertEqual(backup[1].service_name, "web") + + def test_ordered_duplicate_service(self): + containers = self.createContainers() + containers[0]["labels"] = { + "stack-back.ordered": "true", + "stack-back.order.1": "web", + "stack-back.order.2": "web", + } + containers += [ + { + "service": "web", + "labels": {"stack-back.volumes": True}, + "mounts": [ + {"Source": "d", "Destination": "/d", "Type": "volume"}, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + backup = running.containers_for_backup() + self.assertEqual(len(backup), 1) + + def test_ordered_true_no_labels_falls_back(self): + containers = self.createContainers() + containers[0]["labels"] = {"stack-back.ordered": "true"} + containers += [ + { + "service": "web", + "labels": {"stack-back.volumes": True}, + "mounts": [ + {"Source": "d", "Destination": "/d", "Type": "volume"}, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + backup = running.containers_for_backup() + self.assertEqual(len(backup), 1) + self.assertEqual(backup[0].service_name, "web") +