diff --git a/src/restic_compose_backup/cli.py b/src/restic_compose_backup/cli.py index e395e02..f4412da 100644 --- a/src/restic_compose_backup/cli.py +++ b/src/restic_compose_backup/cli.py @@ -1,374 +1,465 @@ -import argparse -import os -import logging - -from restic_compose_backup import ( - alerts, - backup_runner, - log, - restic, -) -from restic_compose_backup.config import Config -from restic_compose_backup.containers import RunningContainers -from restic_compose_backup import cron, utils - -logger = logging.getLogger(__name__) - - -def main(): - """CLI entrypoint""" - args = parse_args() - config = Config() - log.setup(level=args.log_level or config.log_level) - containers = RunningContainers() - - # Ensure log level is propagated to parent container if overridden - if args.log_level: - containers.this_container.set_config_env("LOG_LEVEL", args.log_level) - - if args.action == "status": - status(config, containers) - - elif args.action == "snapshots": - snapshots(config, containers) - - elif args.action == "backup": - backup(config, containers) - - elif args.action == "start-backup-process": - start_backup_process(config, containers) - - elif args.action == "maintenance": - maintenance(config, containers) - - elif args.action == "cleanup": - cleanup(config, containers) - - elif args.action == "alert": - alert(config, containers) - - elif args.action == "version": - import restic_compose_backup - - print(restic_compose_backup.__version__) - - elif args.action == "crontab": - crontab(config) - - elif args.action == "dump-env": - dump_env() - - # Random test stuff here - elif args.action == "test": - nodes = utils.get_swarm_nodes() - print("Swarm nodes:") - for node in nodes: - addr = node.attrs["Status"]["Addr"] - state = node.attrs["Status"]["State"] - print(" - {} {} {}".format(node.id, addr, state)) - - -def status(config, containers): - """Outputs the backup config for the compose setup""" - logger.info("Status for compose project '%s'", containers.project_name) - logger.info("Repository: '%s'", config.repository) - logger.info("Backup currently running?: %s", containers.backup_process_running) - logger.info( - "Include project name in backup path?: %s", - utils.is_true(config.include_project_name), - ) - logger.debug( - "Exclude bind mounts from backups?: %s", - utils.is_true(config.exclude_bind_mounts), - ) - logger.debug( - "Include all compose projects?: %s", - utils.is_true(config.include_all_compose_projects), - ) - logger.debug( - f"Use cache for integrity check?: {utils.is_true(config.check_with_cache)}" - ) - logger.info("Checking docker availability") - - utils.list_containers() - - if containers.stale_backup_process_containers: - utils.remove_containers(containers.stale_backup_process_containers) - - logger.info("Contacting repository") - if not restic.is_initialized(config.repository): - logger.info("Repository is not initialized. Attempting to initialize it.") - result = restic.init_repo(config.repository) - if result == 0: - logger.info("Successfully initialized repository: %s", config.repository) - else: - logger.error("Failed to initialize repository") - - logger.info("%s Detected Config %s", "-" * 25, "-" * 25) - - # Start making snapshots - backup_containers = containers.containers_for_backup() - for container in backup_containers: - logger.info("service: %s", container.service_name) - - if container.volume_backup_enabled: - logger.info(f" - stop during backup: {container.stop_during_backup}") - for mount in container.filter_mounts(): - logger.info( - " - volume: %s -> %s", - mount.source, - container.get_volume_backup_destination(mount, "/volumes"), - ) - - if container.database_backup_enabled: - instance = container.instance - ping = instance.ping() - logger.info( - " - %s (is_ready=%s) -> %s", - instance.container_type, - ping, - instance.backup_destination_path(), - ) - if not ping: - logger.error( - "Database '%s' in service %s cannot be reached", - instance.container_type, - container.service_name, - ) - - if len(backup_containers) == 0: - logger.info("No containers in the project has 'stack-back.*' label") - - logger.info("-" * 67) - - -def backup(config, containers: RunningContainers): - """Request a backup to start""" - # Make sure we don't spawn multiple backup processes - if containers.backup_process_running: - alerts.send( - subject="Backup process container already running", - body=( - "A backup process container is already running. \n" - f"Id: {containers.backup_process_container.id}\n" - f"Name: {containers.backup_process_container.name}\n" - ), - alert_type="ERROR", - ) - raise RuntimeError("Backup process already running") - - # Map all volumes from the backup container into the backup process container - volumes = containers.this_container.volumes - - # Map volumes from other containers we are backing up - mounts = containers.generate_backup_mounts("/volumes") - volumes.update(mounts) - - logger.debug( - "Starting backup container with image %s", containers.this_container.image - ) - try: - result = backup_runner.run( - image=containers.this_container.image, - command="rcb start-backup-process", - volumes=volumes, - environment=containers.this_container.environment, - source_container_id=containers.this_container.id, - labels={ - containers.backup_process_label: "True", - "com.docker.compose.project": containers.project_name, - }, - ) - except Exception as ex: - logger.exception(ex) - alerts.send( - subject="Exception during backup", - body=str(ex), - alert_type="ERROR", - ) - return - - logger.info("Backup container exit code: %s", result) - - # Alert the user if something went wrong - if result != 0: - alerts.send( - subject="Backup process exited with non-zero code", - body=open("backup.log").read(), - alert_type="ERROR", - ) - - -def start_backup_process(config, containers): - """The actual backup process running inside the spawned container""" - if not utils.is_true(os.environ.get("BACKUP_PROCESS_CONTAINER")): - logger.error( - "Cannot run backup process in this container. Use backup command instead. " - "This will spawn a new container with the necessary mounts." - ) - alerts.send( - subject="Cannot run backup process in this container", - body=( - "Cannot run backup process in this container. Use backup command instead. " - "This will spawn a new container with the necessary mounts." - ), - ) - exit(1) - - status(config, containers) - errors = False - - # Did we actually get any volumes mounted? - try: - has_volumes = os.stat("/volumes") is not None - except FileNotFoundError: - logger.warning("Found no volumes to back up") - has_volumes = False - - # Warn if there is nothing to do - if len(containers.containers_for_backup()) == 0 and not has_volumes: - logger.error("No containers for backup found") - exit(1) - - # stop containers labeled to stop during backup - if len(containers.stop_during_backup_containers) > 0: - utils.stop_containers(containers.stop_during_backup_containers) - - # back up volumes - if has_volumes: - try: - logger.info("Backing up volumes") - vol_result = restic.backup_files(config.repository, source="/volumes") - logger.debug("Volume backup exit code: %s", vol_result) - if vol_result != 0: - logger.error("Volume backup exited with non-zero code: %s", vol_result) - errors = True - except Exception as ex: - logger.error("Exception raised during volume backup") - logger.exception(ex) - errors = True - - # back up databases - logger.info("Backing up databases") - for container in containers.containers_for_backup(): - if container.database_backup_enabled: - try: - instance = container.instance - logger.debug( - "Backing up %s in service %s from project %s", - instance.container_type, - instance.service_name, - instance.project_name, - ) - result = instance.backup() - logger.debug("Exit code: %s", result) - if result != 0: - logger.error("Backup command exited with non-zero code: %s", result) - errors = True - except Exception as ex: - logger.exception(ex) - errors = True - - # restart stopped containers after backup - if len(containers.stop_during_backup_containers) > 0: - utils.start_containers(containers.stop_during_backup_containers) - - if errors: - logger.error("Exit code: %s", errors) - exit(1) - - # Only run maintenance tasks if maintenance is not scheduled - if not config.maintenance_schedule: - maintenance(config, containers) - - logger.info("Backup completed") - - -def maintenance(config, containers): - """Run maintenance tasks""" - logger.info("Running maintenance tasks") - result = cleanup(config, containers) - if result != 0: - logger.error("Cleanup exit code: %s", result) - exit(1) - - logger.info("Checking the repository for errors") - check_with_cache = utils.is_true(config.check_with_cache) - result = restic.check(config.repository, with_cache=check_with_cache) - if result != 0: - logger.error("Check exit code: %s", result) - exit(1) - - -def cleanup(config, containers): - """Run forget / prune to minimize storage space""" - logger.info("Forget outdated snapshots") - forget_result = restic.forget( - config.repository, - config.keep_daily, - config.keep_weekly, - config.keep_monthly, - config.keep_yearly, - ) - logger.info("Prune stale data freeing storage space") - prune_result = restic.prune(config.repository) - return forget_result and prune_result - - -def snapshots(config, containers): - """Display restic snapshots""" - stdout, stderr = restic.snapshots(config.repository, last=True) - for line in stdout.decode().split("\n"): - print(line) - - -def alert(config, containers): - """Test alerts""" - logger.info("Testing alerts") - alerts.send( - subject="{}: Test Alert".format(containers.project_name), - body="Test message", - ) - - -def crontab(config): - """Generate the crontab""" - print(cron.generate_crontab(config)) - - -def dump_env(): - """Dump all environment variables to a file that can be sourced from cron""" - print("# This file was generated by stack-back") - for key, value in os.environ.items(): - print("export {}='{}'".format(key, value)) - - -def parse_args(): - parser = argparse.ArgumentParser(prog="restic_compose_backup") - parser.add_argument( - "action", - choices=[ - "status", - "snapshots", - "backup", - "start-backup-process", - "maintenance", - "alert", - "cleanup", - "version", - "crontab", - "dump-env", - "test", - ], - ) - parser.add_argument( - "--log-level", - default=None, - choices=list(log.LOG_LEVELS.keys()), - help="Log level", - ) - return parser.parse_args() - - -if __name__ == "__main__": - main() +import argparse +import os +import logging + +from restic_compose_backup import ( + alerts, + backup_runner, + hooks, + log, + restic, +) +from restic_compose_backup.config import Config +from restic_compose_backup.containers import RunningContainers +from restic_compose_backup import cron, enums, utils + +logger = logging.getLogger(__name__) + + +def main(): + """CLI entrypoint""" + args = parse_args() + config = Config() + log.setup(level=args.log_level or config.log_level) + containers = RunningContainers() + + # Ensure log level is propagated to parent container if overridden + if args.log_level: + containers.this_container.set_config_env("LOG_LEVEL", args.log_level) + + if args.action == "status": + status(config, containers) + + elif args.action == "snapshots": + snapshots(config, containers) + + elif args.action == "backup": + backup(config, containers) + + elif args.action == "start-backup-process": + start_backup_process(config, containers) + + elif args.action == "maintenance": + maintenance(config, containers) + + elif args.action == "cleanup": + cleanup(config, containers) + + elif args.action == "alert": + alert(config, containers) + + elif args.action == "version": + import restic_compose_backup + + print(restic_compose_backup.__version__) + + elif args.action == "crontab": + crontab(config) + + elif args.action == "dump-env": + dump_env() + + # Random test stuff here + elif args.action == "test": + nodes = utils.get_swarm_nodes() + print("Swarm nodes:") + for node in nodes: + addr = node.attrs["Status"]["Addr"] + state = node.attrs["Status"]["State"] + print(" - {} {} {}".format(node.id, addr, state)) + + +def _log_hook_summary(hook): + """Log a single hook's configuration in a readable format.""" + summary = f"[{hook.order}] {hook.cmd}" + if hook.context: + summary += f" | context: {hook.context}" + if hook.on_error != hooks.ON_ERROR_ABORT: + summary += f" | on-error: {hook.on_error}" + summary += f" | from: {hook.source_service_name}" + logger.info(" %s", summary) + + +def status(config, containers): + """Outputs the backup config for the compose setup""" + logger.info("Status for compose project '%s'", containers.project_name) + logger.info("Repository: '%s'", config.repository) + logger.info("Backup currently running?: %s", containers.backup_process_running) + logger.info( + "Include project name in backup path?: %s", + utils.is_true(config.include_project_name), + ) + logger.debug( + "Exclude bind mounts from backups?: %s", + utils.is_true(config.exclude_bind_mounts), + ) + logger.debug( + "Include all compose projects?: %s", + utils.is_true(config.include_all_compose_projects), + ) + logger.debug( + f"Use cache for integrity check?: {utils.is_true(config.check_with_cache)}" + ) + logger.info("Checking docker availability") + + utils.list_containers() + + if containers.stale_backup_process_containers: + utils.remove_containers(containers.stale_backup_process_containers) + + logger.info("Contacting repository") + if not restic.is_initialized(config.repository): + logger.info("Repository is not initialized. Attempting to initialize it.") + result = restic.init_repo(config.repository) + if result == 0: + logger.info("Successfully initialized repository: %s", config.repository) + else: + logger.error("Failed to initialize repository") + + logger.info("%s Detected Config %s", "-" * 25, "-" * 25) + + # Display backup configuration for each container + backup_containers = containers.containers_for_backup() + for container in backup_containers: + logger.info("service: %s", container.service_name) + + if container.volume_backup_enabled: + logger.info(f" - stop during backup: {container.stop_during_backup}") + for mount in container.filter_mounts(): + logger.info( + " - volume: %s -> %s", + mount.source, + container.get_volume_backup_destination(mount, "/volumes"), + ) + + if container.database_backup_enabled: + instance = container.instance + ping = instance.ping() + logger.info( + " - %s (is_ready=%s) -> %s", + instance.container_type, + ping, + instance.backup_destination_path(), + ) + if not ping: + logger.error( + "Database '%s' in service %s cannot be reached", + instance.container_type, + container.service_name, + ) + + if len(backup_containers) == 0: + logger.info("No containers in the project has 'stack-back.*' label") + + # Display backup order if configured + if utils.is_true( + containers.this_container.get_label(enums.LABEL_ORDERED) + ): + logger.info("%s Backup Order %s", "-" * 23, "-" * 23) + for position, container in enumerate(backup_containers, 1): + logger.info(" %d. %s", position, container.service_name) + + # Display hooks + has_hooks = False + for stage in hooks.HOOK_STAGES: + stage_hooks = hooks.collect_hooks( + stage, containers.this_container, backup_containers, + ) + if stage_hooks: + if not has_hooks: + logger.info( + "%s Hooks %s", "-" * 27, "-" * 27, + ) + has_hooks = True + logger.info(" %s:", stage) + for hook in stage_hooks: + _log_hook_summary(hook) + + logger.info("-" * 67) + + +def backup(config, containers: RunningContainers): + """Request a backup to start""" + # Make sure we don't spawn multiple backup processes + if containers.backup_process_running: + alerts.send( + subject="Backup process container already running", + body=( + "A backup process container is already running. \n" + f"Id: {containers.backup_process_container.id}\n" + f"Name: {containers.backup_process_container.name}\n" + ), + alert_type="ERROR", + ) + raise RuntimeError("Backup process already running") + + # Map all volumes from the backup container into the backup process container + volumes = containers.this_container.volumes + + # Map volumes from other containers we are backing up + mounts = containers.generate_backup_mounts("/volumes") + volumes.update(mounts) + + logger.debug( + "Starting backup container with image %s", containers.this_container.image + ) + + # Build labels for the backup process container, forwarding hook and + # ordering labels so the spawned container can read them. + process_labels = { + containers.backup_process_label: "True", + "com.docker.compose.project": containers.project_name, + } + for key, value in containers.this_container._labels.items(): + if ( + key.startswith("stack-back.hooks.") + or key.startswith("stack-back.order.") + or key == enums.LABEL_ORDERED + ): + process_labels[key] = value + + try: + result = backup_runner.run( + image=containers.this_container.image, + command="rcb start-backup-process", + volumes=volumes, + environment=containers.this_container.environment, + source_container_id=containers.this_container.id, + labels=process_labels, + ) + except Exception as ex: + logger.exception(ex) + alerts.send( + subject="Exception during backup", + body=str(ex), + alert_type="ERROR", + ) + return + + logger.info("Backup container exit code: %s", result) + + # Alert the user if something went wrong + if result != 0: + alerts.send( + subject="Backup process exited with non-zero code", + body=open("backup.log").read(), + alert_type="ERROR", + ) + + +def start_backup_process(config, containers): + """The actual backup process running inside the spawned container""" + if not utils.is_true(os.environ.get("BACKUP_PROCESS_CONTAINER")): + logger.error( + "Cannot run backup process in this container. Use backup command instead. " + "This will spawn a new container with the necessary mounts." + ) + alerts.send( + subject="Cannot run backup process in this container", + body=( + "Cannot run backup process in this container. Use backup command instead. " + "This will spawn a new container with the necessary mounts." + ), + ) + exit(1) + + status(config, containers) + errors = False + + # Did we actually get any volumes mounted? + try: + has_volumes = os.stat("/volumes") is not None + except FileNotFoundError: + logger.warning("Found no volumes to back up") + has_volumes = False + + backup_containers = containers.containers_for_backup() + + # Warn if there is nothing to do + if len(backup_containers) == 0 and not has_volumes: + logger.error("No containers for backup found") + exit(1) + + # Collect lifecycle hooks from backup container + target containers + pre_hooks = hooks.collect_hooks( + "pre", containers.this_container, backup_containers, + ) + post_hooks = hooks.collect_hooks( + "post", containers.this_container, backup_containers, + ) + error_hooks = hooks.collect_hooks( + "error", containers.this_container, backup_containers, + ) + finally_hooks = hooks.collect_hooks( + "finally", containers.this_container, backup_containers, + ) + + containers_stopped = False + + try: + # --- pre hooks --- + if not hooks.execute_hooks(pre_hooks, containers): + errors = True + + if not errors: + # stop containers labeled to stop during backup + if len(containers.stop_during_backup_containers) > 0: + utils.stop_containers(containers.stop_during_backup_containers) + containers_stopped = True + + # back up volumes + if has_volumes: + try: + logger.info("Backing up volumes") + vol_result = restic.backup_files(config.repository, source="/volumes") + logger.debug("Volume backup exit code: %s", vol_result) + if vol_result != 0: + logger.error("Volume backup exited with non-zero code: %s", vol_result) + errors = True + except Exception as ex: + logger.error("Exception raised during volume backup") + logger.exception(ex) + errors = True + + # back up databases + logger.info("Backing up databases") + for container in backup_containers: + if container.database_backup_enabled: + try: + instance = container.instance + logger.debug( + "Backing up %s in service %s from project %s", + instance.container_type, + instance.service_name, + instance.project_name, + ) + result = instance.backup() + logger.debug("Exit code: %s", result) + if result != 0: + logger.error("Backup command exited with non-zero code: %s", result) + errors = True + except Exception as ex: + logger.exception(ex) + errors = True + + except Exception as ex: + logger.exception(ex) + errors = True + + finally: + # Always restart stopped containers first, so that all subsequent + # hooks (post, error, finally) can exec into them. + if containers_stopped: + utils.start_containers(containers.stop_during_backup_containers) + + # --- post hooks (only when everything succeeded) --- + if not errors: + if not hooks.execute_hooks(post_hooks, containers): + errors = True + + # --- error hooks (only on failure) --- + if errors: + hooks.execute_hooks(error_hooks, containers) + + # --- finally hooks (always) --- + hooks.execute_hooks(finally_hooks, containers) + + if errors: + logger.error("Exit code: %s", 1) + exit(1) + + # Only run maintenance tasks if maintenance is not scheduled + if not config.maintenance_schedule: + maintenance(config, containers) + + logger.info("Backup completed") + + +def maintenance(config, containers): + """Run maintenance tasks""" + logger.info("Running maintenance tasks") + result = cleanup(config, containers) + if result != 0: + logger.error("Cleanup exit code: %s", result) + exit(1) + + logger.info("Checking the repository for errors") + check_with_cache = utils.is_true(config.check_with_cache) + result = restic.check(config.repository, with_cache=check_with_cache) + if result != 0: + logger.error("Check exit code: %s", result) + exit(1) + + +def cleanup(config, containers): + """Run forget / prune to minimize storage space""" + logger.info("Forget outdated snapshots") + forget_result = restic.forget( + config.repository, + config.keep_daily, + config.keep_weekly, + config.keep_monthly, + config.keep_yearly, + ) + logger.info("Prune stale data freeing storage space") + prune_result = restic.prune(config.repository) + return forget_result and prune_result + + +def snapshots(config, containers): + """Display restic snapshots""" + stdout, stderr = restic.snapshots(config.repository, last=True) + for line in stdout.decode().split("\n"): + print(line) + + +def alert(config, containers): + """Test alerts""" + logger.info("Testing alerts") + alerts.send( + subject="{}: Test Alert".format(containers.project_name), + body="Test message", + ) + + +def crontab(config): + """Generate the crontab""" + print(cron.generate_crontab(config)) + + +def dump_env(): + """Dump all environment variables to a file that can be sourced from cron""" + print("# This file was generated by stack-back") + for key, value in os.environ.items(): + print("export {}='{}'".format(key, value)) + + +def parse_args(): + parser = argparse.ArgumentParser(prog="restic_compose_backup") + parser.add_argument( + "action", + choices=[ + "status", + "snapshots", + "backup", + "start-backup-process", + "maintenance", + "alert", + "cleanup", + "version", + "crontab", + "dump-env", + "test", + ], + ) + parser.add_argument( + "--log-level", + default=None, + choices=list(log.LOG_LEVELS.keys()), + help="Log level", + ) + return parser.parse_args() + + +if __name__ == "__main__": + main() diff --git a/src/restic_compose_backup/containers.py b/src/restic_compose_backup/containers.py index 55ee256..e26477c 100644 --- a/src/restic_compose_backup/containers.py +++ b/src/restic_compose_backup/containers.py @@ -1,494 +1,577 @@ -import logging -from pathlib import Path -import socket -from typing import List - -from restic_compose_backup import enums, utils -from restic_compose_backup.config import config - -logger = logging.getLogger(__name__) - -VOLUME_TYPE_BIND = "bind" -VOLUME_TYPE_VOLUME = "volume" - - -class Container: - """Represents a docker container""" - - container_type = None - - def __init__(self, data: dict): - self._data = data - self._state = data.get("State") - self._config = data.get("Config") - self._mounts = [Mount(mnt, container=self) for mnt in data.get("Mounts")] - - if not self._state: - raise ValueError("Container meta missing State") - if self._config is None: - raise ValueError("Container meta missing Config") - - self._labels = self._config.get("Labels") - if self._labels is None: - raise ValueError("Container meta missing Config->Labels") - - self._include = self._parse_pattern(self.get_label(enums.LABEL_VOLUMES_INCLUDE)) - self._exclude = self._parse_pattern(self.get_label(enums.LABEL_VOLUMES_EXCLUDE)) - - @property - def instance(self) -> "Container": - """Container: Get a service specific subclass instance""" - # TODO: Do this smarter in the future (simple registry) - if self.database_backup_enabled: - from restic_compose_backup import containers_db - - if self.mariadb_backup_enabled: - return containers_db.MariadbContainer(self._data) - if self.mysql_backup_enabled: - return containers_db.MysqlContainer(self._data) - if self.postgresql_backup_enabled: - return containers_db.PostgresContainer(self._data) - else: - return self - - @property - def id(self) -> str: - """str: The id of the container""" - return self._data.get("Id") - - @property - def image(self) -> str: - """Image name""" - return self.get_config("Image") - - @property - def name(self) -> str: - """Container name""" - return self._data["Name"].replace("/", "") - - @property - def service_name(self) -> str: - """Name of the container/service""" - return self.get_label( - "com.docker.compose.service", default="" - ) or self.get_label("com.docker.swarm.service.name", default="") - - @property - def backup_process_label(self) -> str: - """str: The unique backup process label for this project""" - return f"{enums.LABEL_BACKUP_PROCESS}-{self.project_name}" - - @property - def project_name(self) -> str: - """str: Name of the compose setup""" - return self.get_label("com.docker.compose.project", default="") - - @property - def stack_name(self) -> str: - """str: Name of the stack is present""" - return self.get_label("com.docker.stack.namespace") - - @property - def is_oneoff(self) -> bool: - """Was this container started with run command?""" - return self.get_label("com.docker.compose.oneoff", default="False") == "True" - - @property - def environment(self) -> list: - """All configured env vars for the container as a list""" - return self.get_config("Env") - - def remove(self): - self._data.remove() - - def get_config_env(self, name) -> str: - """Get a config environment variable by name""" - # convert to dict and fetch env var by name - data = {i[0 : i.find("=")]: i[i.find("=") + 1 :] for i in self.environment} - return data.get(name) - - def set_config_env(self, name, value): - """Set an environment variable""" - env = self.environment - new_value = f"{name}={value}" - for i, entry in enumerate(env): - if f"{name}=" in entry: - env[i] = new_value - break - else: - env.append(new_value) - - @property - def volumes(self) -> dict: - """ - Return volumes for the container in the following format: - {'/home/user1/': {'bind': '/mnt/vol2', 'mode': 'rw'},} - """ - volumes = {} - for mount in self._mounts: - volumes[mount.source] = { - "bind": mount.destination, - "mode": "rw", - } - - return volumes - - @property - def backup_enabled(self) -> bool: - """Is backup enabled for this container?""" - return any( - [ - self.volume_backup_enabled, - self.database_backup_enabled, - ] - ) - - @property - def volume_backup_enabled(self) -> bool: - """bool: If the ``stack-back.volumes`` label is set""" - explicitly_enabled = utils.is_true(self.get_label(enums.LABEL_VOLUMES_ENABLED)) - explicitly_disabled = utils.is_false( - self.get_label(enums.LABEL_VOLUMES_ENABLED) - ) - automatically_enabled = utils.is_true(config.auto_backup_all) - return explicitly_enabled or (automatically_enabled and not explicitly_disabled) - - @property - def database_backup_enabled(self) -> bool: - """bool: Is database backup enabled in any shape or form?""" - return any( - [ - self.mysql_backup_enabled, - self.mariadb_backup_enabled, - self.postgresql_backup_enabled, - ] - ) - - @property - def mysql_backup_enabled(self) -> bool: - """bool: If the ``stack-back.mysql`` label is set""" - explicity_enabled = utils.is_true(self.get_label(enums.LABEL_MYSQL_ENABLED)) - explicity_disabled = utils.is_false(self.get_label(enums.LABEL_MYSQL_ENABLED)) - automatically_enabled = utils.is_true( - config.auto_backup_all - ) and self.image.startswith("mysql") - return explicity_enabled or (automatically_enabled and not explicity_disabled) - - @property - def mariadb_backup_enabled(self) -> bool: - """bool: If the ``stack-back.mariadb`` label is set""" - explicity_enabled = utils.is_true(self.get_label(enums.LABEL_MARIADB_ENABLED)) - explicity_disabled = utils.is_false(self.get_label(enums.LABEL_MARIADB_ENABLED)) - automatically_enabled = utils.is_true( - config.auto_backup_all - ) and self.image.startswith("mariadb") - return explicity_enabled or (automatically_enabled and not explicity_disabled) - - @property - def postgresql_backup_enabled(self) -> bool: - """bool: If the ``stack-back.postgres`` label is set""" - explicity_enabled = utils.is_true(self.get_label(enums.LABEL_POSTGRES_ENABLED)) - explicity_disabled = utils.is_false( - self.get_label(enums.LABEL_POSTGRES_ENABLED) - ) - automatically_enabled = utils.is_true( - config.auto_backup_all - ) and self.image.startswith("postgres") - return explicity_enabled or (automatically_enabled and not explicity_disabled) - - @property - def stop_during_backup(self) -> bool: - """bool: If the ``stack-back.volumes.stop-during-backup`` label is set""" - return ( - utils.is_true(self.get_label(enums.LABEL_STOP_DURING_BACKUP)) - and not self.database_backup_enabled - ) - - @property - def is_backup_process_container(self) -> bool: - """Is this container the running backup process?""" - return self.get_label(self.backup_process_label) == "True" - - @property - def is_running(self) -> bool: - """bool: Is the container running?""" - return self._state.get("Running", False) - - def get_config(self, name, default=None): - """Get value from config dict""" - return self._config.get(name, default) - - def get_label(self, name, default=None): - """Get a label by name""" - return self._labels.get(name, None) - - def filter_mounts(self): - """Get all mounts for this container matching include/exclude filters""" - filtered = [] - database_mounts = [ - "/var/lib/mysql", - "/var/lib/mariadb", - "/var/lib/postgresql/data", - ] - - # If exclude_bind_mounts is true, only volume mounts are kept in the list of mounts - exclude_bind_mounts = utils.is_true(config.exclude_bind_mounts) - mounts = list( - filter( - lambda m: not exclude_bind_mounts or m.type == "volume", self._mounts - ) - ) - - if not self.volume_backup_enabled: - return filtered - - if self._include: - for mount in mounts: - for pattern in self._include: - if pattern in mount.source: - break - else: - continue - - filtered.append(mount) - - elif self._exclude: - for mount in mounts: - for pattern in self._exclude: - if pattern in mount.source: - break - else: - filtered.append(mount) - else: - for mount in mounts: - if ( - self.database_backup_enabled - and mount.destination in database_mounts - ): - continue - filtered.append(mount) - - return filtered - - def volumes_for_backup(self, source_prefix="/volumes", mode="ro"): - """Get volumes configured for backup""" - mounts = self.filter_mounts() - volumes = {} - for mount in mounts: - volumes[mount.source] = { - "bind": self.get_volume_backup_destination(mount, source_prefix), - "mode": mode, - } - - return volumes - - def get_volume_backup_destination(self, mount, source_prefix) -> str: - """Get the destination path for backups of the given mount""" - destination = Path(source_prefix) - - if utils.is_true(config.include_project_name): - project_name = self.project_name - if project_name != "": - destination /= project_name - - destination /= self.service_name - destination /= Path(utils.strip_root(mount.destination)) - - return str(destination) - - def get_credentials(self) -> dict: - """dict: get credentials for the service""" - raise NotImplementedError("Base container class don't implement this") - - def ping(self) -> bool: - """Check the availability of the service""" - raise NotImplementedError("Base container class don't implement this") - - def backup(self): - """Back up this service""" - raise NotImplementedError("Base container class don't implement this") - - def backup_destination_path(self) -> str: - """Return the path backups will be saved at""" - raise NotImplementedError("Base container class don't implement this") - - def dump_command(self) -> list: - """list: create a dump command restic and use to send data through stdin""" - raise NotImplementedError("Base container class don't implement this") - - def _parse_pattern(self, value: str) -> List[str]: - """list: Safely parse include/exclude pattern from user""" - if not value: - return None - - if type(value) is not str: - return None - - value = value.strip() - if len(value) == 0: - return None - - return value.split(",") - - def __eq__(self, other): - """Compare container by id""" - if other is None: - return False - - if not isinstance(other, Container): - return False - - return self.id == other.id - - def __repr__(self): - return str(self) - - def __str__(self): - return "".format(self.name) - - -class Mount: - """Represents a volume mount (volume or bind)""" - - def __init__(self, data, container=None): - self._data = data - self._container = container - - @property - def container(self) -> Container: - """The container this mount belongs to""" - return self._container - - @property - def type(self) -> str: - """bind/volume""" - return self._data.get("Type") - - @property - def name(self) -> str: - """Name of the mount""" - return self._data.get("Name") - - @property - def source(self) -> str: - """Source of the mount. Volume name or path""" - return self._data.get("Source") - - @property - def destination(self) -> str: - """Destination path for the volume mount in the container""" - return self._data.get("Destination") - - def __repr__(self) -> str: - return str(self) - - def __str__(self) -> str: - return str(self._data) - - def __hash__(self): - """Uniqueness for a volume""" - if self.type == VOLUME_TYPE_VOLUME: - return hash(self.name) - elif self.type == VOLUME_TYPE_BIND: - return hash(self.source) - else: - raise ValueError("Unknown volume type: {}".format(self.type)) - - -class RunningContainers: - def __init__(self): - all_containers = utils.list_containers() - self.containers = [] - self.this_container = None - self.backup_process_container = None - self.stale_backup_process_containers = [] - self.stop_during_backup_containers = [] - - # Find the container we are running in. - # If we don't have this information we cannot continue - for container_data in all_containers: - if container_data.get("Id").startswith(socket.gethostname()): - self.this_container = Container(container_data) - - if not self.this_container: - raise ValueError("Cannot find metadata for backup container") - - # Gather relevant containers - for container_data in all_containers: - container = Container(container_data) - - # Gather stale backup process containers - if ( - self.this_container.image == container.image - and not container.is_running - and container.is_backup_process_container - ): - self.stale_backup_process_containers.append(container) - - # We only care about running containers after this point - if not container.is_running: - continue - - # If not swarm mode we need to filter in compose project - if ( - not config.swarm_mode - and not config.include_all_compose_projects - and container.project_name != self.this_container.project_name - ): - continue - - # Gather stop during backup containers - if container.stop_during_backup: - self.stop_during_backup_containers.append(container) - - # Detect running backup process container - if container.is_backup_process_container: - self.backup_process_container = container - - # Containers started manually are not included - if container.is_oneoff: - continue - - # Do not include the stack-back and backup process containers - if "stack-back" in container.image: - continue - - self.containers.append(container) - - @property - def project_name(self) -> str: - """str: Name of the compose project""" - return self.this_container.project_name - - @property - def backup_process_label(self) -> str: - """str: The backup process label for this project""" - return self.this_container.backup_process_label - - @property - def backup_process_running(self) -> bool: - """Is the backup process container running?""" - return self.backup_process_container is not None - - def containers_for_backup(self) -> list[Container]: - """Obtain all containers with backup enabled""" - return [container for container in self.containers if container.backup_enabled] - - def generate_backup_mounts(self, dest_prefix="/volumes") -> dict: - """Generate mounts for backup for the entire compose setup""" - mounts = {} - for container in self.containers_for_backup(): - if container.volume_backup_enabled: - mounts.update( - container.volumes_for_backup(source_prefix=dest_prefix, mode="ro") - ) - - return mounts - - def get_service(self, name) -> Container: - """Container: Get a service by name""" - for container in self.containers: - if container.service_name == name: - return container - - return None +import logging +from pathlib import Path +import socket +from typing import List + +from restic_compose_backup import enums, utils +from restic_compose_backup.config import config + +logger = logging.getLogger(__name__) + +VOLUME_TYPE_BIND = "bind" +VOLUME_TYPE_VOLUME = "volume" + + +class Container: + """Represents a docker container""" + + container_type = None + + def __init__(self, data: dict): + self._data = data + self._state = data.get("State") + self._config = data.get("Config") + self._mounts = [Mount(mnt, container=self) for mnt in data.get("Mounts")] + + if not self._state: + raise ValueError("Container meta missing State") + if self._config is None: + raise ValueError("Container meta missing Config") + + self._labels = self._config.get("Labels") + if self._labels is None: + raise ValueError("Container meta missing Config->Labels") + + self._include = self._parse_pattern(self.get_label(enums.LABEL_VOLUMES_INCLUDE)) + self._exclude = self._parse_pattern(self.get_label(enums.LABEL_VOLUMES_EXCLUDE)) + + @property + def instance(self) -> "Container": + """Container: Get a service specific subclass instance""" + # TODO: Do this smarter in the future (simple registry) + if self.database_backup_enabled: + from restic_compose_backup import containers_db + + if self.mariadb_backup_enabled: + return containers_db.MariadbContainer(self._data) + if self.mysql_backup_enabled: + return containers_db.MysqlContainer(self._data) + if self.postgresql_backup_enabled: + return containers_db.PostgresContainer(self._data) + else: + return self + + @property + def id(self) -> str: + """str: The id of the container""" + return self._data.get("Id") + + @property + def image(self) -> str: + """Image name""" + return self.get_config("Image") + + @property + def name(self) -> str: + """Container name""" + return self._data["Name"].replace("/", "") + + @property + def service_name(self) -> str: + """Name of the container/service""" + return self.get_label( + "com.docker.compose.service", default="" + ) or self.get_label("com.docker.swarm.service.name", default="") + + @property + def backup_process_label(self) -> str: + """str: The unique backup process label for this project""" + return f"{enums.LABEL_BACKUP_PROCESS}-{self.project_name}" + + @property + def project_name(self) -> str: + """str: Name of the compose setup""" + return self.get_label("com.docker.compose.project", default="") + + @property + def stack_name(self) -> str: + """str: Name of the stack is present""" + return self.get_label("com.docker.stack.namespace") + + @property + def is_oneoff(self) -> bool: + """Was this container started with run command?""" + return self.get_label("com.docker.compose.oneoff", default="False") == "True" + + @property + def environment(self) -> list: + """All configured env vars for the container as a list""" + return self.get_config("Env") + + def remove(self): + self._data.remove() + + def get_config_env(self, name) -> str: + """Get a config environment variable by name""" + # convert to dict and fetch env var by name + data = {i[0 : i.find("=")]: i[i.find("=") + 1 :] for i in self.environment} + return data.get(name) + + def set_config_env(self, name, value): + """Set an environment variable""" + env = self.environment + new_value = f"{name}={value}" + for i, entry in enumerate(env): + if f"{name}=" in entry: + env[i] = new_value + break + else: + env.append(new_value) + + @property + def volumes(self) -> dict: + """ + Return volumes for the container in the following format: + {'/home/user1/': {'bind': '/mnt/vol2', 'mode': 'rw'},} + """ + volumes = {} + for mount in self._mounts: + volumes[mount.source] = { + "bind": mount.destination, + "mode": "rw", + } + + return volumes + + @property + def backup_enabled(self) -> bool: + """Is backup enabled for this container?""" + return any( + [ + self.volume_backup_enabled, + self.database_backup_enabled, + ] + ) + + @property + def volume_backup_enabled(self) -> bool: + """bool: If the ``stack-back.volumes`` label is set""" + explicitly_enabled = utils.is_true(self.get_label(enums.LABEL_VOLUMES_ENABLED)) + explicitly_disabled = utils.is_false( + self.get_label(enums.LABEL_VOLUMES_ENABLED) + ) + automatically_enabled = utils.is_true(config.auto_backup_all) + return explicitly_enabled or (automatically_enabled and not explicitly_disabled) + + @property + def database_backup_enabled(self) -> bool: + """bool: Is database backup enabled in any shape or form?""" + return any( + [ + self.mysql_backup_enabled, + self.mariadb_backup_enabled, + self.postgresql_backup_enabled, + ] + ) + + @property + def mysql_backup_enabled(self) -> bool: + """bool: If the ``stack-back.mysql`` label is set""" + explicity_enabled = utils.is_true(self.get_label(enums.LABEL_MYSQL_ENABLED)) + explicity_disabled = utils.is_false(self.get_label(enums.LABEL_MYSQL_ENABLED)) + automatically_enabled = utils.is_true( + config.auto_backup_all + ) and self.image.startswith("mysql") + return explicity_enabled or (automatically_enabled and not explicity_disabled) + + @property + def mariadb_backup_enabled(self) -> bool: + """bool: If the ``stack-back.mariadb`` label is set""" + explicity_enabled = utils.is_true(self.get_label(enums.LABEL_MARIADB_ENABLED)) + explicity_disabled = utils.is_false(self.get_label(enums.LABEL_MARIADB_ENABLED)) + automatically_enabled = utils.is_true( + config.auto_backup_all + ) and self.image.startswith("mariadb") + return explicity_enabled or (automatically_enabled and not explicity_disabled) + + @property + def postgresql_backup_enabled(self) -> bool: + """bool: If the ``stack-back.postgres`` label is set""" + explicity_enabled = utils.is_true(self.get_label(enums.LABEL_POSTGRES_ENABLED)) + explicity_disabled = utils.is_false( + self.get_label(enums.LABEL_POSTGRES_ENABLED) + ) + automatically_enabled = utils.is_true( + config.auto_backup_all + ) and self.image.startswith("postgres") + return explicity_enabled or (automatically_enabled and not explicity_disabled) + + @property + def stop_during_backup(self) -> bool: + """bool: If the ``stack-back.volumes.stop-during-backup`` label is set""" + return ( + utils.is_true(self.get_label(enums.LABEL_STOP_DURING_BACKUP)) + and not self.database_backup_enabled + ) + + @property + def is_backup_process_container(self) -> bool: + """Is this container the running backup process?""" + return self.get_label(self.backup_process_label) == "True" + + @property + def is_running(self) -> bool: + """bool: Is the container running?""" + return self._state.get("Running", False) + + def get_config(self, name, default=None): + """Get value from config dict""" + return self._config.get(name, default) + + def get_label(self, name, default=None): + """Get a label by name""" + return self._labels.get(name, None) + + def filter_mounts(self): + """Get all mounts for this container matching include/exclude filters""" + filtered = [] + database_mounts = [ + "/var/lib/mysql", + "/var/lib/mariadb", + "/var/lib/postgresql/data", + ] + + # If exclude_bind_mounts is true, only volume mounts are kept in the list of mounts + exclude_bind_mounts = utils.is_true(config.exclude_bind_mounts) + mounts = list( + filter( + lambda m: not exclude_bind_mounts or m.type == "volume", self._mounts + ) + ) + + if not self.volume_backup_enabled: + return filtered + + if self._include: + for mount in mounts: + for pattern in self._include: + if pattern in mount.source: + break + else: + continue + + filtered.append(mount) + + elif self._exclude: + for mount in mounts: + for pattern in self._exclude: + if pattern in mount.source: + break + else: + filtered.append(mount) + else: + for mount in mounts: + if ( + self.database_backup_enabled + and mount.destination in database_mounts + ): + continue + filtered.append(mount) + + return filtered + + def volumes_for_backup(self, source_prefix="/volumes", mode="ro"): + """Get volumes configured for backup""" + mounts = self.filter_mounts() + volumes = {} + for mount in mounts: + volumes[mount.source] = { + "bind": self.get_volume_backup_destination(mount, source_prefix), + "mode": mode, + } + + return volumes + + def get_volume_backup_destination(self, mount, source_prefix) -> str: + """Get the destination path for backups of the given mount""" + destination = Path(source_prefix) + + if utils.is_true(config.include_project_name): + project_name = self.project_name + if project_name != "": + destination /= project_name + + destination /= self.service_name + destination /= Path(utils.strip_root(mount.destination)) + + return str(destination) + + def get_credentials(self) -> dict: + """dict: get credentials for the service""" + raise NotImplementedError("Base container class don't implement this") + + def ping(self) -> bool: + """Check the availability of the service""" + raise NotImplementedError("Base container class don't implement this") + + def backup(self): + """Back up this service""" + raise NotImplementedError("Base container class don't implement this") + + def backup_destination_path(self) -> str: + """Return the path backups will be saved at""" + raise NotImplementedError("Base container class don't implement this") + + def dump_command(self) -> list: + """list: create a dump command restic and use to send data through stdin""" + raise NotImplementedError("Base container class don't implement this") + + def _parse_pattern(self, value: str) -> List[str]: + """list: Safely parse include/exclude pattern from user""" + if not value: + return None + + if type(value) is not str: + return None + + value = value.strip() + if len(value) == 0: + return None + + return value.split(",") + + def __eq__(self, other): + """Compare container by id""" + if other is None: + return False + + if not isinstance(other, Container): + return False + + return self.id == other.id + + def __repr__(self): + return str(self) + + def __str__(self): + return "".format(self.name) + + +class Mount: + """Represents a volume mount (volume or bind)""" + + def __init__(self, data, container=None): + self._data = data + self._container = container + + @property + def container(self) -> Container: + """The container this mount belongs to""" + return self._container + + @property + def type(self) -> str: + """bind/volume""" + return self._data.get("Type") + + @property + def name(self) -> str: + """Name of the mount""" + return self._data.get("Name") + + @property + def source(self) -> str: + """Source of the mount. Volume name or path""" + return self._data.get("Source") + + @property + def destination(self) -> str: + """Destination path for the volume mount in the container""" + return self._data.get("Destination") + + def __repr__(self) -> str: + return str(self) + + def __str__(self) -> str: + return str(self._data) + + def __hash__(self): + """Uniqueness for a volume""" + if self.type == VOLUME_TYPE_VOLUME: + return hash(self.name) + elif self.type == VOLUME_TYPE_BIND: + return hash(self.source) + else: + raise ValueError("Unknown volume type: {}".format(self.type)) + + +class RunningContainers: + def __init__(self): + all_containers = utils.list_containers() + self.containers = [] + self.this_container = None + self.backup_process_container = None + self.stale_backup_process_containers = [] + self.stop_during_backup_containers = [] + + # Find the container we are running in. + # If we don't have this information we cannot continue + for container_data in all_containers: + if container_data.get("Id").startswith(socket.gethostname()): + self.this_container = Container(container_data) + + if not self.this_container: + raise ValueError("Cannot find metadata for backup container") + + # Gather relevant containers + for container_data in all_containers: + container = Container(container_data) + + # Gather stale backup process containers + if ( + self.this_container.image == container.image + and not container.is_running + and container.is_backup_process_container + ): + self.stale_backup_process_containers.append(container) + + # We only care about running containers after this point + if not container.is_running: + continue + + # If not swarm mode we need to filter in compose project + if ( + not config.swarm_mode + and not config.include_all_compose_projects + and container.project_name != self.this_container.project_name + ): + continue + + # Gather stop during backup containers + if container.stop_during_backup: + self.stop_during_backup_containers.append(container) + + # Detect running backup process container + if container.is_backup_process_container: + self.backup_process_container = container + + # Containers started manually are not included + if container.is_oneoff: + continue + + # Do not include the stack-back and backup process containers + if "stack-back" in container.image: + continue + + self.containers.append(container) + + @property + def project_name(self) -> str: + """str: Name of the compose project""" + return self.this_container.project_name + + @property + def backup_process_label(self) -> str: + """str: The backup process label for this project""" + return self.this_container.backup_process_label + + @property + def backup_process_running(self) -> bool: + """Is the backup process container running?""" + return self.backup_process_container is not None + + def containers_for_backup(self) -> list[Container]: + """Obtain all containers with backup enabled. + + When ``stack-back.ordered`` is set on the backup container, the + returned list respects the ``stack-back.order.`` sequence. + Containers not mentioned in the order list are appended afterwards. + """ + enabled = [ + container for container in self.containers + if container.backup_enabled + ] + + order = self._parse_backup_order() + if not order: + return enabled + + containers_by_service = { + container.service_name: container + for container in enabled + } + ordered: list[Container] = [] + seen: set[str] = set() + + for service_name in order: + if service_name in seen: + logger.warning( + "Duplicate service '%s' in backup order, skipping", + service_name, + ) + continue + seen.add(service_name) + + if service_name in containers_by_service: + ordered.append(containers_by_service[service_name]) + else: + logger.warning( + "Ordered service '%s' not found in backup containers", + service_name, + ) + + # Append containers not mentioned in the order list + for container in enabled: + if container.service_name not in seen: + logger.warning( + "Service '%s' not in backup order, appending after ordered services", + container.service_name, + ) + ordered.append(container) + + return ordered + + def _parse_backup_order(self) -> list[str]: + """Parse ``stack-back.order.`` labels from the backup container. + + Returns an ordered list of service names, or an empty list when + ordering is disabled or no order labels are found. + """ + if not self.this_container: + return [] + + if not utils.is_true(self.this_container.get_label(enums.LABEL_ORDERED)): + return [] + + order_map: dict[int, str] = {} + for label, value in self.this_container._labels.items(): + if not label.startswith(enums.LABEL_ORDER_PREFIX): + continue + suffix = label[len(enums.LABEL_ORDER_PREFIX):] + try: + order_num = int(suffix) + except ValueError: + logger.warning("Invalid order number in label '%s'", label) + continue + order_map[order_num] = value + + if not order_map: + logger.warning( + "stack-back.ordered is set but no order labels found, " + "falling back to default order" + ) + return [] + + return [ + order_map[position] + for position in sorted(order_map.keys()) + ] + + def generate_backup_mounts(self, dest_prefix="/volumes") -> dict: + """Generate mounts for backup for the entire compose setup""" + mounts = {} + for container in self.containers_for_backup(): + if container.volume_backup_enabled: + mounts.update( + container.volumes_for_backup(source_prefix=dest_prefix, mode="ro") + ) + + return mounts + + def get_service(self, name) -> Container: + """Container: Get a service by name""" + for container in self.containers: + if container.service_name == name: + return container + + return None diff --git a/src/restic_compose_backup/enums.py b/src/restic_compose_backup/enums.py index 788da14..2ce0ab3 100644 --- a/src/restic_compose_backup/enums.py +++ b/src/restic_compose_backup/enums.py @@ -1,11 +1,14 @@ -# Labels -LABEL_VOLUMES_ENABLED = "stack-back.volumes" -LABEL_VOLUMES_INCLUDE = "stack-back.volumes.include" -LABEL_VOLUMES_EXCLUDE = "stack-back.volumes.exclude" -LABEL_STOP_DURING_BACKUP = "stack-back.volumes.stop-during-backup" - -LABEL_MYSQL_ENABLED = "stack-back.mysql" -LABEL_POSTGRES_ENABLED = "stack-back.postgres" -LABEL_MARIADB_ENABLED = "stack-back.mariadb" - -LABEL_BACKUP_PROCESS = "stack-back.process" +# Labels +LABEL_VOLUMES_ENABLED = "stack-back.volumes" +LABEL_VOLUMES_INCLUDE = "stack-back.volumes.include" +LABEL_VOLUMES_EXCLUDE = "stack-back.volumes.exclude" +LABEL_STOP_DURING_BACKUP = "stack-back.volumes.stop-during-backup" + +LABEL_MYSQL_ENABLED = "stack-back.mysql" +LABEL_POSTGRES_ENABLED = "stack-back.postgres" +LABEL_MARIADB_ENABLED = "stack-back.mariadb" + +LABEL_BACKUP_PROCESS = "stack-back.process" + +LABEL_ORDERED = "stack-back.ordered" +LABEL_ORDER_PREFIX = "stack-back.order." \ No newline at end of file diff --git a/src/restic_compose_backup/hooks.py b/src/restic_compose_backup/hooks.py new file mode 100644 index 0000000..3995d34 --- /dev/null +++ b/src/restic_compose_backup/hooks.py @@ -0,0 +1,265 @@ +"""Backup hooks — pre, post, error, and finally lifecycle hooks. + +Hooks allow running user-defined commands before and after the backup process. +They follow try/catch/finally semantics: + +- **pre**: runs before backup starts +- **post**: runs only if all pre hooks and the backup itself succeeded +- **error**: runs only if a hook or the backup failed +- **finally**: always runs, regardless of outcome + +Hooks can be defined on the backup container (global) or on target containers +(per-service). For **pre** hooks, backup-container hooks execute first +(setup). For **post/error/finally** hooks, the order is reversed: +target-container hooks execute first, then backup-container hooks (teardown). + +Label format:: + + stack-back.hooks...cmd command string + stack-back.hooks...context service name to exec into (optional) + stack-back.hooks...on-error "abort" (default) | "continue" + +When ``context`` is omitted, the hook runs in the container where the label +is defined: backup-container hooks run locally (``commands.run``), target- +container hooks run via ``docker exec`` into that target container. An +explicit ``context`` overrides this and execs into the named service +regardless of where the label is defined. +""" + +import logging +from dataclasses import dataclass +from typing import List, Optional + +from restic_compose_backup import commands + +logger = logging.getLogger(__name__) + +HOOK_STAGES = ("pre", "post", "error", "finally") +LABEL_PREFIX = "stack-back.hooks." +ON_ERROR_ABORT = "abort" +ON_ERROR_CONTINUE = "continue" + + +@dataclass +class Hook: + """A single hook command to execute at a lifecycle stage.""" + + stage: str + order: int + cmd: str + # None → runs in the container where the label is defined + context: Optional[str] = None + on_error: str = ON_ERROR_ABORT + source_container_id: Optional[str] = None # set by collect_hooks() + source_service_name: Optional[str] = None # set by collect_hooks() + + +def parse_hooks_from_labels(labels: dict, stage: str) -> List[Hook]: + """Parse hook definitions from container labels for a given stage. + + Labels follow the pattern:: + + stack-back.hooks...cmd + stack-back.hooks...context (optional) + stack-back.hooks...on-error (optional, default: abort) + + Returns hooks sorted by order number. Entries without a ``cmd`` property + are skipped with a warning. + """ + prefix = f"{LABEL_PREFIX}{stage}." + hook_data: dict[int, dict[str, str]] = {} + + for label, value in labels.items(): + if not label.startswith(prefix): + continue + + # Extract "." from the label suffix after the prefix + label_suffix = label[len(prefix):] + parts = label_suffix.split(".", 1) + if len(parts) != 2: + logger.warning( + "Malformed hook label '%s', expected .", label + ) + continue + + try: + order_num = int(parts[0]) + except ValueError: + logger.warning("Invalid hook order number in label '%s'", label) + continue + + property_name = parts[1] + if order_num not in hook_data: + hook_data[order_num] = {} + hook_data[order_num][property_name] = value + + hooks: List[Hook] = [] + for order_num in sorted(hook_data.keys()): + data = hook_data[order_num] + cmd = data.get("cmd") + if not cmd: + logger.warning("Hook %s.%d has no 'cmd', skipping", stage, order_num) + continue + + on_error = data.get("on-error", ON_ERROR_ABORT) + if on_error not in (ON_ERROR_ABORT, ON_ERROR_CONTINUE): + logger.warning( + "Invalid on-error value '%s' for hook %s.%d, defaulting to '%s'", + on_error, + stage, + order_num, + ON_ERROR_ABORT, + ) + on_error = ON_ERROR_ABORT + + hooks.append( + Hook( + stage=stage, + order=order_num, + cmd=cmd, + context=data.get("context"), + on_error=on_error, + ) + ) + + return hooks + + +def collect_hooks(stage, backup_container, target_containers): + """Collect all hooks for *stage* from the backup container and targets. + + For **pre** hooks the backup-container hooks run first (set up global + state before per-service preparation). + + For **post / error / finally** hooks the order is reversed: target- + container hooks run first, then backup-container hooks (tear down + per-service state before global cleanup). This gives symmetric + setup / teardown semantics:: + + pre: backup-container → target containers (setup) + post: target containers → backup-container (teardown) + error: target containers → backup-container (teardown) + finally: target containers → backup-container (teardown) + + Source container metadata (id, service name) is attached to every hook + so that context-free hooks know where to execute. + """ + backup_hooks: List[Hook] = [] + for hook in parse_hooks_from_labels(backup_container._labels, stage): + hook.source_container_id = backup_container.id + hook.source_service_name = backup_container.service_name + backup_hooks.append(hook) + + target_hooks: List[Hook] = [] + for container in target_containers: + for hook in parse_hooks_from_labels(container._labels, stage): + hook.source_container_id = container.id + hook.source_service_name = container.service_name + target_hooks.append(hook) + + if stage == "pre": + # Setup: global first, then per-service + return backup_hooks + target_hooks + else: + # Teardown (post / error / finally): per-service first, then global + return target_hooks + backup_hooks + + +def execute_hooks(hooks, containers): + """Execute *hooks* in order, resolving each hook's target context. + + Returns ``True`` when every hook either succeeds (exit code 0) or + fails with ``on_error="continue"``. Returns ``False`` as soon as + any hook fails with ``on_error="abort"`` (the default); remaining + hooks in the list are skipped. + + An empty *hooks* list is always considered successful. + """ + if not hooks: + return True + + for hook in hooks: + context_info = f" (context: {hook.context})" if hook.context else "" + logger.info( + "Running %s hook [%d]: %s%s", + hook.stage, + hook.order, + hook.cmd, + context_info, + ) + + try: + exit_code = _execute_hook(hook, containers) + except Exception as ex: + logger.error("Hook execution error: %s", ex) + exit_code = 1 + + if exit_code != 0: + if hook.on_error == ON_ERROR_ABORT: + logger.error( + "Aborting: %s hook [%d] failed with exit code %d (on-error=%s)", + hook.stage, + hook.order, + exit_code, + ON_ERROR_ABORT, + ) + return False + else: + logger.warning( + "Continuing despite %s hook [%d] failure, exit code %d (on-error=%s)", + hook.stage, + hook.order, + exit_code, + ON_ERROR_CONTINUE, + ) + + return True + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + + +def _execute_hook(hook, containers): + """Resolve target context and execute a single hook. + + Returns the command's exit code (0 = success). + """ + this_container = containers.this_container + + if hook.context: + # Explicit context — look up the named service + target = containers.get_service(hook.context) + + # It may be the backup container itself + if target is None and this_container.service_name == hook.context: + return _run_local(hook.cmd) + + if target is None: + logger.error( + "Hook context container '%s' not found, failing hook: %s", + hook.context, + hook.cmd, + ) + return 1 + + return _run_in_container(target.id, hook.cmd) + else: + # No context — run in the source container + if hook.source_container_id == this_container.id: + return _run_local(hook.cmd) + else: + return _run_in_container(hook.source_container_id, hook.cmd) + + +def _run_local(cmd): + """Run a hook command locally in the backup process container.""" + logger.debug("Running hook locally: %s", cmd) + return commands.run(["sh", "-c", cmd]) + + +def _run_in_container(container_id, cmd): + """Run a hook command inside another container via docker exec.""" + logger.debug("Running hook in container %s: %s", container_id, cmd) + return commands.docker_exec(container_id, ["sh", "-c", cmd]) diff --git a/src/tests/unit/fixtures.py b/src/tests/unit/fixtures.py index b3dd0ce..92a6b75 100644 --- a/src/tests/unit/fixtures.py +++ b/src/tests/unit/fixtures.py @@ -1,60 +1,61 @@ -"""Generate test fixtures""" - -from datetime import datetime -import hashlib -import string -import random - - -def generate_sha256(): - """Generate a unique sha256""" - h = hashlib.sha256() - h.update(str(datetime.now().timestamp()).encode()) - return h.hexdigest() - - -def containers(project="default", containers=[]): - """ - Args: - project (str): Name of the compose project - containers (dict): - { - 'containers: [ - 'id': 'something' - 'service': 'service_name', - 'image': 'image:tag', - 'mounts: [{ - 'Source': '/home/user/stuff', - 'Destination': '/srv/stuff', - 'Type': 'bind' / 'volume' - }], - ] - } - """ - - def wrapper(*args, **kwargs): - return [ - { - "Id": container.get("id", generate_sha256()), - "Name": container.get("service") - + "_" - + "".join(random.choice(string.ascii_lowercase) for i in range(16)), - "Config": { - "Image": container.get("image", "image:latest"), - "Labels": { - "com.docker.compose.oneoff": "False", - "com.docker.compose.project": project, - "com.docker.compose.service": container["service"], - **container.get("labels", {}), - }, - }, - "Mounts": container.get("mounts", []), - "State": { - "Status": "running", - "Running": True, - }, - } - for container in containers - ] - - return wrapper +"""Generate test fixtures""" + +from datetime import datetime +import hashlib +import string +import random + + +def generate_sha256(): + """Generate a unique sha256""" + h = hashlib.sha256() + h.update(str(datetime.now().timestamp()).encode()) + return h.hexdigest() + + +def containers(project="default", containers=[]): + """ + Args: + project (str): Name of the compose project + containers (dict): + { + 'containers: [ + 'id': 'something' + 'service': 'service_name', + 'image': 'image:tag', + 'mounts: [{ + 'Source': '/home/user/stuff', + 'Destination': '/srv/stuff', + 'Type': 'bind' / 'volume' + }], + ] + } + """ + + def wrapper(*args, **kwargs): + return [ + { + "Id": container.get("id", generate_sha256()), + "Name": container.get("service") + + "_" + + "".join(random.choice(string.ascii_lowercase) for i in range(16)), + "Config": { + "Image": container.get("image", "image:latest"), + "Env": container.get("env", []), + "Labels": { + "com.docker.compose.oneoff": "False", + "com.docker.compose.project": project, + "com.docker.compose.service": container["service"], + **container.get("labels", {}), + }, + }, + "Mounts": container.get("mounts", []), + "State": { + "Status": "running", + "Running": True, + }, + } + for container in containers + ] + + return wrapper diff --git a/src/tests/unit/test_backup_lifecycle.py b/src/tests/unit/test_backup_lifecycle.py new file mode 100644 index 0000000..b487e3f --- /dev/null +++ b/src/tests/unit/test_backup_lifecycle.py @@ -0,0 +1,291 @@ +"""Unit tests for backup lifecycle orchestration in start_backup_process(). + +These tests verify the correct ordering of operations during a backup run: + + pre hooks → stop containers → backup → restart containers → + post hooks (on success) / error hooks (on failure) → finally hooks + +The ordering is critical because: +- Pre hooks may exec into containers (e.g., enable maintenance mode) +- Containers may be stopped during backup for filesystem consistency +- Post/error/finally hooks need containers running to exec into them + +A previous bug had post hooks executing before stopped containers were +restarted. These tests use a call_log pattern to assert the exact +sequence of operations and would have caught that bug. +""" + +import os +import unittest +from unittest import mock + +import pytest + +from restic_compose_backup import hooks +from restic_compose_backup.cli import start_backup_process + +pytestmark = pytest.mark.unit + + +class BackupLifecycleTests(unittest.TestCase): + """Tests for operation ordering in start_backup_process(). + + Uses a shared call_log list that records the order of operations. + Each mocked dependency appends its name when called. Tests then + assert the exact sequence (or relative ordering) of entries. + """ + + def setUp(self): + self.call_log = [] + + # Patch all external dependencies of start_backup_process + self.mock_status = mock.patch( + "restic_compose_backup.cli.status" + ).start() + self.mock_collect = mock.patch( + "restic_compose_backup.cli.hooks.collect_hooks" + ).start() + self.mock_execute = mock.patch( + "restic_compose_backup.cli.hooks.execute_hooks" + ).start() + self.mock_stop = mock.patch( + "restic_compose_backup.cli.utils.stop_containers" + ).start() + self.mock_start = mock.patch( + "restic_compose_backup.cli.utils.start_containers" + ).start() + self.mock_backup = mock.patch( + "restic_compose_backup.cli.restic.backup_files" + ).start() + self.mock_stat = mock.patch("os.stat").start() + mock.patch.dict( + os.environ, {"BACKUP_PROCESS_CONTAINER": "true"} + ).start() + + # Defaults: volumes exist, all operations succeed + self.mock_stat.return_value = True + self.mock_collect.side_effect = self._collect_one_hook_per_stage + self.mock_execute.side_effect = self._execute_and_record() + self.mock_backup.side_effect = self._record("backup_volumes", 0) + self.mock_stop.side_effect = self._record("stop_containers") + self.mock_start.side_effect = self._record("start_containers") + + def tearDown(self): + mock.patch.stopall() + + # ----- helpers ----- + + def _collect_one_hook_per_stage(self, stage, backup_container, targets): + """Return a single hook for each stage so every stage appears in the log.""" + return [hooks.Hook(stage=stage, order=1, cmd=f"echo {stage}")] + + def _execute_and_record(self, failing_stage=None): + """Side effect for execute_hooks that records the stage name. + + If *failing_stage* matches, the hook returns False (abort). + """ + def side_effect(hook_list, containers): + if not hook_list: + return True + stage = hook_list[0].stage + self.call_log.append(f"hooks:{stage}") + return stage != failing_stage + return side_effect + + def _record(self, name, return_value=None): + """Create a side effect that appends *name* to the call log.""" + def side_effect(*args, **kwargs): + self.call_log.append(name) + return return_value + return side_effect + + def _create_mock_containers(self, has_stop_during_backup=True, + has_database=False): + """Build a mock RunningContainers for lifecycle testing.""" + containers = mock.MagicMock() + containers.this_container._labels = {} + + backup_target = mock.MagicMock() + backup_target.database_backup_enabled = has_database + if has_database: + backup_target.instance.backup.side_effect = ( + self._record("backup_database", 0) + ) + backup_target.instance.container_type = "mysql" + backup_target.instance.service_name = "db" + backup_target.instance.project_name = "test" + containers.containers_for_backup.return_value = [backup_target] + + if has_stop_during_backup: + containers.stop_during_backup_containers = [mock.MagicMock()] + else: + containers.stop_during_backup_containers = [] + + return containers + + def _create_mock_config(self): + """Build a mock Config that skips maintenance.""" + config = mock.MagicMock() + config.maintenance_schedule = "0 3 * * *" # non-empty skips maintenance + return config + + # ----- lifecycle ordering tests ----- + + def test_successful_backup_lifecycle_order(self): + """Happy path: pre → stop → backup → restart → post → finally.""" + config = self._create_mock_config() + containers = self._create_mock_containers() + + start_backup_process(config, containers) + + self.assertEqual(self.call_log, [ + "hooks:pre", + "stop_containers", + "backup_volumes", + "start_containers", + "hooks:post", + "hooks:finally", + ]) + + def test_post_hooks_run_after_container_restart(self): + """Post hooks must execute after stopped containers are restarted. + + This test would have caught the original bug where post hooks + ran inside the try block before the finally block restarted + containers. + """ + config = self._create_mock_config() + containers = self._create_mock_containers() + + start_backup_process(config, containers) + + restart_position = self.call_log.index("start_containers") + post_position = self.call_log.index("hooks:post") + finally_position = self.call_log.index("hooks:finally") + + self.assertGreater( + post_position, restart_position, + "Post hooks must run after containers are restarted", + ) + self.assertGreater( + finally_position, restart_position, + "Finally hooks must run after containers are restarted", + ) + + def test_error_hooks_run_after_container_restart(self): + """On backup failure, error hooks must still run after restart.""" + config = self._create_mock_config() + containers = self._create_mock_containers() + self.mock_backup.side_effect = self._record("backup_volumes", 1) + + with self.assertRaises(SystemExit): + start_backup_process(config, containers) + + restart_position = self.call_log.index("start_containers") + error_position = self.call_log.index("hooks:error") + + self.assertGreater( + error_position, restart_position, + "Error hooks must run after containers are restarted", + ) + + def test_pre_hook_failure_skips_backup_and_stop(self): + """When a pre hook aborts, no backup runs and containers are not stopped.""" + config = self._create_mock_config() + containers = self._create_mock_containers() + self.mock_execute.side_effect = ( + self._execute_and_record(failing_stage="pre") + ) + + with self.assertRaises(SystemExit): + start_backup_process(config, containers) + + self.assertIn("hooks:pre", self.call_log) + self.assertNotIn("stop_containers", self.call_log) + self.assertNotIn("backup_volumes", self.call_log) + self.assertNotIn("hooks:post", self.call_log) + self.assertIn("hooks:error", self.call_log) + self.assertIn("hooks:finally", self.call_log) + + def test_backup_failure_triggers_error_not_post(self): + """On backup failure: error hooks run, post hooks do not.""" + config = self._create_mock_config() + containers = self._create_mock_containers() + self.mock_backup.side_effect = self._record("backup_volumes", 1) + + with self.assertRaises(SystemExit): + start_backup_process(config, containers) + + self.assertIn("hooks:pre", self.call_log) + self.assertIn("backup_volumes", self.call_log) + self.assertNotIn("hooks:post", self.call_log) + self.assertIn("hooks:error", self.call_log) + self.assertIn("hooks:finally", self.call_log) + + def test_post_hook_failure_triggers_error_hooks(self): + """When post hooks fail, error hooks still run.""" + config = self._create_mock_config() + containers = self._create_mock_containers() + self.mock_execute.side_effect = ( + self._execute_and_record(failing_stage="post") + ) + + with self.assertRaises(SystemExit): + start_backup_process(config, containers) + + self.assertEqual(self.call_log, [ + "hooks:pre", + "stop_containers", + "backup_volumes", + "start_containers", + "hooks:post", + "hooks:error", + "hooks:finally", + ]) + + def test_containers_always_restarted_on_failure(self): + """Stopped containers are restarted even when backup fails.""" + config = self._create_mock_config() + containers = self._create_mock_containers() + self.mock_backup.side_effect = self._record("backup_volumes", 1) + + with self.assertRaises(SystemExit): + start_backup_process(config, containers) + + self.assertIn("stop_containers", self.call_log) + self.assertIn("start_containers", self.call_log) + + def test_no_stop_when_no_stop_during_backup_containers(self): + """Without stop-during-backup containers, stop/start are skipped.""" + config = self._create_mock_config() + containers = self._create_mock_containers(has_stop_during_backup=False) + + start_backup_process(config, containers) + + self.assertEqual(self.call_log, [ + "hooks:pre", + "backup_volumes", + "hooks:post", + "hooks:finally", + ]) + + def test_database_backup_runs_between_volumes_and_restart(self): + """Database dumps run after volume backup but before container restart.""" + config = self._create_mock_config() + containers = self._create_mock_containers(has_database=True) + + start_backup_process(config, containers) + + volume_position = self.call_log.index("backup_volumes") + database_position = self.call_log.index("backup_database") + restart_position = self.call_log.index("start_containers") + + self.assertGreater( + database_position, volume_position, + "Database backup must run after volume backup", + ) + self.assertGreater( + restart_position, database_position, + "Container restart must run after database backup", + ) + diff --git a/src/tests/unit/test_end_to_end_lifecycle.py b/src/tests/unit/test_end_to_end_lifecycle.py new file mode 100644 index 0000000..bd0633c --- /dev/null +++ b/src/tests/unit/test_end_to_end_lifecycle.py @@ -0,0 +1,938 @@ +"""End-to-end backup lifecycle tests with real container and hook objects. + +Unlike test_backup_lifecycle.py (which mocks hooks.execute_hooks entirely) +and test_hooks.py (which tests hooks in isolation), these tests exercise +the FULL start_backup_process() flow with real Container objects and real +hook logic. Only the I/O boundary is mocked: + + commands.run → records local shell commands (hooks + restic) + commands.docker_exec → records docker exec calls (hooks into containers) + utils.stop_containers → records container stop operations + utils.start_containers → records container start operations + restic.backup_from_stdin → records database dump operations + +Everything above that boundary runs for real: + + hooks.parse_hooks_from_labels → label parsing from real Container._labels + hooks.collect_hooks → symmetric ordering (setup / teardown) + hooks.execute_hooks → iteration, abort / continue logic + hooks._execute_hook → context resolution (local vs. docker exec) + hooks._run_local → delegates to commands.run (mocked) + hooks._run_in_container → delegates to commands.docker_exec (mocked) + restic.backup_files → delegates to commands.run (mocked) + RunningContainers → container discovery from fixture data + containers_for_backup → ordered backup logic + +This catches bugs that neither test file alone would find: + +- Hook context resolution errors (which container gets exec'd into) +- Symmetric hook ordering (pre: global→targets, post: targets→global) +- Post/error hooks running after container restart (not before) +- Label parsing through to actual command execution +- Interaction between ordered backup and hook ordering +""" + +import os +import unittest +from unittest import mock + +import pytest + +from restic_compose_backup.containers import RunningContainers +from restic_compose_backup.cli import start_backup_process +from . import fixtures +from .conftest import BaseTestCase + +pytestmark = pytest.mark.unit + +list_containers_func = "restic_compose_backup.utils.list_containers" + +# Predictable container IDs for target containers. +# These must NOT start with the mocked hostname (backup_hash[:8]). +WEB_ID = "w" * 64 +DB_ID = "d" * 64 +ALPHA_ID = "a" * 64 +BETA_ID = "b" * 64 + + +class EndToEndBackupTests(BaseTestCase): + """End-to-end lifecycle tests using real Container/Hook objects. + + The call_log records every I/O operation in a readable string format:: + + "local: echo global-pre" hook running locally (commands.run) + "exec@web: echo web-pre" hook docker-exec'd into "web" + "stop_containers" containers stopped + "start_containers" containers restarted + "restic_backup_files" volume backup (restic via commands.run) + "backup_from_stdin@db" database dump (restic.backup_from_stdin) + + Tests assert the exact call_log sequence (or relative ordering) to + verify the full orchestration pipeline. + """ + + def setUp(self): + self.call_log = [] + self.restic_exit_code = 0 + self.failing_local_hooks = {} # {cmd_string: exit_code} + self.id_to_service = {} + + # Track patches explicitly — mock.patch.stopall() would also kill + # the hostname patcher from BaseTestCase.setUpClass. + self._patches = [ + mock.patch( + "restic_compose_backup.commands.run", + side_effect=self._mock_commands_run, + ), + mock.patch( + "restic_compose_backup.commands.docker_exec", + side_effect=self._mock_docker_exec, + ), + mock.patch( + "restic_compose_backup.cli.utils.stop_containers", + side_effect=self._mock_stop, + ), + mock.patch( + "restic_compose_backup.cli.utils.start_containers", + side_effect=self._mock_start, + ), + mock.patch( + "restic_compose_backup.restic.backup_from_stdin", + side_effect=self._mock_backup_from_stdin, + ), + mock.patch("restic_compose_backup.cli.status"), + mock.patch("os.stat", return_value=True), + mock.patch.dict( + os.environ, {"BACKUP_PROCESS_CONTAINER": "true"} + ), + ] + for patch in self._patches: + patch.start() + + def tearDown(self): + for patch in reversed(self._patches): + patch.stop() + + # ------------------------------------------------------------------ # + # I/O mock implementations # + # ------------------------------------------------------------------ # + + def _mock_commands_run(self, cmd): + """Distinguish restic commands from local hook commands. + + Both go through commands.run but produce different call_log entries: + - restic: ``["restic", "-r", "test", "--verbose", "backup", ...]`` + - hook: ``["sh", "-c", "echo something"]`` + """ + if cmd and cmd[0] == "restic": + self.call_log.append("restic_backup_files") + return self.restic_exit_code + hook_cmd = cmd[2] if len(cmd) >= 3 else " ".join(cmd) + self.call_log.append(f"local: {hook_cmd}") + return self.failing_local_hooks.get(hook_cmd, 0) + + def _mock_docker_exec(self, container_id, cmd, **kwargs): + """Record which container was exec'd into, with the command.""" + service = self.id_to_service.get( + container_id, f"unknown({container_id[:12]})" + ) + hook_cmd = cmd[2] if len(cmd) >= 3 else " ".join(cmd) + self.call_log.append(f"exec@{service}: {hook_cmd}") + return 0 + + def _mock_stop(self, containers): + self.call_log.append("stop_containers") + + def _mock_start(self, containers): + self.call_log.append("start_containers") + + def _mock_backup_from_stdin(self, repository, filename, container_id, + source_command, environment=None): + service = self.id_to_service.get( + container_id, f"unknown({container_id[:12]})" + ) + self.call_log.append(f"backup_from_stdin@{service}") + return 0 + + # ------------------------------------------------------------------ # + # Helpers # + # ------------------------------------------------------------------ # + + def _build_running_containers(self, container_defs): + """Create real RunningContainers from fixture definitions. + + Also populates ``self.id_to_service`` so call_log entries + show readable service names instead of raw container IDs. + """ + with mock.patch( + list_containers_func, + fixtures.containers(containers=container_defs), + ): + running = RunningContainers() + + self.id_to_service = { + running.this_container.id: running.this_container.service_name, + } + for container in running.containers: + self.id_to_service[container.id] = container.service_name + + return running + + def _make_config(self): + """Create a mock Config that skips maintenance.""" + cfg = mock.MagicMock() + cfg.repository = "test" + cfg.maintenance_schedule = "0 3 * * *" + return cfg + + # ------------------------------------------------------------------ # + # Scenario 1: Full happy path # + # ------------------------------------------------------------------ # + + def test_full_happy_path_with_hooks_on_backup_and_target(self): + """Complete successful backup with hooks on both backup and target. + + Verifies the exact sequence:: + + pre(backup) → pre(web) → stop → restic backup → start → + post(web) → post(backup) → finally(backup) + + This tests symmetric ordering, context resolution, and restart + happening before post/finally hooks. + """ + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "echo global-pre", + "stack-back.hooks.post.1.cmd": "echo global-post", + "stack-back.hooks.finally.1.cmd": "echo global-finally", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": { + "stack-back.volumes": "true", + "stack-back.volumes.stop-during-backup": "true", + "stack-back.hooks.pre.1.cmd": "echo web-pre", + "stack-back.hooks.post.1.cmd": "echo web-post", + }, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + self.assertEqual(self.call_log, [ + "local: echo global-pre", # pre: backup first (setup) + "exec@web: echo web-pre", # pre: target second + "stop_containers", # stop web + "restic_backup_files", # backup volumes + "start_containers", # restart web + "exec@web: echo web-post", # post: target first (teardown) + "local: echo global-post", # post: backup second + "local: echo global-finally", # finally: backup + ]) + + # ------------------------------------------------------------------ # + # Scenario 2: Post hooks must run after container restart # + # ------------------------------------------------------------------ # + + def test_post_hooks_always_run_after_container_restart(self): + """Post and finally hooks must execute AFTER stopped containers restart. + + This is the bug found during review: post hooks were originally + in the try block, running before the finally block restarted + containers. + """ + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.post.1.cmd": "echo global-post", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": { + "stack-back.volumes": "true", + "stack-back.volumes.stop-during-backup": "true", + "stack-back.hooks.post.1.cmd": "echo web-post", + }, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + restart_idx = self.call_log.index("start_containers") + post_web_idx = self.call_log.index("exec@web: echo web-post") + post_global_idx = self.call_log.index("local: echo global-post") + + self.assertGreater( + post_web_idx, restart_idx, + "Web post hook must run after containers are restarted", + ) + self.assertGreater( + post_global_idx, restart_idx, + "Global post hook must run after containers are restarted", + ) + + # ------------------------------------------------------------------ # + # Scenario 3: Context resolution — all cases # + # ------------------------------------------------------------------ # + + def test_backup_hook_without_context_runs_locally(self): + """A backup container hook without explicit context runs locally. + + Current behavior: backup-container hooks with no context run via + commands.run() (local subprocess) rather than docker exec. This + is functionally equivalent inside the backup process container + but uses a different code path than target-container hooks. + """ + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "echo local-test", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": {"stack-back.volumes": "true"}, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + self.assertIn("local: echo local-test", self.call_log) + self.assertNotIn("exec@backup: echo local-test", self.call_log) + + def test_target_hook_without_context_execs_into_own_container(self): + """A target container hook without explicit context execs into itself. + + The hook is defined on 'web', so it must docker-exec into the web + container — not run locally in the backup process container. + """ + container_defs = self.createContainers() + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": { + "stack-back.volumes": "true", + "stack-back.hooks.pre.1.cmd": "echo self-exec", + }, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + self.assertIn("exec@web: echo self-exec", self.call_log) + self.assertNotIn("local: echo self-exec", self.call_log) + + def test_context_override_backup_hook_execs_into_target(self): + """A backup container hook with explicit context execs into the named service.""" + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "echo ctx-override", + "stack-back.hooks.pre.1.context": "web", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": {"stack-back.volumes": "true"}, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + self.assertIn("exec@web: echo ctx-override", self.call_log) + self.assertNotIn("local: echo ctx-override", self.call_log) + + def test_cross_container_context_web_hook_execs_into_db(self): + """A hook defined on 'web' with context='db' execs into the db container.""" + container_defs = self.createContainers() + container_defs += [ + { + "id": WEB_ID, + "service": "web", + "labels": { + "stack-back.volumes": "true", + "stack-back.hooks.pre.1.cmd": "echo cross-ctx", + "stack-back.hooks.pre.1.context": "db", + }, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }, + { + "id": DB_ID, + "service": "db", + "image": "mysql:8", + "labels": {"stack-back.mysql": "true"}, + "env": ["MYSQL_ROOT_PASSWORD=secret"], + "mounts": [ + {"Source": "mysql_data", "Destination": "/var/lib/mysql", "Type": "volume"}, + ], + }, + ] + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + self.assertIn("exec@db: echo cross-ctx", self.call_log) + self.assertNotIn("exec@web: echo cross-ctx", self.call_log) + self.assertNotIn("local: echo cross-ctx", self.call_log) + + def test_unknown_hook_context_triggers_failure(self): + """A hook referencing a non-existent service triggers the error path.""" + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "echo bad-ctx", + "stack-back.hooks.pre.1.context": "nonexistent", + "stack-back.hooks.error.1.cmd": "echo global-error", + "stack-back.hooks.finally.1.cmd": "echo global-finally", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": {"stack-back.volumes": "true"}, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + + with self.assertRaises(SystemExit): + start_backup_process(self._make_config(), containers) + + # Bad context fails the hook → no backup → error + finally + self.assertNotIn("restic_backup_files", self.call_log) + self.assertIn("local: echo global-error", self.call_log) + self.assertIn("local: echo global-finally", self.call_log) + + # ------------------------------------------------------------------ # + # Scenario 4: Backup failure — error hooks, not post hooks # + # ------------------------------------------------------------------ # + + def test_backup_failure_fires_error_hooks_not_post(self): + """When volume backup fails, error hooks fire; post hooks do not. + + Also verifies symmetric teardown ordering: target error hooks + run before backup-container error hooks. + """ + self.restic_exit_code = 1 + + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.post.1.cmd": "echo global-post", + "stack-back.hooks.error.1.cmd": "echo global-error", + "stack-back.hooks.finally.1.cmd": "echo global-finally", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": { + "stack-back.volumes": "true", + "stack-back.volumes.stop-during-backup": "true", + "stack-back.hooks.error.1.cmd": "echo web-error", + }, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + + with self.assertRaises(SystemExit): + start_backup_process(self._make_config(), containers) + + self.assertEqual(self.call_log, [ + "stop_containers", + "restic_backup_files", # fails (exit code 1) + "start_containers", # restart before hooks + "exec@web: echo web-error", # error: target first (teardown) + "local: echo global-error", # error: backup second + "local: echo global-finally", # finally always + ]) + self.assertNotIn("local: echo global-post", self.call_log) + + def test_error_hooks_run_after_container_restart(self): + """On failure, error hooks must run after containers are restarted.""" + self.restic_exit_code = 1 + + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.error.1.cmd": "echo global-error", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": { + "stack-back.volumes": "true", + "stack-back.volumes.stop-during-backup": "true", + "stack-back.hooks.error.1.cmd": "echo web-error", + }, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + + with self.assertRaises(SystemExit): + start_backup_process(self._make_config(), containers) + + restart_idx = self.call_log.index("start_containers") + web_error_idx = self.call_log.index("exec@web: echo web-error") + global_error_idx = self.call_log.index("local: echo global-error") + + self.assertGreater( + web_error_idx, restart_idx, + "Web error hook must run after restart", + ) + self.assertGreater( + global_error_idx, restart_idx, + "Global error hook must run after restart", + ) + + # ------------------------------------------------------------------ # + # Scenario 5: Pre hook failure — skips everything # + # ------------------------------------------------------------------ # + + def test_pre_hook_failure_skips_backup_and_stop(self): + """When a pre hook aborts, no stop/backup happens; error+finally fire.""" + self.failing_local_hooks["fail-pre"] = 1 + + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "fail-pre", + "stack-back.hooks.error.1.cmd": "echo global-error", + "stack-back.hooks.finally.1.cmd": "echo global-finally", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": { + "stack-back.volumes": "true", + "stack-back.volumes.stop-during-backup": "true", + "stack-back.hooks.pre.1.cmd": "echo web-pre", + }, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + + with self.assertRaises(SystemExit): + start_backup_process(self._make_config(), containers) + + self.assertEqual(self.call_log, [ + "local: fail-pre", # pre hook fails → abort + # No web pre hook (aborted after first failure) + # No stop_containers + # No restic_backup_files + "local: echo global-error", # error fires + "local: echo global-finally", # finally fires + ]) + + # ------------------------------------------------------------------ # + # Scenario 6: Database backup in lifecycle # + # ------------------------------------------------------------------ # + + def test_database_backup_in_lifecycle(self): + """Database dump runs after volume backup, before restart. + + Uses a real MysqlContainer instance (via Container.instance) with + mocked restic.backup_from_stdin at the I/O boundary. + """ + container_defs = self.createContainers() + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": { + "stack-back.volumes": "true", + "stack-back.volumes.stop-during-backup": "true", + }, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + container_defs.append({ + "id": DB_ID, + "service": "db", + "image": "mysql:8", + "labels": {"stack-back.mysql": "true"}, + "env": ["MYSQL_ROOT_PASSWORD=secret"], + "mounts": [ + {"Source": "mysql_data", "Destination": "/var/lib/mysql", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + volume_idx = self.call_log.index("restic_backup_files") + db_idx = self.call_log.index("backup_from_stdin@db") + restart_idx = self.call_log.index("start_containers") + + self.assertGreater( + db_idx, volume_idx, + "DB dump must run after volume backup", + ) + self.assertGreater( + restart_idx, db_idx, + "Restart must run after DB dump", + ) + + # ------------------------------------------------------------------ # + # Scenario 7: Symmetric ordering with multiple targets # + # ------------------------------------------------------------------ # + + def test_symmetric_ordering_multiple_targets(self): + """Pre hooks go backup→alpha→beta; post hooks go alpha→beta→backup. + + Verifies that the backup container is correctly placed first for + setup (pre) and last for teardown (post). Target containers + maintain their discovery order in both stages. + """ + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "echo global-pre", + "stack-back.hooks.post.1.cmd": "echo global-post", + } + container_defs += [ + { + "id": ALPHA_ID, + "service": "alpha", + "labels": { + "stack-back.volumes": "true", + "stack-back.hooks.pre.1.cmd": "echo alpha-pre", + "stack-back.hooks.post.1.cmd": "echo alpha-post", + }, + "mounts": [ + {"Source": "a_data", "Destination": "/a", "Type": "volume"}, + ], + }, + { + "id": BETA_ID, + "service": "beta", + "labels": { + "stack-back.volumes": "true", + "stack-back.hooks.pre.1.cmd": "echo beta-pre", + "stack-back.hooks.post.1.cmd": "echo beta-post", + }, + "mounts": [ + {"Source": "b_data", "Destination": "/b", "Type": "volume"}, + ], + }, + ] + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + # Pre hooks: backup → alpha → beta (setup order) + pre_global = self.call_log.index("local: echo global-pre") + pre_alpha = self.call_log.index("exec@alpha: echo alpha-pre") + pre_beta = self.call_log.index("exec@beta: echo beta-pre") + self.assertLess(pre_global, pre_alpha) + self.assertLess(pre_alpha, pre_beta) + + # Post hooks: alpha → beta → backup (teardown order) + post_alpha = self.call_log.index("exec@alpha: echo alpha-post") + post_beta = self.call_log.index("exec@beta: echo beta-post") + post_global = self.call_log.index("local: echo global-post") + self.assertLess(post_alpha, post_global) + self.assertLess(post_beta, post_global) + + # ------------------------------------------------------------------ # + # Scenario 8: Ordered backup affects hook ordering # + # ------------------------------------------------------------------ # + + def test_ordered_backup_affects_hook_order(self): + """When ordered backup is configured, hooks follow container order. + + Order configured as beta→alpha means beta hooks run before alpha. + """ + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.ordered": "true", + "stack-back.order.1": "beta", + "stack-back.order.2": "alpha", + } + container_defs += [ + { + "id": ALPHA_ID, + "service": "alpha", + "labels": { + "stack-back.volumes": "true", + "stack-back.hooks.pre.1.cmd": "echo alpha-pre", + "stack-back.hooks.post.1.cmd": "echo alpha-post", + }, + "mounts": [ + {"Source": "a_data", "Destination": "/a", "Type": "volume"}, + ], + }, + { + "id": BETA_ID, + "service": "beta", + "labels": { + "stack-back.volumes": "true", + "stack-back.hooks.pre.1.cmd": "echo beta-pre", + "stack-back.hooks.post.1.cmd": "echo beta-post", + }, + "mounts": [ + {"Source": "b_data", "Destination": "/b", "Type": "volume"}, + ], + }, + ] + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + # Pre hooks follow configured order: beta before alpha + pre_beta = self.call_log.index("exec@beta: echo beta-pre") + pre_alpha = self.call_log.index("exec@alpha: echo alpha-pre") + self.assertLess( + pre_beta, pre_alpha, + "Ordered backup: beta pre should run before alpha pre", + ) + + # Post hooks follow configured order within targets: beta before alpha + post_beta = self.call_log.index("exec@beta: echo beta-post") + post_alpha = self.call_log.index("exec@alpha: echo alpha-post") + self.assertLess( + post_beta, post_alpha, + "Ordered backup post: targets maintain configured order", + ) + + # ------------------------------------------------------------------ # + # Scenario 9: Minimal scenario — no hooks, no stops # + # ------------------------------------------------------------------ # + + def test_no_hooks_no_stop_clean_backup(self): + """Minimal scenario: volume backup with no hooks and no stopped containers.""" + container_defs = self.createContainers() + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": {"stack-back.volumes": "true"}, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + self.assertEqual(self.call_log, ["restic_backup_files"]) + + # ------------------------------------------------------------------ # + # Scenario 10: on-error=continue allows backup to proceed # + # ------------------------------------------------------------------ # + + def test_continue_on_error_does_not_abort_pipeline(self): + """A pre hook with on-error=continue allows the backup to proceed.""" + self.failing_local_hooks["soft-fail"] = 1 + + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "soft-fail", + "stack-back.hooks.pre.1.on-error": "continue", + "stack-back.hooks.pre.2.cmd": "echo second-pre", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": {"stack-back.volumes": "true"}, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + # Both pre hooks ran despite first failure + self.assertIn("local: soft-fail", self.call_log) + self.assertIn("local: echo second-pre", self.call_log) + # Backup proceeded + self.assertIn("restic_backup_files", self.call_log) + + # ------------------------------------------------------------------ # + # Scenario 11: Post hook failure triggers error hooks # + # ------------------------------------------------------------------ # + + def test_post_hook_failure_triggers_error_hooks(self): + """When a post hook fails, error hooks still run.""" + self.failing_local_hooks["echo fail-post"] = 1 + + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.post.1.cmd": "echo fail-post", + "stack-back.hooks.error.1.cmd": "echo global-error", + "stack-back.hooks.finally.1.cmd": "echo global-finally", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": {"stack-back.volumes": "true"}, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + + with self.assertRaises(SystemExit): + start_backup_process(self._make_config(), containers) + + self.assertEqual(self.call_log, [ + "restic_backup_files", + "local: echo fail-post", # post hook fails + "local: echo global-error", # error fires + "local: echo global-finally", # finally fires + ]) + + # ------------------------------------------------------------------ # + # Scenario 12: Hooks on all four stages # + # ------------------------------------------------------------------ # + + def test_all_four_stages_with_failure(self): + """All hook stages fire in correct order during a backup failure. + + pre → (backup fails) → error → finally + No post hooks because backup failed. + """ + self.restic_exit_code = 1 + + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "echo pre", + "stack-back.hooks.post.1.cmd": "echo post", + "stack-back.hooks.error.1.cmd": "echo error", + "stack-back.hooks.finally.1.cmd": "echo finally", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": {"stack-back.volumes": "true"}, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + + with self.assertRaises(SystemExit): + start_backup_process(self._make_config(), containers) + + self.assertEqual(self.call_log, [ + "local: echo pre", + "restic_backup_files", # fails + "local: echo error", # error fires + "local: echo finally", # finally fires + ]) + self.assertNotIn("local: echo post", self.call_log) + + # ------------------------------------------------------------------ # + # Scenario 13: Multiple hooks per stage in correct order # + # ------------------------------------------------------------------ # + + def test_multiple_hooks_per_stage_execute_in_order(self): + """Multiple hooks on one container execute in numerical order.""" + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.pre.3.cmd": "echo third", + "stack-back.hooks.pre.1.cmd": "echo first", + "stack-back.hooks.pre.2.cmd": "echo second", + } + container_defs.append({ + "id": WEB_ID, + "service": "web", + "labels": {"stack-back.volumes": "true"}, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }) + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + first_idx = self.call_log.index("local: echo first") + second_idx = self.call_log.index("local: echo second") + third_idx = self.call_log.index("local: echo third") + + self.assertLess(first_idx, second_idx) + self.assertLess(second_idx, third_idx) + # All before backup + backup_idx = self.call_log.index("restic_backup_files") + self.assertLess(third_idx, backup_idx) + + # ------------------------------------------------------------------ # + # Scenario 14: Database with hooks — full lifecycle # + # ------------------------------------------------------------------ # + + def test_database_with_hooks_full_lifecycle(self): + """Full lifecycle with a web container (stopped) and DB container (hooks). + + Verifies that DB hooks fire via docker exec into the DB container + and that DB dump happens after volume backup. + """ + container_defs = self.createContainers() + container_defs[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "echo global-pre", + "stack-back.hooks.finally.1.cmd": "echo global-finally", + } + container_defs += [ + { + "id": WEB_ID, + "service": "web", + "labels": { + "stack-back.volumes": "true", + "stack-back.volumes.stop-during-backup": "true", + }, + "mounts": [ + {"Source": "web_data", "Destination": "/data", "Type": "volume"}, + ], + }, + { + "id": DB_ID, + "service": "db", + "image": "mysql:8", + "labels": { + "stack-back.mysql": "true", + "stack-back.hooks.pre.1.cmd": "echo db-pre", + "stack-back.hooks.post.1.cmd": "echo db-post", + }, + "env": ["MYSQL_ROOT_PASSWORD=secret"], + "mounts": [ + {"Source": "mysql_data", "Destination": "/var/lib/mysql", "Type": "volume"}, + ], + }, + ] + + containers = self._build_running_containers(container_defs) + start_backup_process(self._make_config(), containers) + + self.assertEqual(self.call_log, [ + "local: echo global-pre", # pre: backup first + "exec@db: echo db-pre", # pre: db target + "stop_containers", # stop web (db NOT stopped) + "restic_backup_files", # backup volumes + "backup_from_stdin@db", # dump database + "start_containers", # restart web + "exec@db: echo db-post", # post: target first (teardown) + "local: echo global-finally", # finally: backup + ]) + + diff --git a/src/tests/unit/test_hooks.py b/src/tests/unit/test_hooks.py new file mode 100644 index 0000000..dc3dd85 --- /dev/null +++ b/src/tests/unit/test_hooks.py @@ -0,0 +1,559 @@ +"""Unit tests for backup hooks and ordered backup execution""" + +import unittest +from unittest import mock + +import pytest + +from restic_compose_backup import hooks +from restic_compose_backup.containers import RunningContainers +from . import fixtures +from .conftest import BaseTestCase + +pytestmark = pytest.mark.unit + +list_containers_func = "restic_compose_backup.utils.list_containers" + + +class HookParsingTests(BaseTestCase): + """Tests for parse_hooks_from_labels()""" + + def test_parse_basic_hooks(self): + labels = { + "stack-back.hooks.pre.1.cmd": "echo before", + "stack-back.hooks.pre.1.context": "web", + "stack-back.hooks.pre.2.cmd": "echo also before", + } + result = hooks.parse_hooks_from_labels(labels, "pre") + self.assertEqual(len(result), 2) + self.assertEqual(result[0].cmd, "echo before") + self.assertEqual(result[0].context, "web") + self.assertEqual(result[0].order, 1) + self.assertEqual(result[0].on_error, "abort") + self.assertEqual(result[1].cmd, "echo also before") + self.assertIsNone(result[1].context) + self.assertEqual(result[1].order, 2) + + def test_parse_on_error_continue(self): + labels = { + "stack-back.hooks.pre.1.cmd": "echo test", + "stack-back.hooks.pre.1.on-error": "continue", + } + result = hooks.parse_hooks_from_labels(labels, "pre") + self.assertEqual(len(result), 1) + self.assertEqual(result[0].on_error, "continue") + + def test_parse_invalid_on_error_defaults_to_abort(self): + labels = { + "stack-back.hooks.pre.1.cmd": "echo test", + "stack-back.hooks.pre.1.on-error": "invalid", + } + result = hooks.parse_hooks_from_labels(labels, "pre") + self.assertEqual(result[0].on_error, "abort") + + def test_parse_non_contiguous_order(self): + labels = { + "stack-back.hooks.pre.1.cmd": "echo first", + "stack-back.hooks.pre.5.cmd": "echo third", + "stack-back.hooks.pre.3.cmd": "echo second", + } + result = hooks.parse_hooks_from_labels(labels, "pre") + self.assertEqual(len(result), 3) + self.assertEqual( + [hook.order for hook in result], [1, 3, 5] + ) + + def test_parse_missing_cmd_skipped(self): + labels = { + "stack-back.hooks.pre.1.context": "web", + "stack-back.hooks.pre.2.cmd": "echo valid", + } + result = hooks.parse_hooks_from_labels(labels, "pre") + self.assertEqual(len(result), 1) + self.assertEqual(result[0].order, 2) + + def test_parse_stages_isolated(self): + labels = { + "stack-back.hooks.pre.1.cmd": "echo pre", + "stack-back.hooks.post.1.cmd": "echo post", + "stack-back.hooks.error.1.cmd": "echo error", + "stack-back.hooks.finally.1.cmd": "echo finally", + } + for stage in hooks.HOOK_STAGES: + result = hooks.parse_hooks_from_labels(labels, stage) + self.assertEqual(len(result), 1) + self.assertEqual(result[0].cmd, f"echo {stage}") + + def test_parse_empty_labels(self): + result = hooks.parse_hooks_from_labels({"other": "value"}, "pre") + self.assertEqual(len(result), 0) + + def test_parse_invalid_order_number(self): + labels = { + "stack-back.hooks.pre.abc.cmd": "echo bad", + "stack-back.hooks.pre.1.cmd": "echo good", + } + result = hooks.parse_hooks_from_labels(labels, "pre") + self.assertEqual(len(result), 1) + self.assertEqual(result[0].order, 1) + + +class HookCollectionTests(BaseTestCase): + """Tests for collect_hooks()""" + + def test_backup_hooks_before_target_hooks(self): + containers = self.createContainers() + containers[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "echo backup-pre", + "stack-back.hooks.post.1.cmd": "echo backup-post", + } + containers += [ + { + "service": "web", + "labels": { + "stack-back.volumes": True, + "stack-back.hooks.pre.1.cmd": "echo web-pre", + "stack-back.hooks.post.1.cmd": "echo web-post", + }, + "mounts": [ + {"Source": "data", "Destination": "/data", "Type": "volume"}, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + + backup_list = running.containers_for_backup() + + # pre: backup container hooks first (setup) + pre = hooks.collect_hooks("pre", running.this_container, backup_list) + self.assertEqual(len(pre), 2) + self.assertEqual(pre[0].cmd, "echo backup-pre") + self.assertEqual(pre[0].source_service_name, "backup") + self.assertEqual(pre[1].cmd, "echo web-pre") + self.assertEqual(pre[1].source_service_name, "web") + + # post: target container hooks first (teardown) + post = hooks.collect_hooks("post", running.this_container, backup_list) + self.assertEqual(len(post), 2) + self.assertEqual(post[0].cmd, "echo web-post") + self.assertEqual(post[0].source_service_name, "web") + self.assertEqual(post[1].cmd, "echo backup-post") + self.assertEqual(post[1].source_service_name, "backup") + + def test_no_hooks_returns_empty(self): + containers = self.createContainers() + containers += [ + { + "service": "web", + "labels": {"stack-back.volumes": True}, + "mounts": [ + {"Source": "d", "Destination": "/d", "Type": "volume"}, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + + self.assertEqual( + len( + hooks.collect_hooks( + "pre", running.this_container, running.containers_for_backup() + ) + ), + 0, + ) + + def test_multiple_targets_in_order(self): + containers = self.createContainers() + containers += [ + { + "service": "alpha", + "labels": { + "stack-back.volumes": True, + "stack-back.hooks.pre.1.cmd": "echo alpha", + }, + "mounts": [ + {"Source": "a", "Destination": "/a", "Type": "volume"}, + ], + }, + { + "service": "beta", + "labels": { + "stack-back.volumes": True, + "stack-back.hooks.pre.1.cmd": "echo beta", + }, + "mounts": [ + {"Source": "b", "Destination": "/b", "Type": "volume"}, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + + pre = hooks.collect_hooks( + "pre", running.this_container, running.containers_for_backup() + ) + self.assertEqual(pre[0].source_service_name, "alpha") + self.assertEqual(pre[1].source_service_name, "beta") + + +class HookExecutionTests(BaseTestCase): + """Tests for execute_hooks() and context resolution""" + + @mock.patch("restic_compose_backup.hooks._run_local") + @mock.patch("restic_compose_backup.hooks._run_in_container") + def test_empty_hooks_succeeds(self, mock_remote, mock_local): + containers = self.createContainers() + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + self.assertTrue(hooks.execute_hooks([], running)) + mock_local.assert_not_called() + mock_remote.assert_not_called() + + @mock.patch("restic_compose_backup.hooks._run_local", return_value=0) + def test_backup_hook_runs_locally(self, mock_local): + containers = self.createContainers() + containers[0]["labels"] = {"stack-back.hooks.pre.1.cmd": "echo test"} + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + pre = hooks.collect_hooks( + "pre", running.this_container, running.containers_for_backup() + ) + self.assertTrue(hooks.execute_hooks(pre, running)) + mock_local.assert_called_once_with("echo test") + + @mock.patch("restic_compose_backup.hooks._run_in_container", return_value=0) + def test_target_hook_uses_docker_exec(self, mock_remote): + containers = self.createContainers() + containers += [ + { + "service": "web", + "labels": { + "stack-back.volumes": True, + "stack-back.hooks.pre.1.cmd": "echo web", + }, + "mounts": [ + {"Source": "d", "Destination": "/d", "Type": "volume"}, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + web = running.get_service("web") + pre = hooks.collect_hooks( + "pre", running.this_container, running.containers_for_backup() + ) + self.assertTrue(hooks.execute_hooks(pre, running)) + mock_remote.assert_called_once_with(web.id, "echo web") + + @mock.patch("restic_compose_backup.hooks._run_in_container", return_value=0) + def test_explicit_context_overrides_source(self, mock_remote): + containers = self.createContainers() + containers[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "echo ctx", + "stack-back.hooks.pre.1.context": "web", + } + containers += [ + { + "service": "web", + "labels": {"stack-back.volumes": True}, + "mounts": [ + {"Source": "d", "Destination": "/d", "Type": "volume"}, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + web = running.get_service("web") + pre = hooks.collect_hooks( + "pre", running.this_container, running.containers_for_backup() + ) + self.assertTrue(hooks.execute_hooks(pre, running)) + mock_remote.assert_called_once_with(web.id, "echo ctx") + + @mock.patch("restic_compose_backup.hooks._run_local", return_value=1) + def test_abort_on_failure(self, mock_local): + containers = self.createContainers() + containers[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "fail", + "stack-back.hooks.pre.2.cmd": "echo skip", + } + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + pre = hooks.collect_hooks( + "pre", running.this_container, running.containers_for_backup() + ) + self.assertFalse(hooks.execute_hooks(pre, running)) + mock_local.assert_called_once_with("fail") + + @mock.patch("restic_compose_backup.hooks._run_local", side_effect=[1, 0]) + def test_continue_on_failure(self, mock_local): + containers = self.createContainers() + containers[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "fail", + "stack-back.hooks.pre.1.on-error": "continue", + "stack-back.hooks.pre.2.cmd": "echo ok", + } + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + pre = hooks.collect_hooks( + "pre", running.this_container, running.containers_for_backup() + ) + self.assertTrue(hooks.execute_hooks(pre, running)) + self.assertEqual(mock_local.call_count, 2) + + @mock.patch("restic_compose_backup.hooks._run_in_container") + def test_unknown_context_fails(self, mock_remote): + containers = self.createContainers() + containers[0]["labels"] = { + "stack-back.hooks.pre.1.cmd": "echo x", + "stack-back.hooks.pre.1.context": "nonexistent", + } + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + pre = hooks.collect_hooks( + "pre", running.this_container, running.containers_for_backup() + ) + self.assertFalse(hooks.execute_hooks(pre, running)) + mock_remote.assert_not_called() + + @mock.patch( + "restic_compose_backup.hooks._run_local", side_effect=Exception("boom") + ) + def test_exception_treated_as_failure(self, mock_local): + containers = self.createContainers() + containers[0]["labels"] = {"stack-back.hooks.pre.1.cmd": "explode"} + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + pre = hooks.collect_hooks( + "pre", running.this_container, running.containers_for_backup() + ) + self.assertFalse(hooks.execute_hooks(pre, running)) + + +class BackupOrderingTests(BaseTestCase): + """Tests for ordered backup execution""" + + def test_default_unordered(self): + containers = self.createContainers() + containers += [ + { + "service": "web", + "labels": {"stack-back.volumes": True}, + "mounts": [ + {"Source": "d", "Destination": "/d", "Type": "volume"}, + ], + }, + { + "service": "db", + "labels": {"stack-back.mysql": True}, + "mounts": [ + { + "Source": "m", + "Destination": "/var/lib/mysql", + "Type": "volume", + }, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + self.assertEqual(len(running.containers_for_backup()), 2) + + def test_ordered_containers(self): + containers = self.createContainers() + containers[0]["labels"] = { + "stack-back.ordered": "true", + "stack-back.order.1": "db", + "stack-back.order.2": "web", + } + containers += [ + { + "service": "web", + "labels": {"stack-back.volumes": True}, + "mounts": [ + {"Source": "d", "Destination": "/d", "Type": "volume"}, + ], + }, + { + "service": "db", + "labels": {"stack-back.mysql": True}, + "mounts": [ + { + "Source": "m", + "Destination": "/var/lib/mysql", + "Type": "volume", + }, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + backup = running.containers_for_backup() + self.assertEqual(backup[0].service_name, "db") + self.assertEqual(backup[1].service_name, "web") + + def test_ordered_with_gaps(self): + containers = self.createContainers() + containers[0]["labels"] = { + "stack-back.ordered": "true", + "stack-back.order.1": "db", + "stack-back.order.5": "web", + } + containers += [ + { + "service": "web", + "labels": {"stack-back.volumes": True}, + "mounts": [ + {"Source": "d", "Destination": "/d", "Type": "volume"}, + ], + }, + { + "service": "db", + "labels": {"stack-back.mysql": True}, + "mounts": [ + { + "Source": "m", + "Destination": "/var/lib/mysql", + "Type": "volume", + }, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + backup = running.containers_for_backup() + self.assertEqual(backup[0].service_name, "db") + self.assertEqual(backup[1].service_name, "web") + + def test_ordered_unknown_service_skipped(self): + containers = self.createContainers() + containers[0]["labels"] = { + "stack-back.ordered": "true", + "stack-back.order.1": "nonexistent", + "stack-back.order.2": "web", + } + containers += [ + { + "service": "web", + "labels": {"stack-back.volumes": True}, + "mounts": [ + {"Source": "d", "Destination": "/d", "Type": "volume"}, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + backup = running.containers_for_backup() + self.assertEqual(len(backup), 1) + self.assertEqual(backup[0].service_name, "web") + + def test_unordered_container_appended(self): + containers = self.createContainers() + containers[0]["labels"] = { + "stack-back.ordered": "true", + "stack-back.order.1": "db", + } + containers += [ + { + "service": "web", + "labels": {"stack-back.volumes": True}, + "mounts": [ + {"Source": "d", "Destination": "/d", "Type": "volume"}, + ], + }, + { + "service": "db", + "labels": {"stack-back.mysql": True}, + "mounts": [ + { + "Source": "m", + "Destination": "/var/lib/mysql", + "Type": "volume", + }, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + backup = running.containers_for_backup() + self.assertEqual(len(backup), 2) + self.assertEqual(backup[0].service_name, "db") + self.assertEqual(backup[1].service_name, "web") + + def test_ordered_duplicate_service(self): + containers = self.createContainers() + containers[0]["labels"] = { + "stack-back.ordered": "true", + "stack-back.order.1": "web", + "stack-back.order.2": "web", + } + containers += [ + { + "service": "web", + "labels": {"stack-back.volumes": True}, + "mounts": [ + {"Source": "d", "Destination": "/d", "Type": "volume"}, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + backup = running.containers_for_backup() + self.assertEqual(len(backup), 1) + + def test_ordered_true_no_labels_falls_back(self): + containers = self.createContainers() + containers[0]["labels"] = {"stack-back.ordered": "true"} + containers += [ + { + "service": "web", + "labels": {"stack-back.volumes": True}, + "mounts": [ + {"Source": "d", "Destination": "/d", "Type": "volume"}, + ], + }, + ] + with mock.patch( + list_containers_func, fixtures.containers(containers=containers) + ): + running = RunningContainers() + backup = running.containers_for_backup() + self.assertEqual(len(backup), 1) + self.assertEqual(backup[0].service_name, "web") +