diff --git a/sunbeam-python/sunbeam/commands/storage.py b/sunbeam-python/sunbeam/commands/storage.py new file mode 100644 index 000000000..34fa9c3bd --- /dev/null +++ b/sunbeam-python/sunbeam/commands/storage.py @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +import logging + +import click +from rich.console import Console + +from sunbeam.core.deployment import Deployment +from sunbeam.storage.registry import StorageBackendRegistry + +LOG = logging.getLogger(__name__) +console = Console() + +CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} + + +@click.group("storage", context_settings=CONTEXT_SETTINGS) +@click.pass_context +def storage(ctx): + """Manage Cinder storage backends. + + Provides commands to add, remove, configure and list storage backends. + Supports multiple backend types including Hitachi VSP and others. + """ + # Ensure we have a deployment object + if not hasattr(ctx, "obj") or not isinstance(ctx.obj, Deployment): + raise click.ClickException( + "Storage commands require a valid deployment context. " + "Please ensure sunbeam is properly initialized." + ) + + +def register_storage_commands(deployment: Deployment) -> None: + """Register storage backend commands with the storage group. + + This function is called from main.py to register all storage backend + commands dynamically based on available backends. + """ + try: + registry = StorageBackendRegistry() + registry.register_cli_commands(storage, deployment) + LOG.debug("Storage backend commands registered successfully") + except Exception as e: + LOG.error(f"Failed to register storage backend commands: {e}") + # Don't raise here as we want the CLI to still work diff --git a/sunbeam-python/sunbeam/core/common.py b/sunbeam-python/sunbeam/core/common.py index 17675e87e..df3c5b20b 100644 --- a/sunbeam-python/sunbeam/core/common.py +++ b/sunbeam-python/sunbeam/core/common.py @@ -599,3 +599,45 @@ def convert_retry_failure_as_result(retry_state: RetryCallState) -> Result: return Result(ResultType.FAILED, str(retry_state.outcome.exception())) else: return Result(ResultType.FAILED) + + +def friendly_terraform_lock_retry_callback(retry_state: RetryCallState) -> Result: + """Friendly retry callback for Terraform state lock exceptions. + + Shows user-friendly messages during lock retries + instead of verbose Terraform output. + """ + from sunbeam.core.terraform import TerraformStateLockedException + + if retry_state.outcome is not None: + exception = retry_state.outcome.exception() + if isinstance(exception, TerraformStateLockedException): + # Extract lock ID from the error message if possible + lock_id = "unknown" + error_str = str(exception) + if "ID:" in error_str: + try: + # Extract lock ID from Terraform output + lines = error_str.split("\n") + for line in lines: + if "ID:" in line: + lock_id = line.split("ID:")[1].strip() + break + except Exception: + LOG.debug( + "Failed to extract lock ID from Terraform output: %s", + error_str, + ) + pass + + return Result( + ResultType.FAILED, + f"Terraform state is locked (ID: {lock_id}). " + f"This usually resolves automatically. " + f"If it persists, use 'sunbeam plans unlock ' to " + f"clear stale locks.", + ) + else: + return Result(ResultType.FAILED, str(exception)) + else: + return Result(ResultType.FAILED, "Operation failed after retries") diff --git a/sunbeam-python/sunbeam/main.py b/sunbeam-python/sunbeam/main.py index 931e74301..7e95a2d75 100644 --- a/sunbeam-python/sunbeam/main.py +++ b/sunbeam-python/sunbeam/main.py @@ -20,6 +20,7 @@ from sunbeam.commands import prepare_node as prepare_node_cmds from sunbeam.commands import proxy as proxy_cmds from sunbeam.commands import sso as sso_cmd +from sunbeam.commands import storage as storage_cmds from sunbeam.commands import utils as utils_cmds from sunbeam.core import deployments as deployments_jobs from sunbeam.provider import commands as provider_cmds @@ -113,6 +114,7 @@ def main(): cli.add_command(launch_cmds.launch) cli.add_command(openrc_cmds.openrc) cli.add_command(dasboard_url_cmds.dashboard_url) + cli.add_command(storage_cmds.storage) # Add identity group cli.add_command(identity_group) @@ -157,6 +159,9 @@ def main(): juju.add_command(juju_cmds.register_controller) juju.add_command(juju_cmds.unregister_controller) + # Register storage backend commands + storage_cmds.register_storage_commands(deployment) + # Register the features after all groups,commands are registered deployment.get_feature_manager().register(cli, deployment) diff --git a/sunbeam-python/sunbeam/storage/__init__.py b/sunbeam-python/sunbeam/storage/__init__.py new file mode 100644 index 000000000..005b6db0d --- /dev/null +++ b/sunbeam-python/sunbeam/storage/__init__.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Sunbeam Storage Backends. + +This module provides a pluggable storage backend system for Sunbeam. +""" + +# Backends are loaded dynamically by the registry - no hardcoded imports needed diff --git a/sunbeam-python/sunbeam/storage/backends/__init__.py b/sunbeam-python/sunbeam/storage/backends/__init__.py new file mode 100644 index 000000000..f4e0ac8a7 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/__init__.py @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Sunbeam Storage Backend Implementations. + +This package contains implementations of various storage backends for Sunbeam. +""" diff --git a/sunbeam-python/sunbeam/storage/backends/dellsc/__init__.py b/sunbeam-python/sunbeam/storage/backends/dellsc/__init__.py new file mode 100644 index 000000000..dbc28643e --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/dellsc/__init__.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Dell Storage Center backend for Sunbeam storage.""" + +from .backend import DellSCBackend, DellSCConfig +from .cli import DellscCLI + +__all__ = ["DellSCBackend", "DellSCConfig", "DellscCLI"] \ No newline at end of file diff --git a/sunbeam-python/sunbeam/storage/backends/dellsc/backend.py b/sunbeam-python/sunbeam/storage/backends/dellsc/backend.py new file mode 100644 index 000000000..1f257b4a6 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/dellsc/backend.py @@ -0,0 +1,397 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Dell Storage Center storage backend implementation using base step classes.""" + +import logging +from typing import Any, Dict, Literal, Set + +import click + +try: + import yaml as _yaml # type: ignore + + yaml: Any = _yaml +except Exception: # yaml optional; handle gracefully at runtime + yaml = None + +# Import pydantic Field directly +from pydantic import Field +from rich.console import Console + +from sunbeam.core.common import BaseStep +from sunbeam.core.deployment import Deployment +from sunbeam.core.juju import JujuHelper +from sunbeam.core.manifest import Manifest +from sunbeam.core.terraform import TerraformHelper +from sunbeam.storage.base import StorageBackendBase +from sunbeam.storage.models import StorageBackendConfig +from sunbeam.storage.steps import ( + BaseStorageBackendConfigUpdateStep, + BaseStorageBackendDeployStep, + BaseStorageBackendDestroyStep, +) + +LOG = logging.getLogger(__name__) +console = Console() + + +class DellSCConfig(StorageBackendConfig): + """Static configuration model for Dell Storage Center storage backend. + + This model includes all configuration options supported by the + cinder-volume-dellsc charm as defined in charmcraft.yaml. + """ + + # Required fields (inherited from StorageBackendConfig) + # name: str (from base class) + + # Mandatory connection parameters + san_ip: str = Field(..., description="Dell Storage Center management IP or hostname") + san_username: str = Field(..., description="SAN management username") + san_password: str = Field(..., description="SAN management password") + dell_sc_ssn: int = Field(default=64702, description="Storage Center System Serial Number") + protocol: Literal["fc", "iscsi"] = Field( + default="fc", description="Front-end protocol (fc or iscsi)" + ) + + # Backend configuration + volume_backend_name: str = Field( + default="", description="Name that Cinder will report for this backend" + ) + backend_availability_zone: str = Field( + default="", description="Availability zone to associate with this backend" + ) + + # Dell Storage Center specific options + dell_sc_api_port: int = Field( + default=3033, description="Dell Storage Center API port" + ) + dell_sc_server_folder: str = Field( + default="openstack", description="Server folder name on Dell SC" + ) + dell_sc_volume_folder: str = Field( + default="openstack", description="Volume folder name on Dell SC" + ) + dell_server_os: str = Field( + default="Red Hat Linux 6.x", description="Server OS type for Dell SC" + ) + dell_sc_verify_cert: bool = Field( + default=False, description="Verify SSL certificate for Dell SC API" + ) + + # Provisioning options + san_thin_provision: bool = Field( + default=True, description="Enable thin provisioning" + ) + + # Domain and network filtering + excluded_domain_ips: str = Field( + default="", description="Comma-separated list of excluded domain IPs" + ) + included_domain_ips: str = Field( + default="", description="Comma-separated list of included domain IPs" + ) + + # Dual DSM configuration + secondary_san_ip: str = Field( + default="", description="Secondary Dell Storage Center management IP" + ) + secondary_san_username: str = Field( + default="Admin", description="Secondary SAN management username" + ) + secondary_san_password: str = Field( + default="", description="Secondary SAN management password" + ) + secondary_sc_api_port: int = Field( + default=3033, description="Secondary Dell Storage Center API port" + ) + + # API timeout configuration + dell_api_async_rest_timeout: int = Field( + default=15, description="Async REST API timeout in seconds" + ) + dell_api_sync_rest_timeout: int = Field( + default=30, description="Sync REST API timeout in seconds" + ) + + # SSH connection settings + ssh_conn_timeout: int = Field( + default=30, description="SSH connection timeout in seconds" + ) + ssh_max_pool_conn: int = Field( + default=5, description="Maximum SSH pool connections" + ) + ssh_min_pool_conn: int = Field( + default=1, description="Minimum SSH pool connections" + ) + + # Juju secrets for credentials (not charm config options) + san_credentials_secret: str = Field( + default="", description="Juju secret URI for SAN credentials" + ) + secondary_san_credentials_secret: str = Field( + default="", description="Juju secret URI for secondary SAN credentials" + ) + + +class DellSCBackend(StorageBackendBase): + """Dell Storage Center storage backend implementation.""" + + name = "dellsc" + display_name = "Dell Storage Center" + charm_name = "cinder-volume-dellsc" + + def __init__(self): + """Initialize Dell Storage Center backend.""" + super().__init__() + self.tfplan = "dellsc-backend-plan" + self.tfplan_dir = "deploy-dellsc-backend" + + charm_channel = ( + "latest/edge" # Use edge for development, change to stable for production + ) + charm_revision = 3 + charm_base = "ubuntu@24.04" + backend_endpoint = "cinder-volume" + units = 1 + additional_integrations = [] + + @property + def config_class(self) -> type[StorageBackendConfig]: + """Return the configuration class for Dell Storage Center backend.""" + return DellSCConfig + + def _get_credential_fields(self) -> Set[str]: + """Get set of credential field names that should be excluded from charm config. + + For Dell SC backend, we exclude all credential fields and secret URIs. + """ + return { + # meta + "name", + # primary array credentials + "san_username", + "san_password", + # secondary array credentials + "secondary_san_username", + "secondary_san_password", + # juju secret URIs + "san_credentials_secret", + "secondary_san_credentials_secret", + } + + def get_field_mapping(self) -> Dict[str, str]: + """Get mapping from config fields to charm config options. + + Uses base class automatic mapping and excludes credential fields. + """ + # Start from the base automatic mapping + mapping = super().get_field_mapping() + + # Exclude credential fields + exclude = self._get_credential_fields() + return {k: v for k, v in mapping.items() if k not in exclude} + + def get_terraform_variables( + self, backend_name: str, config: StorageBackendConfig, model: str + ) -> Dict[str, Any]: + """Generate Terraform variables for Dell Storage Center backend deployment.""" + # Map our configuration fields to the correct charm configuration option names + config_dict = config.model_dump() + field_mapping = self.get_field_mapping() + + # Filter config using base class method, excluding credential fields + charm_config = self._filter_config_for_charm( + config_dict, field_mapping, exclude_fields=self._get_credential_fields() + ) + + # Build Terraform variables to match the plan's expected format + tfvars = { + "machine_model": model, + "charm_dellsc_name": self.charm_name, + "charm_dellsc_base": self.charm_base, + "charm_dellsc_channel": self.charm_channel, + "charm_dellsc_endpoint": self.backend_endpoint, + "charm_dellsc_revision": self.charm_revision, + "dellsc_backends": { + backend_name: { + "charm_config": charm_config, + # Main array credentials (always required) + "san_username": config_dict.get("san_username", ""), + "san_password": config_dict.get("san_password", ""), + # Secondary array credentials (optional) + "secondary_san_username": config_dict.get("secondary_san_username", ""), + "secondary_san_password": config_dict.get("secondary_san_password", ""), + } + }, + } + + return tfvars + + def _get_default_config(self) -> DellSCConfig: + """Get a default configuration instance for comparison. + + This creates a config instance with all fields set to their Pydantic defaults, + allowing proper filtering of default values in _filter_config_for_charm(). + """ + # Create instance with minimal required fields, letting Pydantic set defaults + return DellSCConfig( + name="dummy", # Required field + san_ip="dummy", # Required field + san_username="dummy", # Required field + san_password="dummy", # Required field, noqa: S106 + # All other fields will use their Pydantic Field() defaults: + # dell_sc_ssn=64702, protocol="fc", volume_backend_name="", + # backend_availability_zone="", dell_sc_api_port=3033, etc. + ) + + def prompt_for_config(self, backend_name: str) -> DellSCConfig: + """Prompt user for Dell Storage Center-specific configuration.""" + return self._prompt_for_config(backend_name) + + def _prompt_for_config(self, backend_name: str) -> DellSCConfig: + """Prompt user for Dell Storage Center backend configuration.""" + console.print( + "\n[bold blue]Dell Storage Center Backend Configuration[/bold blue]" + ) + console.print("Please provide the required configuration options:") + + # Prompt for required fields + san_ip = click.prompt( + "Management IP/FQDN", type=str, value_proc=self._validate_ip_or_fqdn + ) + dell_sc_ssn = click.prompt( + "Storage Center System Serial Number", type=int, default=64702 + ) + protocol = click.prompt( + "Protocol", + type=click.Choice(["fc", "iscsi"], case_sensitive=False), + default="fc", + ) + + # Main array credentials (will be automatically converted to Juju secret) + console.print("\n[bold yellow]Array Credentials[/bold yellow]") + console.print( + "These credentials will be automatically stored in a Juju secret." + ) + san_username = click.prompt("SAN username", type=str, default="admin") + san_password = click.prompt("SAN password", type=str, hide_input=True) + + # Optional: prompt for volume backend name (defaults to backend name) + volume_backend_name = click.prompt( + "Volume backend name", type=str, default=backend_name, show_default=True + ) + + # Optional: Dual DSM configuration + secondary_san_username = "" + secondary_san_password = "" + secondary_san_ip = "" + + configure_dual_dsm = click.confirm( + "\nConfigure dual DSM (high availability)?", default=False + ) + if configure_dual_dsm: + console.print("\n[bold yellow]Secondary DSM Configuration[/bold yellow]") + secondary_san_ip = click.prompt( + "Secondary management IP/FQDN", type=str, value_proc=self._validate_ip_or_fqdn + ) + secondary_san_username = click.prompt( + "Secondary SAN username", type=str, default="Admin" + ) + secondary_san_password = click.prompt( + "Secondary SAN password", type=str, hide_input=True + ) + + return DellSCConfig( + name=backend_name, + san_ip=san_ip, + san_username=san_username, + san_password=san_password, + dell_sc_ssn=dell_sc_ssn, + protocol=protocol, + volume_backend_name=volume_backend_name, + secondary_san_ip=secondary_san_ip, + secondary_san_username=secondary_san_username, + secondary_san_password=secondary_san_password, + ) + + # Implementation of abstract methods from StorageBackendBase + def create_deploy_step( + self, + deployment: Deployment, + client, + tfhelper: TerraformHelper, + jhelper: JujuHelper, + manifest: Manifest, + backend_name: str, + backend_config: StorageBackendConfig, + model: str, + ) -> BaseStep: + """Create a deployment step for Dell Storage Center backend.""" + return DellSCDeployStep( + deployment, + client, + tfhelper, + jhelper, + manifest, + backend_name, + backend_config, + self, + model, + ) + + def create_destroy_step( + self, + deployment: Deployment, + client, + tfhelper: TerraformHelper, + jhelper: JujuHelper, + manifest: Manifest, + backend_name: str, + model: str, + ) -> BaseStep: + """Create a destruction step for Dell Storage Center backend.""" + return DellSCDestroyStep( + deployment, + client, + tfhelper, + jhelper, + manifest, + backend_name, + self, + model, + ) + + def create_update_config_step( + self, + deployment: Deployment, + backend_name: str, + config_updates: Dict[str, Any], + ) -> BaseStep: + """Create a configuration update step for Dell Storage Center backend.""" + return DellSCUpdateConfigStep( + deployment, + self, + backend_name, + config_updates, + ) + + +# Dell Storage Center-specific step implementations using base step classes +class DellSCDeployStep(BaseStorageBackendDeployStep): + """Deploy Dell Storage Center storage backend using base step class.""" + + def get_terraform_variables(self) -> Dict[str, Any]: + """Get Terraform variables for Dell Storage Center backend deployment.""" + return self.backend_instance.get_terraform_variables( + self.backend_name, self.backend_config, self.model + ) + + +class DellSCDestroyStep(BaseStorageBackendDestroyStep): + """Destroy Dell Storage Center storage backend using base step class.""" + + +class DellSCUpdateConfigStep(BaseStorageBackendConfigUpdateStep): + """Update Dell Storage Center storage backend configuration using base step class.""" diff --git a/sunbeam-python/sunbeam/storage/backends/dellsc/cli.py b/sunbeam-python/sunbeam/storage/backends/dellsc/cli.py new file mode 100644 index 000000000..ad8e94a16 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/dellsc/cli.py @@ -0,0 +1,344 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""CLI functionality for Dell Storage Center storage backend. + +This module contains all CLI-related code moved from backend.py, +including command registration and helper functions. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, Optional + +import click +import pydantic +from rich.console import Console + +try: + import yaml as _yaml # type: ignore + + yaml: Any = _yaml +except Exception: # yaml optional; handle gracefully at runtime + yaml = None + +from sunbeam.core.deployment import Deployment +from sunbeam.storage.backends.dellsc.backend import DellSCBackend +from sunbeam.storage.service import StorageBackendService + +console = Console() + + +class DellscCLI: + """CLI functionality for Dell Storage Center storage backend.""" + + def __init__(self, backend: DellSCBackend): + self.backend = backend + + def _load_config_file(self, path: Optional[Path]) -> Dict[str, Any]: + """Load YAML or JSON config file into a dictionary. + + YAML is preferred if PyYAML is available, otherwise JSON is used. + """ + if not path: + return {} + text = path.read_text() + if yaml is not None: + return dict(yaml.safe_load(text) or {}) + + return dict(json.loads(text)) + + def register_add_cli(self, add: click.Group) -> None: # noqa: C901 + """Register 'sunbeam storage add dellsc'. + + Includes typed options and a --config-file flag. + """ + + def _click_type_for(field_info) -> click.types.ParamType: + # Map pydantic field to Click type + ann = getattr(field_info, "annotation", None) + typ = None + if ann is not None: + typ = str(ann) + elif hasattr(field_info, "type_"): + typ = str(field_info.type_) + if typ and ("int" in typ): + return click.INT + if typ and ("float" in typ): + return click.FLOAT + if typ and ("bool" in typ): + return click.BOOL + return click.STRING + + def _build_params(required_all: bool) -> list: + params: list = [] + # name option (not required; prompt in interactive mode) + params.append( + click.Option(["--name"], type=str, required=False, help="Backend name") + ) + # config file (optional) + params.append( + click.Option( + ["--config-file"], + type=click.Path(exists=True, dir_okay=False, path_type=Path), + required=False, + help="YAML/JSON config file", + ) + ) + # Model-derived options + fields = getattr(self.backend.config_class, "model_fields", {}) + for fname, finfo in fields.items(): + if fname == "name": + continue + opt_name = "--" + fname.replace("_", "-") + click_type = _click_type_for(finfo) + # Determine requiredness for add (respect model) + # For interactive UX, keep CLI options optional; the model + # enforces requiredness. + is_required = False + # Help text + descr = None + if hasattr(finfo, "field_info") and hasattr( + finfo.field_info, "description" + ): + descr = finfo.field_info.description + elif hasattr(finfo, "description"): + descr = finfo.description + params.append( + click.Option( + [opt_name], type=click_type, required=is_required, help=descr + ) + ) + return params + + def _build_config_from_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: + # Extract name and field values from kwargs + # (Click converts dashes to underscores) + cfg: Dict[str, Any] = {} + for k, v in kwargs.items(): + if v is None: + continue + cfg[k] = v + return cfg + + def add_callback(**kwargs): + deployment: Deployment = click.get_current_context().obj + cfg_file = kwargs.pop("config_file", None) + file_cfg = self._load_config_file(cfg_file) + cli_cfg = _build_config_from_kwargs(kwargs) + # Determine if interactive: no config-file and no CLI options supplied + provided_cli_values = {k: v for k, v in cli_cfg.items() if v is not None} + interactive = not file_cfg and not provided_cli_values + + if interactive: + # Prompt for name and full config via helper + console.print( + f"[blue]Setting up {self.backend.display_name} backend[/blue]" + ) + backend_name = click.prompt("Backend name", type=str) + config_instance = self.backend.prompt_for_config(backend_name) + config_instance.name = backend_name + self.backend.add_backend( + deployment, backend_name, config_instance, console + ) + return + + # Non-interactive path: merge file and CLI, validate + merged = {**file_cfg, **provided_cli_values} + try: + config_instance = self.backend.config_class(**merged) + except pydantic.ValidationError as e: + console.print("[red]Configuration validation error:[/red]") + for error in e.errors(): + field_name = error.get("loc", ["unknown"])[0] + # Convert field name to CLI parameter format + cli_param = f"--{field_name.replace('_', '-')}" + console.print(f" {cli_param}: {error['msg']}") + raise click.Abort() + backend_name = merged.get("name") + if not backend_name: + raise click.BadParameter( + "--name is required when not running interactively" + ) + self.backend.add_backend(deployment, backend_name, config_instance, console) + + # Build command dynamically with parameters + params = _build_params(required_all=False) + help_text = ( + f"Add {self.backend.display_name} backend.\n\n" + "Behavior:\n" + "- If no options are provided, runs in interactive mode and prompts " + "for all required fields.\n" + "- If options and/or --config-file are provided, runs non-" + "interactively and validates against the model.\n" + "- In non-interactive mode, --name is required (or supplied via " + "--config-file).\n\n" + "Examples:\n" + " sunbeam storage add dellsc\n" + " sunbeam storage add dellsc --config-file dellsc.yaml\n" + " sunbeam storage add dellsc --name mydellsc --san-ip 10.0.0.10 " + "--protocol fc\n" + ) + cmd = click.Command( + name=self.backend.name, + params=params, + callback=add_callback, + help=help_text, + ) + add.add_command(cmd) + + def register_cli( # noqa: C901 + self, + remove: click.Group, + config_show: click.Group, + config_set: click.Group, + config_options: click.Group, + deployment: Deployment, + ) -> None: + """Register management commands for Dell Storage Center backend.""" + + @click.command(name=self.backend.name) + @click.argument("backend_name", type=str) + @click.option("--yes", "-y", is_flag=True, help="Skip confirmation prompt") + @click.pass_context + def remove_dellsc(ctx, backend_name: str, yes: bool): + service = self.backend._get_service(deployment) + if not service.backend_exists(backend_name, self.backend.name): + console.print(f"[red]Error: Backend '{backend_name}' not found[/red]") + raise click.Abort() + if not yes: + click.confirm( + f"Remove {self.backend.display_name} backend '{backend_name}'?", + abort=True, + ) + try: + self.backend.remove_backend(deployment, backend_name, console) + except Exception as e: + console.print(f"[red]Error removing backend: {e}[/red]") + raise click.Abort() + + remove.add_command(remove_dellsc) + + @click.command(name=self.backend.name) + @click.argument("backend_name", type=str) + @click.pass_context + def config_show_dellsc(ctx, backend_name: str): + service = self.backend._get_service(deployment) + config = service.get_backend_config(backend_name, self.backend.name) + self.backend.display_config_table(backend_name, config) + + config_show.add_command(config_show_dellsc) + + # Build typed options for config set (only provided options are updated) + def _build_set_params() -> list: + params = [] + # Add config-file option first + params.append( + click.Option( + ["--config-file"], + type=click.Path( + exists=True, dir_okay=False, readable=True, path_type=Path + ), + required=False, + help="YAML/JSON config file with updates", + ) + ) + fields = getattr(self.backend.config_class, "model_fields", {}) + for fname, finfo in fields.items(): + if fname == "name": + continue + opt = "--" + fname.replace("_", "-") + # For updates, make everything optional with default None + # so we can detect presence + click_type: click.ParamType = click.STRING + ann = getattr(finfo, "annotation", None) + if ann is not None and ("bool" in str(ann)): + click_type = click.BOOL + elif ann is not None and ("int" in str(ann)): + click_type = click.INT + elif ann is not None and ("float" in str(ann)): + click_type = click.FLOAT + descr = None + if hasattr(finfo, "field_info") and hasattr( + finfo.field_info, "description" + ): + descr = finfo.field_info.description + elif hasattr(finfo, "description"): + descr = finfo.description + params.append( + click.Option( + [opt], type=click_type, required=False, default=None, help=descr + ) + ) + return params + + def set_callback(backend_name: str, **kwargs): + cfg_file = kwargs.pop("config_file", None) + file_cfg = {} + if cfg_file: + file_cfg = self._load_config_file(cfg_file) + # Only include keys that were explicitly provided (value not None) + updates = {k: v for k, v in kwargs.items() if v is not None} + updates = {**file_cfg, **updates} + try: + # Get current configuration and merge with updates for validation + service = StorageBackendService(deployment) + current_config = service.get_backend_config( + backend_name, self.backend.name + ) + + # Create a merged config for validation + # Current config comes from charm (kebab-case keys), + # convert to snake_case + merged_config = {} + for key, value in current_config.items(): + snake_key = key.replace("-", "_") + merged_config[snake_key] = value + + # Updates from CLI are already in snake_case + # (Click converts --protocol to protocol) + merged_config.update(updates) + merged_config["name"] = backend_name # Ensure name is set + _ = self.backend.config_class(**merged_config) + except pydantic.ValidationError as e: + console.print("[red]Configuration validation error:[/red]") + for error in e.errors(): + field_name = error["loc"][0] if error.get("loc") else "unknown" + # Convert field name to CLI parameter format + cli_param = f"--{str(field_name).replace('_', '-')}" + console.print(f" {cli_param}: {error['msg']}") + raise click.ClickException( + "Configuration update failed due to validation error" + ) + try: + self.backend.update_backend_config(deployment, backend_name, updates) + console.print( + ( + f"[green]Configuration updated for {self.backend.display_name} " + f"backend '{backend_name}'[/green]" + ) + ) + except Exception as e: + console.print(f"[red]Failed to update configuration: {e}[/red]") + raise click.ClickException(f"Configuration update failed: {e}") + + set_params = _build_set_params() + # Add backend_name as a required argument + set_params.insert(0, click.Argument(["backend_name"], type=str, required=True)) + set_cmd = click.Command( + name=self.backend.name, + params=set_params, + callback=set_callback, + help="Set configuration options", + ) + config_set.add_command(set_cmd) + + @click.command(name=self.backend.name) + @click.argument("backend_name", type=str, required=False) + @click.pass_context + def config_options_dellsc(ctx, backend_name: str | None = None): + self.backend.display_config_options() + + config_options.add_command(config_options_dellsc) diff --git a/sunbeam-python/sunbeam/storage/backends/dellsc/deploy-dellsc-backend/main.tf b/sunbeam-python/sunbeam/storage/backends/dellsc/deploy-dellsc-backend/main.tf new file mode 100644 index 000000000..35eb78acd --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/dellsc/deploy-dellsc-backend/main.tf @@ -0,0 +1,140 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +terraform { + required_providers { + juju = { + source = "juju/juju" + version = "= 0.20.0" + } + } +} + +provider "juju" {} + +data "juju_model" "machine_model" { + name = var.machine_model +} + +# Create Juju secrets for Dell Storage Center backend credentials +# Main array credentials (required for all backends) +resource "juju_secret" "dellsc_san_credentials" { + for_each = { + for backend_name, backend_config in var.dellsc_backends : + backend_name => backend_config + if backend_config.san_username != "" && backend_config.san_password != "" + } + + model = data.juju_model.machine_model.name + name = "${each.key}-san-credentials" + value = { + username = each.value.san_username + password = each.value.san_password + } +} + +# Secondary array credentials (optional, for dual DSM) +resource "juju_secret" "dellsc_secondary_san_credentials" { + for_each = { + for backend_name, backend_config in var.dellsc_backends : + backend_name => backend_config + if backend_config.secondary_san_username != "" && backend_config.secondary_san_password != "" + } + + model = data.juju_model.machine_model.name + name = "${each.key}-secondary-san-credentials" + value = { + username = each.value.secondary_san_username + password = each.value.secondary_san_password + } +} + +# Grant access to secrets for Dell Storage Center backend applications +resource "juju_access_secret" "dellsc_san_credentials_access" { + for_each = { + for backend_name, backend_config in var.dellsc_backends : + backend_name => backend_config + if backend_config.san_username != "" && backend_config.san_password != "" + } + + model = data.juju_model.machine_model.name + secret_id = juju_secret.dellsc_san_credentials[each.key].secret_id + applications = [each.key] + + # Ensure proper dependency ordering to avoid provider bugs + depends_on = [juju_application.dellsc_backends] + + lifecycle { + # Prevent destruction when applications list becomes empty + prevent_destroy = false + # Create before destroy to avoid empty applications list state + create_before_destroy = true + } +} + +resource "juju_access_secret" "dellsc_secondary_san_credentials_access" { + for_each = { + for backend_name, backend_config in var.dellsc_backends : + backend_name => backend_config + if backend_config.secondary_san_username != "" && backend_config.secondary_san_password != "" + } + + model = data.juju_model.machine_model.name + secret_id = juju_secret.dellsc_secondary_san_credentials[each.key].secret_id + applications = [each.key] + + # Ensure proper dependency ordering to avoid provider bugs + depends_on = [juju_application.dellsc_backends] + + lifecycle { + # Prevent destruction when applications list becomes empty + prevent_destroy = false + # Create before destroy to avoid empty applications list state + create_before_destroy = true + } +} + +# Deploy Dell Storage Center backend charms +resource "juju_application" "dellsc_backends" { + for_each = var.dellsc_backends + + name = each.key + model = data.juju_model.machine_model.name + units = 1 + + charm { + name = var.charm_dellsc_name + channel = var.charm_dellsc_channel + revision = var.charm_dellsc_revision + base = var.charm_dellsc_base + } + + config = merge({ + volume-backend-name = each.key + }, each.value.charm_config, { + # Main array credentials - always required + san-credentials-secret = contains(keys(juju_secret.dellsc_san_credentials), each.key) ? juju_secret.dellsc_san_credentials[each.key].secret_id : "" + + # Secondary array credentials - only if dual DSM is configured + secondary-san-credentials-secret = contains(keys(juju_secret.dellsc_secondary_san_credentials), each.key) ? juju_secret.dellsc_secondary_san_credentials[each.key].secret_id : "" + }) + + endpoint_bindings = var.endpoint_bindings +} + +# Integrate Dell Storage Center backends with main cinder-volume +resource "juju_integration" "dellsc_to_cinder_volume" { + for_each = var.dellsc_backends + + model = var.machine_model + + application { + name = juju_application.dellsc_backends[each.key].name + endpoint = var.charm_dellsc_endpoint + } + + application { + name = "cinder-volume" + endpoint = var.charm_dellsc_endpoint + } +} diff --git a/sunbeam-python/sunbeam/storage/backends/dellsc/deploy-dellsc-backend/outputs.tf b/sunbeam-python/sunbeam/storage/backends/dellsc/deploy-dellsc-backend/outputs.tf new file mode 100644 index 000000000..295810f44 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/dellsc/deploy-dellsc-backend/outputs.tf @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +output "dellsc_backend_applications" { + description = "Map of deployed Dell Storage Center backend applications" + value = { + for name, app in juju_application.dellsc_backends : name => { + name = app.name + model = app.model + units = app.units + } + } +} diff --git a/sunbeam-python/sunbeam/storage/backends/dellsc/deploy-dellsc-backend/test-variables.tfvars b/sunbeam-python/sunbeam/storage/backends/dellsc/deploy-dellsc-backend/test-variables.tfvars new file mode 100644 index 000000000..7464d4c48 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/dellsc/deploy-dellsc-backend/test-variables.tfvars @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +# Test variables for Dell Storage Center backend deployment + +machine_model = "openstack" + +charm_dellsc_name = "cinder-volume-dellsc" +charm_dellsc_base = "ubuntu@24.04" +charm_dellsc_channel = "latest/edge" +charm_dellsc_endpoint = "cinder-volume" +charm_dellsc_revision = 1 + +dellsc_backends = { + "dellsc-test" = { + charm_config = { + san-ip = "192.168.1.100" + dell-sc-ssn = "64702" + protocol = "fc" + dell-sc-api-port = "3033" + dell-sc-server-folder = "openstack" + dell-sc-volume-folder = "openstack" + dell-server-os = "Red Hat Linux 6.x" + dell-sc-verify-cert = "false" + san-thin-provision = "true" + dell-api-async-rest-timeout = "15" + dell-api-sync-rest-timeout = "30" + ssh-conn-timeout = "30" + ssh-max-pool-conn = "5" + ssh-min-pool-conn = "1" + } + + # Main array credentials + san_username = "admin" + san_password = "password123" + + # Secondary array credentials (optional, for dual DSM) + secondary_san_username = "" + secondary_san_password = "" + } +} diff --git a/sunbeam-python/sunbeam/storage/backends/dellsc/deploy-dellsc-backend/variables.tf b/sunbeam-python/sunbeam/storage/backends/dellsc/deploy-dellsc-backend/variables.tf new file mode 100644 index 000000000..1ec1ebcd0 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/dellsc/deploy-dellsc-backend/variables.tf @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +variable "machine_model" { + description = "Name of the machine model to deploy to" + type = string +} + +variable "charm_dellsc_name" { + description = "Name of the Dell Storage Center charm" + type = string + default = "cinder-volume-dellsc" +} + +variable "charm_dellsc_base" { + description = "Base for the Dell Storage Center charm" + type = string + default = "ubuntu@24.04" +} + +variable "charm_dellsc_channel" { + description = "Operator channel for Dell Storage Center backend deployment" + type = string + default = "latest/edge" +} + +variable "charm_dellsc_endpoint" { + description = "Endpoint name for Dell Storage Center backend integration" + type = string + default = "cinder-volume" +} + +variable "charm_dellsc_revision" { + description = "Operator channel revision for Dell Storage Center backend deployment" + type = number + default = null +} + +variable "dellsc_backends" { + description = "Map of Dell Storage Center backend configurations" + type = map(object({ + charm_config = map(string) + + # Main array credentials (required) + san_username = string + san_password = string + + # Secondary array credentials (optional, for dual DSM) + secondary_san_username = optional(string, "") + secondary_san_password = optional(string, "") + })) + default = {} +} + +variable "machine_ids" { + description = "List of machine ids to include" + type = list(string) + default = [] +} + +variable "endpoint_bindings" { + description = "Endpoint bindings for the applications" + type = set(map(string)) + default = null +} diff --git a/sunbeam-python/sunbeam/storage/backends/hitachi/backend.py b/sunbeam-python/sunbeam/storage/backends/hitachi/backend.py new file mode 100644 index 000000000..ef3875988 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/hitachi/backend.py @@ -0,0 +1,608 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Hitachi VSP storage backend implementation using base step classes.""" + +import logging +from typing import Any, Dict, Literal, Set + +import click + +try: + import yaml as _yaml # type: ignore + + yaml: Any = _yaml +except Exception: # yaml optional; handle gracefully at runtime + yaml = None + +# Import pydantic Field directly +from pydantic import Field +from rich.console import Console + +from sunbeam.core.common import BaseStep +from sunbeam.core.deployment import Deployment +from sunbeam.core.juju import JujuHelper +from sunbeam.core.manifest import Manifest +from sunbeam.core.terraform import TerraformHelper +from sunbeam.storage.base import StorageBackendBase +from sunbeam.storage.models import StorageBackendConfig +from sunbeam.storage.steps import ( + BaseStorageBackendConfigUpdateStep, + BaseStorageBackendDeployStep, + BaseStorageBackendDestroyStep, +) + +LOG = logging.getLogger(__name__) +console = Console() + + +class HitachiConfig(StorageBackendConfig): + """Static configuration model for Hitachi VSP storage backend. + + This model includes all configuration options supported by the + cinder-volume-hitachi charm as defined in charmcraft.yaml. + """ + + # Required fields (inherited from StorageBackendConfig) + # name: str (from base class) + + # Mandatory connection parameters + hitachi_storage_id: str = Field( + ..., description="Storage system product number/serial" + ) + hitachi_pools: str = Field( + ..., description="Comma-separated list of DP pool names/IDs" + ) + san_ip: str = Field(..., description="Hitachi VSP management IP or hostname") + san_username: str = Field(..., description="SAN management username") + san_password: str = Field(..., description="SAN management password") + protocol: Literal["FC", "iSCSI"] = Field( + ..., description="Front-end protocol (FC or iSCSI)" + ) + + # Backend configuration + volume_backend_name: str = Field( + default="", description="Name that Cinder will report for this backend" + ) + backend_availability_zone: str = Field( + default="", description="Availability zone to associate with this backend" + ) + + # Optional host-group / zoning controls + hitachi_target_ports: str = Field( + default="", description="Comma-separated front-end port labels" + ) + hitachi_compute_target_ports: str = Field( + default="", description="Comma-separated compute-node port IDs" + ) + hitachi_ldev_range: str = Field( + default="", description="LDEV range usable by the driver" + ) + hitachi_zoning_request: bool = Field( + default=False, description="Request FC zone-manager to create zoning" + ) + + # Copy & replication tuning + hitachi_copy_speed: int = Field( + default=3, description="Copy bandwidth throttle (1-15)" + ) + hitachi_copy_check_interval: int = Field( + default=3, description="Seconds between sync copy-status polls" + ) + hitachi_async_copy_check_interval: int = Field( + default=10, description="Seconds between async copy-status polls" + ) + + # iSCSI authentication + use_chap_auth: bool = Field( + default=False, description="Use CHAP authentication for iSCSI" + ) + + # Array ranges and controls + hitachi_discard_zero_page: bool = Field( + default=True, description="Enable zero-page reclamation in DP-VOLs" + ) + hitachi_exec_retry_interval: int = Field( + default=5, description="Seconds to wait before retrying REST API call" + ) + hitachi_extend_timeout: int = Field( + default=600, description="Max seconds to wait for volume extension" + ) + hitachi_group_create: bool = Field( + default=False, description="Automatically create host groups or iSCSI targets" + ) + hitachi_group_delete: bool = Field( + default=False, description="Automatically delete unused host groups" + ) + hitachi_group_name_format: str = Field( + default="", description="Python format string for naming host groups" + ) + hitachi_host_mode_options: str = Field( + default="", description="Comma-separated host mode options" + ) + hitachi_lock_timeout: int = Field( + default=7200, description="Max seconds for array login/unlock operations" + ) + hitachi_lun_retry_interval: int = Field( + default=1, description="Seconds before retrying LUN mapping" + ) + hitachi_lun_timeout: int = Field( + default=50, description="Max seconds to wait for LUN mapping" + ) + hitachi_port_scheduler: bool = Field( + default=False, description="Enable round-robin WWN registration" + ) + + # Mirror/replication settings + hitachi_mirror_compute_target_ports: str = Field( + default="", description="Compute-node port names for GAD" + ) + hitachi_mirror_ldev_range: str = Field( + default="", description="LDEV range for secondary storage" + ) + hitachi_mirror_pair_target_number: int = Field( + default=0, description="Host group number for GAD on secondary" + ) + hitachi_mirror_pool: str = Field( + default="", description="DP pool name/ID on secondary storage" + ) + hitachi_mirror_rest_api_ip: str = Field( + default="", description="REST API IP on secondary storage" + ) + hitachi_mirror_rest_api_port: int = Field( + default=443, description="REST API port on secondary storage" + ) + hitachi_mirror_rest_pair_target_ports: str = Field( + default="", description="Pair-target port names for GAD" + ) + hitachi_mirror_snap_pool: str = Field( + default="", description="Snapshot pool on secondary storage" + ) + hitachi_mirror_ssl_cert_path: str = Field( + default="", description="CA_BUNDLE for secondary REST endpoint" + ) + hitachi_mirror_ssl_cert_verify: bool = Field( + default=False, description="Validate SSL cert of secondary REST" + ) + hitachi_mirror_storage_id: str = Field( + default="", description="Product number of secondary storage" + ) + hitachi_mirror_target_ports: str = Field( + default="", description="Controller node port IDs for GAD" + ) + hitachi_mirror_use_chap_auth: bool = Field( + default=False, description="Use CHAP auth for GAD on secondary" + ) + + # Replication settings + hitachi_pair_target_number: int = Field( + default=0, description="Host group number for primary replication" + ) + hitachi_path_group_id: int = Field( + default=0, description="Path group ID for remote replication" + ) + hitachi_quorum_disk_id: int = Field( + default=0, description="Quorum disk ID for Global-Active Device" + ) + hitachi_replication_copy_speed: int = Field( + default=3, description="Copy speed for remote replication" + ) + hitachi_replication_number: int = Field( + default=0, description="Instance number for REST API on replication" + ) + hitachi_replication_status_check_long_interval: int = Field( + default=600, description="Poll interval after initial check" + ) + hitachi_replication_status_check_short_interval: int = Field( + default=5, description="Initial poll interval" + ) + hitachi_replication_status_check_timeout: int = Field( + default=86400, description="Max seconds for status change" + ) + + # REST API settings + hitachi_rest_another_ldev_mapped_retry_timeout: int = Field( + default=600, description="Retry seconds when LDEV allocation fails" + ) + hitachi_rest_connect_timeout: int = Field( + default=30, description="Max seconds to establish REST connection" + ) + hitachi_rest_disable_io_wait: bool = Field( + default=True, description="Detach volumes without waiting for I/O drain" + ) + hitachi_rest_get_api_response_timeout: int = Field( + default=1800, description="Max seconds for sync REST GET" + ) + hitachi_rest_job_api_response_timeout: int = Field( + default=1800, description="Max seconds for async REST PUT/DELETE" + ) + hitachi_rest_keep_session_loop_interval: int = Field( + default=180, description="Seconds between keep-alive loops" + ) + hitachi_rest_pair_target_ports: str = Field( + default="", description="Pair-target port names for REST operations" + ) + hitachi_rest_server_busy_timeout: int = Field( + default=7200, description="Max seconds when REST API returns busy" + ) + hitachi_rest_tcp_keepalive: bool = Field( + default=True, description="Enable TCP keepalive for REST connections" + ) + hitachi_rest_tcp_keepcnt: int = Field( + default=4, description="Number of TCP keepalive probes" + ) + hitachi_rest_tcp_keepidle: int = Field( + default=60, description="Seconds before sending first TCP keepalive" + ) + hitachi_rest_tcp_keepintvl: int = Field( + default=15, description="Seconds between TCP keepalive probes" + ) + hitachi_rest_timeout: int = Field( + default=30, description="Max seconds for each REST API call" + ) + hitachi_restore_timeout: int = Field( + default=86400, description="Max seconds to wait for restore operation" + ) + + # Snapshot settings + hitachi_snap_pool: str = Field(default="", description="Pool name/ID for snapshots") + hitachi_state_transition_timeout: int = Field( + default=900, description="Max seconds for volume state transition" + ) + + # Juju secrets for credentials (not charm config options) + san_credentials_secret: str = Field( + default="", description="Juju secret URI for SAN credentials" + ) + chap_credentials_secret: str = Field( + default="", description="Juju secret URI for CHAP credentials" + ) + hitachi_mirror_chap_credentials_secret: str = Field( + default="", description="Juju secret URI for mirror CHAP credentials" + ) + hitachi_mirror_rest_credentials_secret: str = Field( + default="", description="Juju secret URI for mirror REST credentials" + ) + + chap_username: str = Field( + default="", description="CHAP username for secret creation" + ) + chap_password: str = Field( + default="", description="CHAP password for secret creation" + ) + hitachi_mirror_chap_username: str = Field( + default="", description="Mirror CHAP username for secret creation" + ) + hitachi_mirror_chap_password: str = Field( + default="", description="Mirror CHAP password for secret creation" + ) + hitachi_mirror_rest_username: str = Field( + default="", description="Mirror REST username for secret creation" + ) + hitachi_mirror_rest_password: str = Field( + default="", description="Mirror REST password for secret creation" + ) + + +class HitachiBackend(StorageBackendBase): + """Hitachi storage backend implementation.""" + + name = "hitachi" + display_name = "Hitachi VSP Storage" + charm_name = "cinder-volume-hitachi" + + def __init__(self): + """Initialize Hitachi backend.""" + super().__init__() + self.tfplan = "hitachi-backend-plan" + self.tfplan_dir = "deploy-hitachi-backend" + + charm_channel = ( + "latest/edge" # Use edge for development, change to stable for production + ) + charm_revision = 2 + charm_base = "ubuntu@24.04" # Updated to match uploaded charm base + backend_endpoint = "cinder-volume" + units = 1 + additional_integrations = [] + + @property + def config_class(self) -> type[StorageBackendConfig]: + """Return the configuration class for Hitachi backend.""" + return HitachiConfig + + def _get_credential_fields(self) -> Set[str]: + """Get set of credential field names that should be excluded from charm config. + + For Hitachi backend, we exclude all credential fields and secret URIs. + """ + return { + # meta + "name", + # primary array credentials + "san_username", + "san_password", + # CHAP credentials + "chap_username", + "chap_password", + # mirror CHAP credentials + "hitachi_mirror_chap_username", + "hitachi_mirror_chap_password", + # mirror REST credentials + "hitachi_mirror_rest_username", + "hitachi_mirror_rest_password", + # juju secret URIs + "san_credentials_secret", + "chap_credentials_secret", + "hitachi_mirror_chap_credentials_secret", + "hitachi_mirror_rest_credentials_secret", + } + + def get_field_mapping(self) -> Dict[str, str]: + """Get mapping from config fields to charm config options. + + Uses base class automatic mapping and excludes credential fields. + """ + # Start from the base automatic mapping + mapping = super().get_field_mapping() + + # Exclude credential fields + exclude = self._get_credential_fields() + return {k: v for k, v in mapping.items() if k not in exclude} + + # CLI registration uses base class implementation + + def get_terraform_variables( + self, backend_name: str, config: StorageBackendConfig, model: str + ) -> Dict[str, Any]: + """Generate Terraform variables for Hitachi backend deployment.""" + # Map our configuration fields to the correct charm configuration option names + config_dict = config.model_dump() + field_mapping = self.get_field_mapping() + + # Filter config using base class method, excluding credential fields + charm_config = self._filter_config_for_charm( + config_dict, field_mapping, exclude_fields=self._get_credential_fields() + ) + + # Build Terraform variables to match the plan's expected format + tfvars = { + "machine_model": model, + "charm_hitachi_name": self.charm_name, + "charm_hitachi_base": self.charm_base, + "charm_hitachi_channel": self.charm_channel, + "charm_hitachi_endpoint": self.backend_endpoint, + "charm_hitachi_revision": self.charm_revision, + "hitachi_backends": { + backend_name: { + "charm_config": charm_config, + # Main array credentials (always required) + "san_username": config_dict.get("san_username", ""), + "san_password": config_dict.get("san_password", ""), + # CHAP credentials (optional) + "use_chap_auth": config_dict.get("use_chap_auth", False), + "chap_username": config_dict.get("chap_username", ""), + "chap_password": config_dict.get("chap_password", ""), + # Mirror CHAP credentials (optional) + "hitachi_mirror_chap_username": config_dict.get( + "hitachi_mirror_chap_username", "" + ), + "hitachi_mirror_chap_password": config_dict.get( + "hitachi_mirror_chap_password", "" + ), + # Mirror REST API credentials (optional) + "hitachi_mirror_rest_username": config_dict.get( + "hitachi_mirror_rest_username", "" + ), + "hitachi_mirror_rest_password": config_dict.get( + "hitachi_mirror_rest_password", "" + ), + } + }, + } + + return tfvars + + def _get_default_config(self) -> HitachiConfig: + """Get a default configuration instance for comparison.""" + return HitachiConfig( + name="dummy", + hitachi_storage_id="dummy", + hitachi_pools="dummy", + san_ip="dummy", + san_username="dummy", + san_password="dummy", # noqa: S106 + protocol="FC", + ) + + def prompt_for_config(self, backend_name: str) -> HitachiConfig: + """Prompt user for Hitachi-specific configuration.""" + return self._prompt_for_config(backend_name) + + # IP/FQDN validation uses base class implementation + + def _prompt_for_config(self, backend_name: str) -> HitachiConfig: + """Prompt user for Hitachi backend configuration.""" + console.print( + "\n[bold blue]Hitachi VSP Storage Backend Configuration[/bold blue]" + ) + console.print("Please provide the required configuration options:") + + # Prompt for required fields + hitachi_storage_id = click.prompt("Array serial number", type=str) + hitachi_pools = click.prompt("Storage pools (comma separated)", type=str) + protocol = click.prompt( + "Protocol", + type=click.Choice(["FC", "iSCSI"], case_sensitive=False), + default="FC", + ) + san_ip = click.prompt( + "Management IP/FQDN", type=str, value_proc=self._validate_ip_or_fqdn + ) + + # Main array credentials (will be automatically converted to Juju secret) + console.print("\n[bold yellow]Array Credentials[/bold yellow]") + console.print( + "These credentials will be automatically stored in a Juju secret." + ) + san_username = click.prompt("SAN username", type=str, default="maintenance") + san_password = click.prompt("SAN password", type=str, hide_input=True) + + # Optional: prompt for volume backend name (defaults to backend name) + volume_backend_name = click.prompt( + "Volume backend name", type=str, default=backend_name, show_default=True + ) + + # Optional: CHAP authentication for iSCSI + chap_username = "" + chap_password = "" + use_chap_auth = False + if protocol.lower() == "iscsi": + use_chap_auth = click.confirm( + "\nUse CHAP authentication for iSCSI?", default=False + ) + if use_chap_auth: + console.print("[bold yellow]CHAP Credentials[/bold yellow]") + console.print( + "These credentials will be automatically stored in a Juju secret." + ) + chap_username = click.prompt("CHAP username", type=str) + chap_password = click.prompt("CHAP password", type=str, hide_input=True) + + # Optional: Mirror/GAD configuration + hitachi_mirror_chap_username = "" + hitachi_mirror_chap_password = "" + hitachi_mirror_rest_username = "" + hitachi_mirror_rest_password = "" + + configure_mirror = click.confirm( + "\nConfigure mirror/replication (GAD) settings?", default=False + ) + if configure_mirror: + console.print( + "\n[bold yellow]Mirror/Replication Configuration[/bold yellow]" + ) + + # Mirror CHAP credentials + if click.confirm("Configure mirror CHAP credentials?", default=False): + console.print("[bold yellow]Mirror CHAP Credentials[/bold yellow]") + console.print( + "These credentials will be automatically stored in a Juju secret." + ) + hitachi_mirror_chap_username = click.prompt( + "Mirror CHAP username", type=str + ) + hitachi_mirror_chap_password = click.prompt( + "Mirror CHAP password", type=str, hide_input=True + ) + + # Mirror REST API credentials + if click.confirm("Configure mirror REST API credentials?", default=False): + console.print("[bold yellow]Mirror REST API Credentials[/bold yellow]") + console.print( + "These credentials will be automatically stored in a Juju secret." + ) + hitachi_mirror_rest_username = click.prompt( + "Mirror REST API username", type=str + ) + hitachi_mirror_rest_password = click.prompt( + "Mirror REST API password", type=str, hide_input=True + ) + + return HitachiConfig( + name=backend_name, + hitachi_storage_id=hitachi_storage_id, + hitachi_pools=hitachi_pools, + protocol=protocol, + san_ip=san_ip, + san_username=san_username, + san_password=san_password, + volume_backend_name=volume_backend_name, + use_chap_auth=use_chap_auth, + chap_username=chap_username, + chap_password=chap_password, + hitachi_mirror_chap_username=hitachi_mirror_chap_username, + hitachi_mirror_chap_password=hitachi_mirror_chap_password, + hitachi_mirror_rest_username=hitachi_mirror_rest_username, + hitachi_mirror_rest_password=hitachi_mirror_rest_password, + ) + + # Implementation of abstract methods from StorageBackendBase + def create_deploy_step( + self, + deployment: Deployment, + client, + tfhelper: TerraformHelper, + jhelper: JujuHelper, + manifest: Manifest, + backend_name: str, + backend_config: StorageBackendConfig, + model: str, + ) -> BaseStep: + """Create a deployment step for Hitachi backend.""" + return HitachiDeployStep( + deployment, + client, + tfhelper, + jhelper, + manifest, + backend_name, + backend_config, + self, + model, + ) + + def create_destroy_step( + self, + deployment: Deployment, + client, + tfhelper: TerraformHelper, + jhelper: JujuHelper, + manifest: Manifest, + backend_name: str, + model: str, + ) -> BaseStep: + """Create a destruction step for Hitachi backend.""" + return HitachiDestroyStep( + deployment, + client, + tfhelper, + jhelper, + manifest, + backend_name, + self, + model, + ) + + def create_update_config_step( + self, + deployment: Deployment, + backend_name: str, + config_updates: Dict[str, Any], + ) -> BaseStep: + """Create a configuration update step for Hitachi backend.""" + return HitachiUpdateConfigStep( + deployment, + self, + backend_name, + config_updates, + ) + + +# Hitachi-specific step implementations using base step classes +class HitachiDeployStep(BaseStorageBackendDeployStep): + """Deploy Hitachi storage backend using base step class.""" + + def get_terraform_variables(self) -> Dict[str, Any]: + """Get Terraform variables for Hitachi backend deployment.""" + return self.backend_instance.get_terraform_variables( + self.backend_name, self.backend_config, self.model + ) + + +class HitachiDestroyStep(BaseStorageBackendDestroyStep): + """Destroy Hitachi storage backend using base step class.""" + + +class HitachiUpdateConfigStep(BaseStorageBackendConfigUpdateStep): + """Update Hitachi storage backend configuration using base step class.""" diff --git a/sunbeam-python/sunbeam/storage/backends/hitachi/cli.py b/sunbeam-python/sunbeam/storage/backends/hitachi/cli.py new file mode 100644 index 000000000..e06effd76 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/hitachi/cli.py @@ -0,0 +1,344 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""CLI functionality for Hitachi storage backend. + +This module contains all CLI-related code moved from backend.py, +including command registration and helper functions. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, Optional + +import click +import pydantic +from rich.console import Console + +try: + import yaml as _yaml # type: ignore + + yaml: Any = _yaml +except Exception: # yaml optional; handle gracefully at runtime + yaml = None + +from sunbeam.core.deployment import Deployment +from sunbeam.storage.backends.hitachi.backend import HitachiBackend +from sunbeam.storage.service import StorageBackendService + +console = Console() + + +class HitachiCLI: + """CLI functionality for Hitachi storage backend.""" + + def __init__(self, backend: HitachiBackend): + self.backend = backend + + def _load_config_file(self, path: Optional[Path]) -> Dict[str, Any]: + """Load YAML or JSON config file into a dictionary. + + YAML is preferred if PyYAML is available, otherwise JSON is used. + """ + if not path: + return {} + text = path.read_text() + if yaml is not None: + return dict(yaml.safe_load(text) or {}) + + return dict(json.loads(text)) + + def register_add_cli(self, add: click.Group) -> None: # noqa: C901 + """Register 'sunbeam storage add hitachi'. + + Includes typed options and a --config-file flag. + """ + + def _click_type_for(field_info) -> click.types.ParamType: + # Map pydantic field to Click type + ann = getattr(field_info, "annotation", None) + typ = None + if ann is not None: + typ = str(ann) + elif hasattr(field_info, "type_"): + typ = str(field_info.type_) + if typ and ("int" in typ): + return click.INT + if typ and ("float" in typ): + return click.FLOAT + if typ and ("bool" in typ): + return click.BOOL + return click.STRING + + def _build_params(required_all: bool) -> list: + params: list = [] + # name option (not required; prompt in interactive mode) + params.append( + click.Option(["--name"], type=str, required=False, help="Backend name") + ) + # config file (optional) + params.append( + click.Option( + ["--config-file"], + type=click.Path(exists=True, dir_okay=False, path_type=Path), + required=False, + help="YAML/JSON config file", + ) + ) + # Model-derived options + fields = getattr(self.backend.config_class, "model_fields", {}) + for fname, finfo in fields.items(): + if fname == "name": + continue + opt_name = "--" + fname.replace("_", "-") + click_type = _click_type_for(finfo) + # Determine requiredness for add (respect model) + # For interactive UX, keep CLI options optional; the model + # enforces requiredness. + is_required = False + # Help text + descr = None + if hasattr(finfo, "field_info") and hasattr( + finfo.field_info, "description" + ): + descr = finfo.field_info.description + elif hasattr(finfo, "description"): + descr = finfo.description + params.append( + click.Option( + [opt_name], type=click_type, required=is_required, help=descr + ) + ) + return params + + def _build_config_from_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: + # Extract name and field values from kwargs + # (Click converts dashes to underscores) + cfg: Dict[str, Any] = {} + for k, v in kwargs.items(): + if v is None: + continue + cfg[k] = v + return cfg + + def add_callback(**kwargs): + deployment: Deployment = click.get_current_context().obj + cfg_file = kwargs.pop("config_file", None) + file_cfg = self._load_config_file(cfg_file) + cli_cfg = _build_config_from_kwargs(kwargs) + # Determine if interactive: no config-file and no CLI options supplied + provided_cli_values = {k: v for k, v in cli_cfg.items() if v is not None} + interactive = not file_cfg and not provided_cli_values + + if interactive: + # Prompt for name and full config via helper + console.print( + f"[blue]Setting up {self.backend.display_name} backend[/blue]" + ) + backend_name = click.prompt("Backend name", type=str) + config_instance = self.backend.prompt_for_config(backend_name) + config_instance.name = backend_name + self.backend.add_backend( + deployment, backend_name, config_instance, console + ) + return + + # Non-interactive path: merge file and CLI, validate + merged = {**file_cfg, **provided_cli_values} + try: + config_instance = self.backend.config_class(**merged) + except pydantic.ValidationError as e: + console.print("[red]Configuration validation error:[/red]") + for error in e.errors(): + field_name = error.get("loc", ["unknown"])[0] + # Convert field name to CLI parameter format + cli_param = f"--{field_name.replace('_', '-')}" + console.print(f" {cli_param}: {error['msg']}") + raise click.Abort() + backend_name = merged.get("name") + if not backend_name: + raise click.BadParameter( + "--name is required when not running interactively" + ) + self.backend.add_backend(deployment, backend_name, config_instance, console) + + # Build command dynamically with parameters + params = _build_params(required_all=False) + help_text = ( + f"Add {self.backend.display_name} backend.\n\n" + "Behavior:\n" + "- If no options are provided, runs in interactive mode and prompts " + "for all required fields.\n" + "- If options and/or --config-file are provided, runs non-" + "interactively and validates against the model.\n" + "- In non-interactive mode, --name is required (or supplied via " + "--config-file).\n\n" + "Examples:\n" + " sunbeam storage add hitachi\n" + " sunbeam storage add hitachi --config-file hitachi.yaml\n" + " sunbeam storage add hitachi --name myhitachi --san-ip 10.0.0.10 " + "--protocol FC\n" + ) + cmd = click.Command( + name=self.backend.name, + params=params, + callback=add_callback, + help=help_text, + ) + add.add_command(cmd) + + def register_cli( # noqa: C901 + self, + remove: click.Group, + config_show: click.Group, + config_set: click.Group, + config_options: click.Group, + deployment: Deployment, + ) -> None: + """Register management commands for Hitachi backend.""" + + @click.command(name=self.backend.name) + @click.argument("backend_name", type=str) + @click.option("--yes", "-y", is_flag=True, help="Skip confirmation prompt") + @click.pass_context + def remove_hitachi(ctx, backend_name: str, yes: bool): + service = self.backend._get_service(deployment) + if not service.backend_exists(backend_name, self.backend.name): + console.print(f"[red]Error: Backend '{backend_name}' not found[/red]") + raise click.Abort() + if not yes: + click.confirm( + f"Remove {self.backend.display_name} backend '{backend_name}'?", + abort=True, + ) + try: + self.backend.remove_backend(deployment, backend_name, console) + except Exception as e: + console.print(f"[red]Error removing backend: {e}[/red]") + raise click.Abort() + + remove.add_command(remove_hitachi) + + @click.command(name=self.backend.name) + @click.argument("backend_name", type=str) + @click.pass_context + def config_show_hitachi(ctx, backend_name: str): + service = self.backend._get_service(deployment) + config = service.get_backend_config(backend_name, self.backend.name) + self.backend.display_config_table(backend_name, config) + + config_show.add_command(config_show_hitachi) + + # Build typed options for config set (only provided options are updated) + def _build_set_params() -> list: + params = [] + # Add config-file option first + params.append( + click.Option( + ["--config-file"], + type=click.Path( + exists=True, dir_okay=False, readable=True, path_type=Path + ), + required=False, + help="YAML/JSON config file with updates", + ) + ) + fields = getattr(self.backend.config_class, "model_fields", {}) + for fname, finfo in fields.items(): + if fname == "name": + continue + opt = "--" + fname.replace("_", "-") + # For updates, make everything optional with default None + # so we can detect presence + click_type: click.ParamType = click.STRING + ann = getattr(finfo, "annotation", None) + if ann is not None and ("bool" in str(ann)): + click_type = click.BOOL + elif ann is not None and ("int" in str(ann)): + click_type = click.INT + elif ann is not None and ("float" in str(ann)): + click_type = click.FLOAT + descr = None + if hasattr(finfo, "field_info") and hasattr( + finfo.field_info, "description" + ): + descr = finfo.field_info.description + elif hasattr(finfo, "description"): + descr = finfo.description + params.append( + click.Option( + [opt], type=click_type, required=False, default=None, help=descr + ) + ) + return params + + def set_callback(backend_name: str, **kwargs): + cfg_file = kwargs.pop("config_file", None) + file_cfg = {} + if cfg_file: + file_cfg = self._load_config_file(cfg_file) + # Only include keys that were explicitly provided (value not None) + updates = {k: v for k, v in kwargs.items() if v is not None} + updates = {**file_cfg, **updates} + try: + # Get current configuration and merge with updates for validation + service = StorageBackendService(deployment) + current_config = service.get_backend_config( + backend_name, self.backend.name + ) + + # Create a merged config for validation + # Current config comes from charm (kebab-case keys), + # convert to snake_case + merged_config = {} + for key, value in current_config.items(): + snake_key = key.replace("-", "_") + merged_config[snake_key] = value + + # Updates from CLI are already in snake_case + # (Click converts --protocol to protocol) + merged_config.update(updates) + merged_config["name"] = backend_name # Ensure name is set + _ = self.backend.config_class(**merged_config) + except pydantic.ValidationError as e: + console.print("[red]Configuration validation error:[/red]") + for error in e.errors(): + field_name = error["loc"][0] if error.get("loc") else "unknown" + # Convert field name to CLI parameter format + cli_param = f"--{str(field_name).replace('_', '-')}" + console.print(f" {cli_param}: {error['msg']}") + raise click.ClickException( + "Configuration update failed due to validation error" + ) + try: + self.backend.update_backend_config(deployment, backend_name, updates) + console.print( + ( + f"[green]Configuration updated for {self.backend.display_name} " + f"backend '{backend_name}'[/green]" + ) + ) + except Exception as e: + console.print(f"[red]Failed to update configuration: {e}[/red]") + raise click.ClickException(f"Configuration update failed: {e}") + + set_params = _build_set_params() + # Add backend_name as a required argument + set_params.insert(0, click.Argument(["backend_name"], type=str, required=True)) + set_cmd = click.Command( + name=self.backend.name, + params=set_params, + callback=set_callback, + help="Set configuration options", + ) + config_set.add_command(set_cmd) + + @click.command(name=self.backend.name) + @click.argument("backend_name", type=str, required=False) + @click.pass_context + def config_options_hitachi(ctx, backend_name: str | None = None): + self.backend.display_config_options() + + config_options.add_command(config_options_hitachi) diff --git a/sunbeam-python/sunbeam/storage/backends/hitachi/deploy-hitachi-backend/main.tf b/sunbeam-python/sunbeam/storage/backends/hitachi/deploy-hitachi-backend/main.tf new file mode 100644 index 000000000..ee0e566d3 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/hitachi/deploy-hitachi-backend/main.tf @@ -0,0 +1,222 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +terraform { + required_providers { + juju = { + source = "juju/juju" + version = "= 0.20.0" + } + } +} + +provider "juju" {} + +data "juju_model" "machine_model" { + name = var.machine_model +} + +# Create Juju secrets for Hitachi backend credentials +# Main array credentials (required for all backends) +resource "juju_secret" "hitachi_san_credentials" { + for_each = { + for backend_name, backend_config in var.hitachi_backends : + backend_name => backend_config + if backend_config.san_username != "" && backend_config.san_password != "" + } + + model = data.juju_model.machine_model.name + name = "${each.key}-san-credentials" + value = { + username = each.value.san_username + password = each.value.san_password + } +} + +# CHAP credentials (optional, only for iSCSI with CHAP auth) +resource "juju_secret" "hitachi_chap_credentials" { + for_each = { + for backend_name, backend_config in var.hitachi_backends : + backend_name => backend_config + if backend_config.use_chap_auth == true && backend_config.chap_username != "" && backend_config.chap_password != "" + } + + model = data.juju_model.machine_model.name + name = "${each.key}-chap-credentials" + value = { + username = each.value.chap_username + password = each.value.chap_password + } +} + +# Mirror CHAP credentials (optional, for GAD replication) +resource "juju_secret" "hitachi_mirror_chap_credentials" { + for_each = { + for backend_name, backend_config in var.hitachi_backends : + backend_name => backend_config + if backend_config.hitachi_mirror_chap_username != "" && backend_config.hitachi_mirror_chap_password != "" + } + + model = data.juju_model.machine_model.name + name = "${each.key}-mirror-chap-credentials" + value = { + username = each.value.hitachi_mirror_chap_username + password = each.value.hitachi_mirror_chap_password + } +} + +# Mirror REST API credentials (optional, for GAD replication) +resource "juju_secret" "hitachi_mirror_rest_credentials" { + for_each = { + for backend_name, backend_config in var.hitachi_backends : + backend_name => backend_config + if backend_config.hitachi_mirror_rest_username != "" && backend_config.hitachi_mirror_rest_password != "" + } + + model = data.juju_model.machine_model.name + name = "${each.key}-mirror-rest-credentials" + value = { + username = each.value.hitachi_mirror_rest_username + password = each.value.hitachi_mirror_rest_password + } +} + +# Grant access to secrets for Hitachi backend applications +resource "juju_access_secret" "hitachi_san_credentials_access" { + for_each = { + for backend_name, backend_config in var.hitachi_backends : + backend_name => backend_config + if backend_config.san_username != "" && backend_config.san_password != "" + } + + model = data.juju_model.machine_model.name + secret_id = juju_secret.hitachi_san_credentials[each.key].secret_id + applications = [each.key] + + # Ensure proper dependency ordering to avoid provider bugs + depends_on = [juju_application.hitachi_backends] + + lifecycle { + # Prevent destruction when applications list becomes empty + prevent_destroy = false + # Create before destroy to avoid empty applications list state + create_before_destroy = true + } +} + +resource "juju_access_secret" "hitachi_chap_credentials_access" { + for_each = { + for backend_name, backend_config in var.hitachi_backends : + backend_name => backend_config + if backend_config.use_chap_auth == true && backend_config.chap_username != "" && backend_config.chap_password != "" + } + + model = data.juju_model.machine_model.name + secret_id = juju_secret.hitachi_chap_credentials[each.key].secret_id + applications = [each.key] + + # Ensure proper dependency ordering to avoid provider bugs + depends_on = [juju_application.hitachi_backends] + + lifecycle { + # Prevent destruction when applications list becomes empty + prevent_destroy = false + # Create before destroy to avoid empty applications list state + create_before_destroy = true + } +} + +resource "juju_access_secret" "hitachi_mirror_chap_credentials_access" { + for_each = { + for backend_name, backend_config in var.hitachi_backends : + backend_name => backend_config + if backend_config.hitachi_mirror_chap_username != "" && backend_config.hitachi_mirror_chap_password != "" + } + + model = data.juju_model.machine_model.name + secret_id = juju_secret.hitachi_mirror_chap_credentials[each.key].secret_id + applications = [each.key] + + # Ensure proper dependency ordering to avoid provider bugs + depends_on = [juju_application.hitachi_backends] + + lifecycle { + # Prevent destruction when applications list becomes empty + prevent_destroy = false + # Create before destroy to avoid empty applications list state + create_before_destroy = true + } +} + +resource "juju_access_secret" "hitachi_mirror_rest_credentials_access" { + for_each = { + for backend_name, backend_config in var.hitachi_backends : + backend_name => backend_config + if backend_config.hitachi_mirror_rest_username != "" && backend_config.hitachi_mirror_rest_password != "" + } + + model = data.juju_model.machine_model.name + secret_id = juju_secret.hitachi_mirror_rest_credentials[each.key].secret_id + applications = [each.key] + + # Ensure proper dependency ordering to avoid provider bugs + depends_on = [juju_application.hitachi_backends] + + lifecycle { + # Prevent destruction when applications list becomes empty + prevent_destroy = false + # Create before destroy to avoid empty applications list state + create_before_destroy = true + } +} + +# Deploy Hitachi storage backend charms +resource "juju_application" "hitachi_backends" { + for_each = var.hitachi_backends + + name = each.key + model = data.juju_model.machine_model.name + units = 1 + + charm { + name = var.charm_hitachi_name + channel = var.charm_hitachi_channel + revision = var.charm_hitachi_revision + base = var.charm_hitachi_base + } + + config = merge({ + volume-backend-name = each.key + }, each.value.charm_config, { + # Main array credentials - always required + san-credentials-secret = contains(keys(juju_secret.hitachi_san_credentials), each.key) ? juju_secret.hitachi_san_credentials[each.key].secret_id : "" + + # CHAP credentials - only if CHAP auth is enabled and credentials provided + chap-credentials-secret = contains(keys(juju_secret.hitachi_chap_credentials), each.key) ? juju_secret.hitachi_chap_credentials[each.key].secret_id : "" + + # Mirror CHAP credentials - only if mirror CHAP credentials provided + hitachi-mirror-chap-credentials-secret = contains(keys(juju_secret.hitachi_mirror_chap_credentials), each.key) ? juju_secret.hitachi_mirror_chap_credentials[each.key].secret_id : "" + + # Mirror REST API credentials - only if mirror REST credentials provided + hitachi-mirror-rest-credentials-secret = contains(keys(juju_secret.hitachi_mirror_rest_credentials), each.key) ? juju_secret.hitachi_mirror_rest_credentials[each.key].secret_id : "" + }) + + endpoint_bindings = var.endpoint_bindings +} + +# Integrate Hitachi backends with main cinder-volume +resource "juju_integration" "hitachi_to_cinder_volume" { + for_each = var.hitachi_backends + + model = var.machine_model + + application { + name = juju_application.hitachi_backends[each.key].name + endpoint = var.charm_hitachi_endpoint + } + + application { + name = "cinder-volume" + endpoint = var.charm_hitachi_endpoint + } +} diff --git a/sunbeam-python/sunbeam/storage/backends/hitachi/deploy-hitachi-backend/outputs.tf b/sunbeam-python/sunbeam/storage/backends/hitachi/deploy-hitachi-backend/outputs.tf new file mode 100644 index 000000000..493c8151b --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/hitachi/deploy-hitachi-backend/outputs.tf @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +output "hitachi_backend_applications" { + description = "Map of deployed Hitachi backend applications" + value = { + for name, app in juju_application.hitachi_backends : name => { + name = app.name + model = app.model + units = app.units + } + } +} diff --git a/sunbeam-python/sunbeam/storage/backends/hitachi/deploy-hitachi-backend/test-variables.tfvars b/sunbeam-python/sunbeam/storage/backends/hitachi/deploy-hitachi-backend/test-variables.tfvars new file mode 100644 index 000000000..7a07fb91a --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/hitachi/deploy-hitachi-backend/test-variables.tfvars @@ -0,0 +1,35 @@ +machine_model = "openstack" +charm_hitachi_channel = "latest/edge" +charm_hitachi_revision = null + +hitachi_backends = { + "hitachi-backend-1" = { + charm_config = { + "hitachi-storage-id" = "123456" + "hitachi-pools" = "pool1,pool2" + "san-ip" = "192.168.1.100" + "protocol" = "FC" + "volume-backend-name" = "hitachi-backend-1" + } + + # Main array credentials (always required) + san_username = "maintenance" + san_password = "secret123" + + # CHAP credentials (optional - using iSCSI example) + use_chap_auth = true + chap_username = "iscsi_user" + chap_password = "iscsi_pass" + + # Mirror CHAP credentials (optional) + hitachi_mirror_chap_username = "mirror_chap_user" + hitachi_mirror_chap_password = "mirror_chap_pass" + + # Mirror REST API credentials (optional) + hitachi_mirror_rest_username = "mirror_rest_user" + hitachi_mirror_rest_password = "mirror_rest_pass" + } +} + +machine_ids = [] +endpoint_bindings = null diff --git a/sunbeam-python/sunbeam/storage/backends/hitachi/deploy-hitachi-backend/variables.tf b/sunbeam-python/sunbeam/storage/backends/hitachi/deploy-hitachi-backend/variables.tf new file mode 100644 index 000000000..64efd6261 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/hitachi/deploy-hitachi-backend/variables.tf @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +variable "machine_model" { + description = "Name of the machine model to deploy to" + type = string +} + +variable "charm_hitachi_name" { + description = "Name of the Hitachi charm" + type = string + default = "cinder-volume-hitachi" +} + +variable "charm_hitachi_base" { + description = "Base for the Hitachi charm" + type = string + default = "ubuntu@24.04" +} + +variable "charm_hitachi_channel" { + description = "Operator channel for Hitachi backend deployment" + type = string + default = "latest/edge" +} + +variable "charm_hitachi_endpoint" { + description = "Endpoint name for Hitachi backend integration" + type = string + default = "cinder-volume" +} + +variable "charm_hitachi_revision" { + description = "Operator channel revision for Hitachi backend deployment" + type = number + default = null +} + +variable "hitachi_backends" { + description = "Map of Hitachi backend configurations" + type = map(object({ + charm_config = map(string) + + # Main array credentials (required) + san_username = string + san_password = string + + # CHAP credentials (optional) + use_chap_auth = optional(bool, false) + chap_username = optional(string, "") + chap_password = optional(string, "") + + # Mirror CHAP credentials (optional) + hitachi_mirror_chap_username = optional(string, "") + hitachi_mirror_chap_password = optional(string, "") + + # Mirror REST API credentials (optional) + hitachi_mirror_rest_username = optional(string, "") + hitachi_mirror_rest_password = optional(string, "") + })) + default = {} +} + +variable "machine_ids" { + description = "List of machine ids to include" + type = list(string) + default = [] +} + +variable "endpoint_bindings" { + description = "Endpoint bindings for the applications" + type = set(map(string)) + default = null +} diff --git a/sunbeam-python/sunbeam/storage/backends/purestorage/backend.py b/sunbeam-python/sunbeam/storage/backends/purestorage/backend.py new file mode 100644 index 000000000..8c9b513c3 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/purestorage/backend.py @@ -0,0 +1,407 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Pure Storage FlashArray backend implementation using base step classes.""" + +import logging +from typing import Any, Dict, Literal, Set + +import click + +try: + import yaml as _yaml # type: ignore + + yaml: Any = _yaml +except Exception: # yaml optional; handle gracefully at runtime + yaml = None + +# Import pydantic Field directly +from pydantic import Field +from rich.console import Console + +from sunbeam.core.common import BaseStep +from sunbeam.core.deployment import Deployment +from sunbeam.core.juju import JujuHelper +from sunbeam.core.manifest import Manifest +from sunbeam.core.terraform import TerraformHelper +from sunbeam.storage.base import StorageBackendBase +from sunbeam.storage.models import StorageBackendConfig +from sunbeam.storage.steps import ( + BaseStorageBackendConfigUpdateStep, + BaseStorageBackendDeployStep, + BaseStorageBackendDestroyStep, +) + +LOG = logging.getLogger(__name__) +console = Console() + + +class PureStorageConfig(StorageBackendConfig): + """Configuration model for Pure Storage FlashArray backend. + + This model includes the essential configuration options for deploying + a Pure Storage backend. Additional configuration can be managed dynamically + through the charm configuration system. + """ + + # Required fields (inherited from StorageBackendConfig) + # name: str (from base class) + + # Mandatory connection parameters + san_ip: str = Field( + ..., description="Pure Storage FlashArray management IP or hostname" + ) + pure_api_token: str = Field( + ..., description="REST API authorization token from FlashArray" + ) + protocol: Literal["iscsi", "fc", "nvme"] = Field( + default="fc", description="Pure Storage protocol (iscsi, fc, nvme)" + ) + + # Optional backend configuration + volume_backend_name: str = Field( + default="", description="Name that Cinder will report for this backend" + ) + backend_availability_zone: str = Field( + default="", description="Availability zone to associate with this backend" + ) + + # Protocol-specific options + pure_iscsi_cidr: str = Field( + default="0.0.0.0/0", + description="CIDR of FlashArray iSCSI targets hosts can connect to", + ) + pure_iscsi_cidr_list: str = Field( + default="", description="Comma-separated list of CIDR for iSCSI targets" + ) + pure_nvme_cidr: str = Field( + default="0.0.0.0/0", + description="CIDR of FlashArray NVMe targets hosts can connect to", + ) + pure_nvme_cidr_list: str = Field( + default="", description="Comma-separated list of CIDR for NVMe targets" + ) + pure_nvme_transport: Literal["roce", "tcp"] = Field( + default="roce", description="NVMe transport layer (roce or tcp)" + ) + + # Host and protocol tuning + pure_host_personality: str = Field( + default="", description="Host personality for protocol tuning" + ) + + # Storage management + pure_automatic_max_oversubscription_ratio: bool = Field( + default=True, description="Automatically determine oversubscription ratio" + ) + pure_eradicate_on_delete: bool = Field( + default=False, + description="Immediately eradicate volumes on delete " + "(WARNING: not recoverable)", + ) + + # Replication settings + pure_replica_interval_default: int = Field( + default=3600, description="Snapshot replication interval in seconds" + ) + pure_replica_retention_short_term_default: int = Field( + default=14400, + description="Retain all snapshots on target for this time (seconds)", + ) + pure_replica_retention_long_term_per_day_default: int = Field( + default=3, description="Retain how many snapshots for each day" + ) + pure_replica_retention_long_term_default: int = Field( + default=7, description="Retain snapshots per day on target for this time (days)" + ) + pure_replication_pg_name: str = Field( + default="cinder-group", + description="Pure Protection Group name for async replication", + ) + pure_replication_pod_name: str = Field( + default="cinder-pod", description="Pure Pod name for sync replication" + ) + + # Advanced replication + pure_trisync_enabled: bool = Field( + default=False, description="Enable 3-site replication (sync + async)" + ) + pure_trisync_pg_name: str = Field( + default="cinder-trisync", + description="Protection Group name for trisync replication", + ) + + # SSL and security + driver_ssl_cert_verify: bool = Field( + default=False, description="Enable SSL certificate verification" + ) + driver_ssl_cert_path: str = Field( + default="", description="Path to SSL certificate file or directory" + ) + + # Performance options + use_multipath_for_image_xfer: bool = Field( + default=True, description="Enable multipathing for image transfer operations" + ) + + +class PureStorageBackend(StorageBackendBase): + """Pure Storage FlashArray backend implementation.""" + + name = "purestorage" + display_name = "Pure Storage FlashArray" + charm_name = "cinder-volume-purestorage" + + def __init__(self): + """Initialize Pure Storage backend.""" + super().__init__() + self.tfplan = "purestorage-backend-plan" + self.tfplan_dir = "deploy-purestorage-backend" + + charm_channel = ( + "latest/edge" # Use edge for development, change to stable for production + ) + charm_revision = None # Let Juju pick the latest + charm_base = "ubuntu@24.04" + backend_endpoint = "cinder-volume" + units = 1 + additional_integrations = [] + + @property + def config_class(self) -> type[StorageBackendConfig]: + """Return the configuration class for Pure Storage backend.""" + return PureStorageConfig + + def _get_credential_fields(self) -> Set[str]: + """Get set of credential field names that should be excluded from charm config. + + For Pure Storage backend, we only exclude the meta name field. + """ + return {"name"} # Pure Storage uses API token directly in config + + def get_field_mapping(self) -> Dict[str, str]: + """Get mapping from config fields to charm config options. + + Uses base class automatic mapping and excludes credential fields. + """ + # Start from the base automatic mapping + mapping = super().get_field_mapping() + + # Exclude credential fields + exclude = self._get_credential_fields() + return {k: v for k, v in mapping.items() if k not in exclude} + + # CLI registration uses base class implementation + + def get_terraform_variables( + self, backend_name: str, config: StorageBackendConfig, model: str + ) -> Dict[str, Any]: + """Generate Terraform variables for Pure Storage backend deployment.""" + # Map our configuration fields to the correct charm configuration option names + config_dict = config.model_dump() + field_mapping = self.get_field_mapping() + + # Filter config using base class method + charm_config = self._filter_config_for_charm( + config_dict, field_mapping, exclude_fields=self._get_credential_fields() + ) + + # Build Terraform variables to match the plan's expected format + tfvars = { + "machine_model": model, + "charm_purestorage_name": self.charm_name, + "charm_purestorage_base": self.charm_base, + "charm_purestorage_channel": self.charm_channel, + "charm_purestorage_endpoint": self.backend_endpoint, + "charm_purestorage_revision": self.charm_revision, + "purestorage_backends": { + backend_name: { + "charm_config": charm_config, + } + }, + } + + return tfvars + + def _get_default_config(self) -> PureStorageConfig: + """Get a default configuration instance for comparison.""" + return PureStorageConfig( + name="dummy", + san_ip="dummy", + pure_api_token="dummy", # noqa: S106 + ) + + def prompt_for_config(self, backend_name: str) -> PureStorageConfig: + """Prompt user for Pure Storage-specific configuration.""" + return self._prompt_for_config(backend_name) + + # IP/FQDN validation uses base class implementation + + def _prompt_for_config(self, backend_name: str) -> PureStorageConfig: + """Prompt user for Pure Storage backend configuration.""" + console.print( + "\n[bold blue]Pure Storage FlashArray Backend Configuration[/bold blue]" + ) + console.print("Please provide the required configuration options:") + + # Prompt for required fields + san_ip = click.prompt( + "FlashArray management IP/FQDN", + type=str, + value_proc=self._validate_ip_or_fqdn, + ) + pure_api_token = click.prompt("Pure API token", type=str, hide_input=True) + protocol = click.prompt( + "Protocol", + type=click.Choice(["iscsi", "fc", "nvme"], case_sensitive=False), + default="fc", + ) + + # Optional: prompt for volume backend name (defaults to backend name) + volume_backend_name = click.prompt( + "Volume backend name", type=str, default=backend_name, show_default=True + ) + + # Protocol-specific configuration + pure_iscsi_cidr = "0.0.0.0/0" + pure_nvme_cidr = "0.0.0.0/0" + pure_nvme_transport: Literal["roce", "tcp"] = "roce" + + if protocol.lower() == "iscsi": + console.print("\n[bold yellow]iSCSI Configuration[/bold yellow]") + pure_iscsi_cidr = click.prompt( + "iSCSI target CIDR", type=str, default="0.0.0.0/0", show_default=True + ) + elif protocol.lower() == "nvme": + console.print("\n[bold yellow]NVMe Configuration[/bold yellow]") + pure_nvme_cidr = click.prompt( + "NVMe target CIDR", type=str, default="0.0.0.0/0", show_default=True + ) + transport_choice = click.prompt( + "NVMe transport", + type=click.Choice(["roce", "tcp"], case_sensitive=False), + default="roce", + ) + pure_nvme_transport = transport_choice # type: ignore[assignment] + + # Optional: Host personality + pure_host_personality = "" + if click.confirm( + "\nConfigure host personality for protocol tuning?", default=False + ): + personalities = [ + "aix", + "esxi", + "hitachi-vsp", + "hpux", + "oracle-vm-server", + "solaris", + "vms", + ] + pure_host_personality = click.prompt( + "Host personality", + type=click.Choice(personalities, case_sensitive=False), + default="esxi", + ) + + # Optional: Storage management settings + pure_eradicate_on_delete = False + if click.confirm( + "\nEnable immediate volume eradication on delete?" + "(WARNING: not recoverable)", + default=False, + ): + pure_eradicate_on_delete = True + + return PureStorageConfig( + name=backend_name, + san_ip=san_ip, + pure_api_token=pure_api_token, + protocol=protocol, + volume_backend_name=volume_backend_name, + pure_iscsi_cidr=pure_iscsi_cidr, + pure_nvme_cidr=pure_nvme_cidr, + pure_nvme_transport=pure_nvme_transport, + pure_host_personality=pure_host_personality, + pure_eradicate_on_delete=pure_eradicate_on_delete, + ) + + # Implementation of abstract methods from StorageBackendBase + def create_deploy_step( + self, + deployment: Deployment, + client, + tfhelper: TerraformHelper, + jhelper: JujuHelper, + manifest: Manifest, + backend_name: str, + backend_config: StorageBackendConfig, + model: str, + ) -> BaseStep: + """Create a deployment step for Pure Storage backend.""" + return PureStorageDeployStep( + deployment, + client, + tfhelper, + jhelper, + manifest, + backend_name, + backend_config, + self, + model, + ) + + def create_destroy_step( + self, + deployment: Deployment, + client, + tfhelper: TerraformHelper, + jhelper: JujuHelper, + manifest: Manifest, + backend_name: str, + model: str, + ) -> BaseStep: + """Create a destruction step for Pure Storage backend.""" + return PureStorageDestroyStep( + deployment, + client, + tfhelper, + jhelper, + manifest, + backend_name, + self, + model, + ) + + def create_update_config_step( + self, + deployment: Deployment, + backend_name: str, + config_updates: Dict[str, Any], + ) -> BaseStep: + """Create a configuration update step for Pure Storage backend.""" + return PureStorageUpdateConfigStep( + deployment, + self, + backend_name, + config_updates, + ) + + +# Pure Storage-specific step implementations using base step classes +class PureStorageDeployStep(BaseStorageBackendDeployStep): + """Deploy Pure Storage backend using base step class.""" + + def get_terraform_variables(self) -> Dict[str, Any]: + """Get Terraform variables for Pure Storage backend deployment.""" + return self.backend_instance.get_terraform_variables( + self.backend_name, self.backend_config, self.model + ) + + +class PureStorageDestroyStep(BaseStorageBackendDestroyStep): + """Destroy Pure Storage backend using base step class.""" + + +class PureStorageUpdateConfigStep(BaseStorageBackendConfigUpdateStep): + """Update Pure Storage backend configuration using base step class.""" diff --git a/sunbeam-python/sunbeam/storage/backends/purestorage/cli.py b/sunbeam-python/sunbeam/storage/backends/purestorage/cli.py new file mode 100644 index 000000000..3fa93cd85 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/purestorage/cli.py @@ -0,0 +1,346 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""CLI functionality for Pure Storage storage backend. + +This module contains all CLI-related code following the Hitachi pattern, +including command registration and helper functions. +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import Any, Dict, Optional + +import click +import pydantic +from rich.console import Console + +try: + import yaml as _yaml # type: ignore + + yaml: Any = _yaml +except Exception: # yaml optional; handle gracefully at runtime + yaml = None + +from sunbeam.core.deployment import Deployment +from sunbeam.storage.backends.purestorage.backend import PureStorageBackend +from sunbeam.storage.service import StorageBackendService + +LOG = logging.getLogger(__name__) +console = Console() + + +class PurestorageCLI: + """CLI functionality for Pure Storage storage backend.""" + + def __init__(self, backend: PureStorageBackend): + self.backend = backend + + def _load_config_file(self, path: Optional[Path]) -> Dict[str, Any]: + """Load YAML or JSON config file into a dictionary. + + YAML is preferred if PyYAML is available, otherwise JSON is used. + """ + if not path: + return {} + text = path.read_text() + if yaml is not None: + return dict(yaml.safe_load(text) or {}) + + return dict(json.loads(text)) + + def register_add_cli(self, add: click.Group) -> None: # noqa: C901 + """Register 'sunbeam storage add purestorage'. + + Includes typed options and a --config-file flag. + """ + + def _click_type_for(field_info) -> click.types.ParamType: + # Map pydantic field to Click type + ann = getattr(field_info, "annotation", None) + typ = None + if ann is not None: + typ = str(ann) + elif hasattr(field_info, "type_"): + typ = str(field_info.type_) + if typ and ("int" in typ): + return click.INT + if typ and ("float" in typ): + return click.FLOAT + if typ and ("bool" in typ): + return click.BOOL + return click.STRING + + def _build_params(required_all: bool) -> list: + params: list = [] + # name option (not required; prompt in interactive mode) + params.append( + click.Option(["--name"], type=str, required=False, help="Backend name") + ) + # config file (optional) + params.append( + click.Option( + ["--config-file"], + type=click.Path(exists=True, dir_okay=False, path_type=Path), + required=False, + help="YAML/JSON config file", + ) + ) + # Model-derived options + fields = getattr(self.backend.config_class, "model_fields", {}) + for fname, finfo in fields.items(): + if fname == "name": + continue + opt_name = "--" + fname.replace("_", "-") + click_type = _click_type_for(finfo) + # Determine requiredness for add (respect model) + # For interactive UX, keep CLI options optional; the model + # enforces requiredness. + is_required = False + # Help text + descr = None + if hasattr(finfo, "field_info") and hasattr( + finfo.field_info, "description" + ): + descr = finfo.field_info.description + elif hasattr(finfo, "description"): + descr = finfo.description + params.append( + click.Option( + [opt_name], type=click_type, required=is_required, help=descr + ) + ) + return params + + def _build_config_from_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: + # Extract name and field values from kwargs + # (Click converts dashes to underscores) + cfg: Dict[str, Any] = {} + for k, v in kwargs.items(): + if v is None: + continue + cfg[k] = v + return cfg + + def add_callback(**kwargs): + deployment: Deployment = click.get_current_context().obj + cfg_file = kwargs.pop("config_file", None) + file_cfg = self._load_config_file(cfg_file) + cli_cfg = _build_config_from_kwargs(kwargs) + # Determine if interactive: no config-file and no CLI options supplied + provided_cli_values = {k: v for k, v in cli_cfg.items() if v is not None} + interactive = not file_cfg and not provided_cli_values + + if interactive: + # Prompt for name and full config via helper + console.print( + f"[blue]Setting up {self.backend.display_name} backend[/blue]" + ) + backend_name = click.prompt("Backend name", type=str) + config_instance = self.backend.prompt_for_config(backend_name) + config_instance.name = backend_name + self.backend.add_backend( + deployment, backend_name, config_instance, console + ) + return + + # Non-interactive path: merge file and CLI, validate + merged = {**file_cfg, **provided_cli_values} + try: + config_instance = self.backend.config_class(**merged) + except pydantic.ValidationError as e: + console.print("[red]Configuration validation error:[/red]") + for error in e.errors(): + field_name = error.get("loc", ["unknown"])[0] + # Convert field name to CLI parameter format + cli_param = f"--{field_name.replace('_', '-')}" + console.print(f" {cli_param}: {error['msg']}") + raise click.Abort() + backend_name = merged.get("name") + if not backend_name: + raise click.BadParameter( + "--name is required when not running interactively" + ) + self.backend.add_backend(deployment, backend_name, config_instance, console) + + # Build command dynamically with parameters + params = _build_params(required_all=False) + help_text = ( + f"Add {self.backend.display_name} backend.\n\n" + "Behavior:\n" + "- If no options are provided, runs in interactive mode and prompts " + "for all required fields.\n" + "- If options and/or --config-file are provided, runs non-" + "interactively and validates against the model.\n" + "- In non-interactive mode, --name is required (or supplied via " + "--config-file).\n\n" + "Examples:\n" + " sunbeam storage add purestorage\n" + " sunbeam storage add purestorage --config-file purestorage.yaml\n" + " sunbeam storage add purestorage --name mypure --san-ip 10.0.0.10 " + "--pure-api-token mytoken\n" + ) + cmd = click.Command( + name=self.backend.name, + params=params, + callback=add_callback, + help=help_text, + ) + add.add_command(cmd) + + def register_cli( # noqa: C901 + self, + remove: click.Group, + config_show: click.Group, + config_set: click.Group, + config_options: click.Group, + deployment: Deployment, + ) -> None: + """Register management commands for Pure Storage backend.""" + + @click.command(name=self.backend.name) + @click.argument("backend_name", type=str) + @click.option("--yes", "-y", is_flag=True, help="Skip confirmation prompt") + @click.pass_context + def remove_purestorage(ctx, backend_name: str, yes: bool): + service = self.backend._get_service(deployment) + if not service.backend_exists(backend_name, self.backend.name): + console.print(f"[red]Error: Backend '{backend_name}' not found[/red]") + raise click.Abort() + if not yes: + click.confirm( + f"Remove {self.backend.display_name} backend '{backend_name}'?", + abort=True, + ) + try: + self.backend.remove_backend(deployment, backend_name, console) + except Exception as e: + console.print(f"[red]Error removing backend: {e}[/red]") + raise click.Abort() + + remove.add_command(remove_purestorage) + + @click.command(name=self.backend.name) + @click.argument("backend_name", type=str) + @click.pass_context + def config_show_purestorage(ctx, backend_name: str): + service = self.backend._get_service(deployment) + config = service.get_backend_config(backend_name, self.backend.name) + self.backend.display_config_table(backend_name, config) + + config_show.add_command(config_show_purestorage) + + # Build typed options for config set (only provided options are updated) + def _build_set_params() -> list: + params = [] + # Add config-file option first + params.append( + click.Option( + ["--config-file"], + type=click.Path( + exists=True, dir_okay=False, readable=True, path_type=Path + ), + required=False, + help="YAML/JSON config file with updates", + ) + ) + fields = getattr(self.backend.config_class, "model_fields", {}) + for fname, finfo in fields.items(): + if fname == "name": + continue + opt = "--" + fname.replace("_", "-") + # For updates, make everything optional with default None + # so we can detect presence + click_type: click.ParamType = click.STRING + ann = getattr(finfo, "annotation", None) + if ann is not None and ("bool" in str(ann)): + click_type = click.BOOL + elif ann is not None and ("int" in str(ann)): + click_type = click.INT + elif ann is not None and ("float" in str(ann)): + click_type = click.FLOAT + descr = None + if hasattr(finfo, "field_info") and hasattr( + finfo.field_info, "description" + ): + descr = finfo.field_info.description + elif hasattr(finfo, "description"): + descr = finfo.description + params.append( + click.Option( + [opt], type=click_type, required=False, default=None, help=descr + ) + ) + return params + + def set_callback(backend_name: str, **kwargs): + cfg_file = kwargs.pop("config_file", None) + file_cfg = {} + if cfg_file: + file_cfg = self._load_config_file(cfg_file) + # Only include keys that were explicitly provided (value not None) + updates = {k: v for k, v in kwargs.items() if v is not None} + updates = {**file_cfg, **updates} + try: + # Get current configuration and merge with updates for validation + service = StorageBackendService(deployment) + current_config = service.get_backend_config( + backend_name, self.backend.name + ) + + # Create a merged config for validation + # Current config comes from charm (kebab-case keys), + # convert to snake_case + merged_config = {} + for key, value in current_config.items(): + snake_key = key.replace("-", "_") + merged_config[snake_key] = value + + # Updates from CLI are already in snake_case + # (Click converts --protocol to protocol) + merged_config.update(updates) + merged_config["name"] = backend_name # Ensure name is set + _ = self.backend.config_class(**merged_config) + except pydantic.ValidationError as e: + console.print("[red]Configuration validation error:[/red]") + for error in e.errors(): + field_name = error["loc"][0] if error.get("loc") else "unknown" + # Convert field name to CLI parameter format + cli_param = f"--{str(field_name).replace('_', '-')}" + console.print(f" {cli_param}: {error['msg']}") + raise click.ClickException( + "Configuration update failed due to validation error" + ) + try: + self.backend.update_backend_config(deployment, backend_name, updates) + console.print( + ( + f"[green]Configuration updated for {self.backend.display_name} " + f"backend '{backend_name}'[/green]" + ) + ) + except Exception as e: + console.print(f"[red]Failed to update configuration: {e}[/red]") + raise click.ClickException(f"Configuration update failed: {e}") + + set_params = _build_set_params() + # Add backend_name as a required argument + set_params.insert(0, click.Argument(["backend_name"], type=str, required=True)) + set_cmd = click.Command( + name=self.backend.name, + params=set_params, + callback=set_callback, + help="Set configuration options", + ) + config_set.add_command(set_cmd) + + @click.command(name=self.backend.name) + @click.pass_context + def config_options_purestorage(ctx): + self.backend.display_config_options() + + + config_options.add_command(config_options_purestorage) diff --git a/sunbeam-python/sunbeam/storage/backends/purestorage/deploy-purestorage-backend/main.tf b/sunbeam-python/sunbeam/storage/backends/purestorage/deploy-purestorage-backend/main.tf new file mode 100644 index 000000000..7913939ef --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/purestorage/deploy-purestorage-backend/main.tf @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +terraform { + required_providers { + juju = { + source = "juju/juju" + version = "= 0.20.0" + } + } +} + +provider "juju" {} + +data "juju_model" "machine_model" { + name = var.machine_model +} + +# Deploy Pure Storage backend charms +resource "juju_application" "purestorage_backends" { + for_each = var.purestorage_backends + + name = each.key + model = data.juju_model.machine_model.name + units = 1 + + charm { + name = var.charm_purestorage_name + channel = var.charm_purestorage_channel + revision = var.charm_purestorage_revision + base = var.charm_purestorage_base + } + + config = merge({ + volume-backend-name = each.key + }, each.value.charm_config) + + endpoint_bindings = var.endpoint_bindings +} + +# Integrate Pure Storage backends with main cinder-volume +resource "juju_integration" "purestorage_to_cinder_volume" { + for_each = var.purestorage_backends + + model = var.machine_model + + application { + name = juju_application.purestorage_backends[each.key].name + endpoint = var.charm_purestorage_endpoint + } + + application { + name = "cinder-volume" + endpoint = var.charm_purestorage_endpoint + } +} diff --git a/sunbeam-python/sunbeam/storage/backends/purestorage/deploy-purestorage-backend/outputs.tf b/sunbeam-python/sunbeam/storage/backends/purestorage/deploy-purestorage-backend/outputs.tf new file mode 100644 index 000000000..e648676da --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/purestorage/deploy-purestorage-backend/outputs.tf @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +output "purestorage_backend_applications" { + description = "Map of deployed Pure Storage backend applications" + value = { + for name, app in juju_application.purestorage_backends : name => { + name = app.name + model = app.model + units = app.units + } + } +} diff --git a/sunbeam-python/sunbeam/storage/backends/purestorage/deploy-purestorage-backend/variables.tf b/sunbeam-python/sunbeam/storage/backends/purestorage/deploy-purestorage-backend/variables.tf new file mode 100644 index 000000000..6c6bacf6f --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/purestorage/deploy-purestorage-backend/variables.tf @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +variable "machine_model" { + description = "Name of the machine model to deploy to" + type = string +} + +variable "charm_purestorage_name" { + description = "Name of the Pure Storage charm" + type = string + default = "cinder-volume-purestorage" +} + +variable "charm_purestorage_base" { + description = "Base for the Pure Storage charm" + type = string + default = "ubuntu@24.04" +} + +variable "charm_purestorage_channel" { + description = "Operator channel for Pure Storage backend deployment" + type = string + default = "latest/edge" +} + +variable "charm_purestorage_endpoint" { + description = "Endpoint name for Pure Storage backend integration" + type = string + default = "cinder-volume" +} + +variable "charm_purestorage_revision" { + description = "Operator channel revision for Pure Storage backend deployment" + type = number + default = null +} + +variable "purestorage_backends" { + description = "Map of Pure Storage backend configurations" + type = map(object({ + charm_config = map(string) + })) + default = {} +} + +variable "machine_ids" { + description = "List of machine ids to include" + type = list(string) + default = [] +} + +variable "endpoint_bindings" { + description = "Endpoint bindings for the applications" + type = set(map(string)) + default = null +} + diff --git a/sunbeam-python/sunbeam/storage/base.py b/sunbeam-python/sunbeam/storage/base.py new file mode 100644 index 000000000..7e94b7dbc --- /dev/null +++ b/sunbeam-python/sunbeam/storage/base.py @@ -0,0 +1,670 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Storage backend base class with integrated Terraform functionality.""" + +import ipaddress +import logging +import re +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Any, Dict, List, Optional, Set + +import click +from packaging.version import Version +from rich.console import Console +from rich.table import Table + +from sunbeam.core.common import BaseStep, run_plan +from sunbeam.core.deployment import Deployment +from sunbeam.core.juju import JujuHelper +from sunbeam.core.manifest import Manifest +from sunbeam.core.terraform import TerraformHelper, TerraformInitStep + +from .models import ( + BackendAlreadyExistsException, + BackendNotFoundException, + StorageBackendConfig, +) +from .service import StorageBackendService +from .steps import ValidateStoragePrerequisitesStep + +LOG = logging.getLogger(__name__) +console = Console() + +# Juju application name validation pattern +# Based on Juju's naming rules: must start with letter, contain only +# letters, numbers, hyphens. Cannot end with hyphen, cannot have +# consecutive hyphens, cannot have numbers after final hyphen +JUJU_APP_NAME_PATTERN = re.compile(r"^[a-z]([a-z0-9]*(-[a-z0-9]*)*)?$") + +# Regex pattern for validating FQDN (Fully Qualified Domain Name) +FQDN_PATTERN = ( + r"^[a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?" + r"(\.[a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?)*$" +) + + +def validate_juju_application_name(name: str) -> bool: + """Validate that a name is a valid Juju application name. + + Args: + name: The application name to validate + + Returns: + True if valid, False otherwise + """ + if not name: + return False + + # Check basic pattern + if not JUJU_APP_NAME_PATTERN.match(name): + return False + + # Additional checks for edge cases + if name.endswith("-"): + return False + + if "--" in name: + return False + + # Check that numbers don't appear after the final hyphen + if "-" in name: + parts = name.split("-") + last_part = parts[-1] + if any(char.isdigit() for char in last_part): + return False + + return True + + +class StorageBackendBase(ABC): + """Base class for storage backends with integrated Terraform functionality.""" + + name: str = "base" + display_name: str = "Base Storage Backend" + version = Version("0.0.1") + tf_plan_location = "FEATURE_REPO" # Plans stored in feature directory + user_manifest = None # Path to user manifest file + + def __init__(self): + """Initialize storage backend.""" + self.tfplan = "storage-backend-plan" + self.tfplan_dir = "deploy-storage-backend" + self._manifest: Optional[Manifest] = None + self.service: Optional[StorageBackendService] = None + + def _get_service(self, deployment: Deployment) -> StorageBackendService: + """Get or create the storage backend service.""" + if self.service is None: + self.service = StorageBackendService(deployment) + return self.service + + # Common CLI registration pattern (Abstraction 3: CLI registration) + def register_add_cli(self, add: click.Group) -> None: # noqa: F811 + """Register 'sunbeam storage add ' command. + + Default implementation delegates to CLI class following the pattern. + Subclasses can override if they need custom behavior. + """ + cli_class = self._get_cli_class() + cli = cli_class(self) + cli.register_add_cli(add) + + def register_cli( # noqa: F811 + self, + remove: click.Group, + config_show: click.Group, + config_set: click.Group, + config_options: click.Group, + deployment: Deployment, + ) -> None: + """Register management commands for this backend. + + Default implementation delegates to CLI class following the pattern. + Subclasses can override if they need custom behavior. + """ + cli_class = self._get_cli_class() + cli = cli_class(self) + cli.register_cli(remove, config_show, config_set, config_options, deployment) + + # Terraform-related properties and methods + @property + def manifest(self) -> Manifest: + """Return the manifest.""" + if self._manifest: + return self._manifest + + manifest = click.get_current_context().obj.get_manifest(self.user_manifest) + self._manifest = manifest + if self._manifest is None: + raise ValueError("Failed to load manifest") + return self._manifest + + @property + def tfvar_config_key(self) -> str: + """Config key for storing Terraform variables in clusterd.""" + return f"TerraformVarsStorageBackends{self.name.title()}" + + # Abstract methods that each backend must implement + @abstractmethod + def create_deploy_step( + self, + deployment: Deployment, + client, + tfhelper: TerraformHelper, + jhelper: JujuHelper, + manifest: Manifest, + backend_name: str, + backend_config: StorageBackendConfig, + model: str, + ) -> BaseStep: + """Create a deployment step for this backend.""" + pass + + @abstractmethod + def create_destroy_step( + self, + deployment: Deployment, + client, + tfhelper: TerraformHelper, + jhelper: JujuHelper, + manifest: Manifest, + backend_name: str, + model: str, + ) -> BaseStep: + """Create a destruction step for this backend.""" + pass + + @abstractmethod + def create_update_config_step( + self, + deployment: Deployment, + backend_name: str, + config_updates: Dict[str, Any], + ) -> BaseStep: + """Create a configuration update step for this backend.""" + pass + + def register_terraform_plan(self, deployment: Deployment) -> None: + """Register storage backend Terraform plan with deployment system.""" + import shutil + + from sunbeam.core.terraform import TerraformHelper + + # Get the plan source path + backend_self_contained = ( + Path(__file__).parent / "backends" / self.name / self.tfplan_dir + ) + + if backend_self_contained.exists(): + plan_source = backend_self_contained + else: + raise FileNotFoundError( + f"Terraform plan not found at {backend_self_contained}" + ) + + # Copy plan to deployment's plans directory + dst = deployment.plans_directory / self.tfplan_dir + shutil.copytree(plan_source, dst, dirs_exist_ok=True) + + # Create TerraformHelper + env = {} + env.update(deployment._get_juju_clusterd_env()) + env.update(deployment.get_proxy_settings()) + + tfhelper = TerraformHelper( + path=dst, + plan=self.tfplan, + tfvar_map={}, + backend="http", + env=env, + clusterd_address=deployment.get_clusterd_http_address(), + ) + + # Register the helper with the deployment's tfhelpers + deployment._tfhelpers[self.tfplan] = tfhelper + + def add_backend( + self, + deployment: Deployment, + backend_name: str, + config: StorageBackendConfig, + console: Console, + ) -> None: + """Add a storage backend using Terraform deployment.""" + # Validate backend name follows Juju application naming rules + if not validate_juju_application_name(backend_name): + raise click.ClickException( + f"Invalid backend name '{backend_name}'. " + f"Backend names must be valid Juju application names: " + f"start with a letter, contain only lowercase letters, numbers," + f"and hyphens, cannot end with hyphen, cannot" + f"have consecutive hyphens, and cannot have numbers" + f"after the final hyphen." + ) + + service = self._get_service(deployment) + if service.backend_exists(backend_name, self.name): + raise BackendAlreadyExistsException( + f"Backend '{backend_name}' already exists" + ) + + # Register our Terraform plan with the deployment system + self.register_terraform_plan(deployment) + + # Get standard Sunbeam helpers + client = deployment.get_client() + tfhelper = deployment.get_tfhelper(self.tfplan) + jhelper = JujuHelper(deployment.juju_controller) + + plan = [ + ValidateStoragePrerequisitesStep(deployment, client, jhelper), + TerraformInitStep(tfhelper), + self.create_deploy_step( + deployment, + client, + tfhelper, + jhelper, + self.manifest, + backend_name, + config, + deployment.openstack_machines_model, + ), + ] + + run_plan(plan, console) + + def _get_field_descriptions(self, config_class) -> dict: + """Extract field descriptions from a Pydantic v2 model class.""" + desc: Dict[str, str] = {} + if hasattr(config_class, "model_fields"): + for field_name, field_info in config_class.model_fields.items(): + desc[field_name] = getattr( + getattr(field_info, "field_info", field_info), + "description", + "No description available", + ) + return desc + + def _format_config_value(self, key: str, value) -> str: + """Format configuration value for display, masking sensitive data.""" + display_value = str(value) + if any(s in key.lower() for s in ["password", "secret", "token", "key"]): + display_value = "*" * min(8, len(display_value)) if display_value else "" + if len(display_value) > 23: + display_value = display_value[:20] + "..." + return display_value + + def _extract_field_info(self, field_info) -> tuple: + """Extract field type, default value, and description from field info.""" + if hasattr(field_info, "type_"): + field_type = str(field_info.type_).replace("", "") + elif hasattr(field_info, "annotation"): + field_type = ( + str(field_info.annotation).replace("", "") + ) + else: + field_type = "str" + + if hasattr(field_info, "default"): + default_value = ( + str(field_info.default) if field_info.default is not ... else "Required" + ) + else: + default_value = "Unknown" + + if hasattr(field_info, "field_info") and hasattr( + field_info.field_info, "description" + ): + description = field_info.field_info.description or "No description" + elif hasattr(field_info, "description"): + description = field_info.description or "No description" + else: + description = "No description" + + return field_type, default_value, description + + def display_config_options(self) -> None: + """Display available configuration options for this backend.""" + console.print( + f"[blue]Available configuration options for {self.display_name}:[/blue]" + ) + config_class = self.config_class + fields = getattr(config_class, "model_fields", {}) + if not fields: + console.print( + " Configuration options are managed dynamically via Terraform." + ) + console.print( + " Use 'sunbeam storage config show' to see current configuration." + ) + return + + table = Table(show_header=True, header_style="bold blue") + table.add_column("Option", style="cyan") + table.add_column("Type", style="green") + table.add_column("Default", style="yellow") + table.add_column("Description", style="white") + + for field_name, finfo in fields.items(): + if field_name == "name": + continue + try: + ftype, default, descr = self._extract_field_info(finfo) + table.add_row(field_name, ftype, default, descr) + except Exception: + table.add_row(field_name, "str", "Unknown", "Configuration option") + + console.print(table) + + def display_config_table(self, backend_name: str, config: dict) -> None: + """Display current configuration in a formatted table for this backend.""" + table = Table( + title=f"Configuration for {self.display_name} backend '{backend_name}'", + show_header=True, + header_style="bold blue", + title_style="bold cyan", + border_style="blue", + ) + + table.add_column("Option", style="cyan", no_wrap=True, width=30) + table.add_column("Value", style="green", width=25) + table.add_column("Description", style="dim", width=50) + + field_descriptions = self._get_field_descriptions(self.config_class) + for key, value in sorted(config.items()): + # Skip empty values (None, empty string, empty dict, empty list) + # But keep 0 and False as valid values + if ( + value is None + or value == "" + or (isinstance(value, (dict, list)) and len(value) == 0) + ): + continue + + display_value = self._format_config_value(key, value) + description = field_descriptions.get(key, "Configuration option") + if len(description) > 47: + description = description[:44] + "..." + table.add_row(key, display_value, description) + + if not config: + console.print( + ( + f"[yellow]No configuration found for {self.name} " + f"backend '{backend_name}'[/yellow]" + ) + ) + else: + console.print(table) + console.print( + ( + f"[green]Configuration displayed for {self.display_name} " + f"backend '{backend_name}'[/green]" + ) + ) + + def remove_backend( + self, deployment: Deployment, backend_name: str, console: Console + ) -> None: + """Remove a storage backend using Terraform.""" + service = self._get_service(deployment) + if not service.backend_exists(backend_name, self.name): + raise BackendNotFoundException(f"Backend '{backend_name}' not found") + + # Register our Terraform plan with the deployment system + self.register_terraform_plan(deployment) + + # Get standard Sunbeam helpers + client = deployment.get_client() + tfhelper = deployment.get_tfhelper(self.tfplan) + jhelper = JujuHelper(deployment.juju_controller) + + # Create removal plan - each backend should implement its own destroy step + plan = [ + ValidateStoragePrerequisitesStep(deployment, client, jhelper), + TerraformInitStep(tfhelper), + self.create_destroy_step( + deployment, + client, + tfhelper, + jhelper, + self.manifest, + backend_name, + deployment.openstack_machines_model, + ), + ] + + run_plan(plan, console) + + def update_backend_config( + self, deployment: Deployment, backend_name: str, config_updates: Dict[str, Any] + ) -> None: + """Update backend configuration using Terraform.""" + service = self._get_service(deployment) + if not service.backend_exists(backend_name, self.name): + raise BackendNotFoundException(f"Backend '{backend_name}' not found") + + # Ensure the Terraform plan is registered so we can obtain its tfhelper + self.register_terraform_plan(deployment) + + plan = [ + TerraformInitStep(deployment.get_tfhelper(self.tfplan)), + self.create_update_config_step(deployment, backend_name, config_updates), + ] + + run_plan(plan, console) + + @property + def config_class(self) -> type[StorageBackendConfig]: + """Return the configuration class for this backend.""" + return StorageBackendConfig + + # Backend-specific properties that subclasses should override + @property + def backend_type(self) -> str: + """Backend type identifier. Override in subclasses.""" + return self.name + + @property + def charm_name(self) -> str: + """Charm name for this backend. Override in subclasses.""" + raise NotImplementedError("Subclasses must define charm_name") + + @property + def charm_channel(self) -> str: + """Charm channel for this backend. Override in subclasses.""" + return "stable" + + @property + def charm_revision(self) -> Optional[int]: + """Charm revision for this backend. Override in subclasses.""" + return None + + @property + def charm_base(self) -> str: + """Charm base for this backend. Override in subclasses.""" + return "ubuntu@22.04" + + @property + def backend_endpoint(self) -> str: + """Backend endpoint name for integration. Override in subclasses.""" + return "cinder-volume" + + @property + def units(self) -> int: + """Number of units to deploy. Override in subclasses.""" + return 1 + + @property + def additional_integrations(self) -> List[str]: + """Additional integrations for this backend. Override in subclasses.""" + return [] + + @abstractmethod + def get_terraform_variables( + self, backend_name: str, config: StorageBackendConfig, model: str + ) -> Dict[str, Any]: + """Generate Terraform variables for this backend. Override in subclasses.""" + raise NotImplementedError("Subclasses must implement get_terraform_variables") + + def get_field_mapping(self) -> Dict[str, str]: + """Get mapping from config fields to charm config options. + + Maps Pydantic field names (with underscores) to charm config option + names (with hyphens). Uses the config_class to automatically generate + the mapping from Pydantic model fields. + """ + config_class = self.config_class + # Use model_fields for Pydantic v2 + model_fields = getattr(config_class, "model_fields", {}) + field_names = model_fields.keys() if model_fields else [] + + return {key: key.replace("_", "-") for key in field_names} + + @abstractmethod + def prompt_for_config(self, backend_name: str) -> StorageBackendConfig: + """Prompt user for backend-specific configuration. Override in subclasses.""" + raise NotImplementedError("Subclasses must implement prompt_for_config") + + # Common utility methods + def _filter_config_for_charm( + self, + config_dict: Dict[str, Any], + field_mapping: Dict[str, str], + exclude_fields: Set[str] | None = None, + ) -> Dict[str, Any]: + """Filter configuration dictionary for charm deployment. + + Only includes explicitly set values (non-default, non-empty) to avoid + sending unnecessary configuration to the charm. + + Args: + config_dict: Configuration dictionary to filter + field_mapping: Mapping from config keys to charm config keys + exclude_fields: Set of fields to exclude from filtering + + Returns: + Filtered charm configuration dictionary + """ + exclude_fields = exclude_fields or set() + + # Get default values for comparison + default_config = self._get_default_config() + default_dict = default_config.model_dump() + + charm_config = {} + for key, value in config_dict.items(): + # Skip excluded fields + if key in exclude_fields: + continue + + if key in field_mapping: + # Only include explicitly set values (non-default, non-empty) + if self._should_include_config_value(key, value, default_dict.get(key)): + charm_config[field_mapping[key]] = value + + return charm_config + + def _should_include_config_value( + self, key: str, value: Any, default_value: Any + ) -> bool: + """Determine if a configuration value should be included in charm config. + + Args: + key: Configuration field name + value: Current value + default_value: Default value for this field + + Returns: + True if the value should be sent to the charm, False otherwise + """ + # Always include the 'name' field as it's required + if key == "name": + return True + + # Skip None values + if value is None: + return False + + # Skip empty strings + if isinstance(value, str) and value.strip() == "": + return False + + # Skip empty lists + if isinstance(value, list) and len(value) == 0: + return False + + # Skip empty dictionaries + if isinstance(value, dict) and len(value) == 0: + return False + + # Skip values that match the default + if value == default_value: + return False + + # Include all other values + return True + + @abstractmethod + def _get_default_config(self) -> StorageBackendConfig: + """Get a default configuration instance for comparison. + + Subclasses must implement this to provide a default config instance + with dummy values for required fields. + """ + raise NotImplementedError("Subclasses must implement _get_default_config") + + def _get_credential_fields(self) -> Set[str]: + """Get set of credential field names that should be excluded from charm config. + + Subclasses can override this to specify which fields contain credentials + that should not be sent to the charm configuration. + + Returns: + Set of field names containing credentials + """ + return {"name"} # Default: only exclude the name field + + # Common utility methods (Abstraction 2: IP/FQDN validation) + @staticmethod + def _validate_ip_or_fqdn(value: str) -> str: + """Validate IP address or FQDN. + + Args: + value: IP address or FQDN to validate + + Returns: + The validated value + + Raises: + click.BadParameter: If value is not a valid IP or FQDN + """ + try: + ipaddress.ip_address(value) + return value + except ValueError: + # If not a valid IP, check if it's a valid FQDN + if re.match(FQDN_PATTERN, value): + return value + raise click.BadParameter("Must be a valid IP address or FQDN") + + def _get_cli_class(self): + """Get the CLI class for this backend. + + Subclasses should override this to return their CLI class. + Default implementation attempts to import based on naming convention. + """ + try: + # Try to import CLI class based on naming convention + module_path = f"sunbeam.storage.backends.{self.name}.cli" + cli_module = __import__(module_path, fromlist=[f"{self.name.title()}CLI"]) + cli_class_name = f"{self.name.title()}CLI" + return getattr(cli_module, cli_class_name) + except (ImportError, AttributeError): + raise NotImplementedError( + f"Subclasses must implement _get_cli_class or " + f"follow naming convention: " + f"sunbeam.storage.backends.{self.name}.cli.{self.name.title()}CLI" + ) diff --git a/sunbeam-python/sunbeam/storage/models.py b/sunbeam-python/sunbeam/storage/models.py new file mode 100644 index 000000000..9aa09f205 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/models.py @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Storage backend models and exceptions.""" + +from typing import Any, Dict + +from pydantic import BaseModel, Field + +from sunbeam.core.common import SunbeamException + +# ============================================================================= +# Exceptions +# ============================================================================= + + +class StorageBackendException(SunbeamException): + """Base exception for storage backend operations.""" + + pass + + +class BackendNotFoundException(StorageBackendException): + """Raised when storage backend is not found.""" + + pass + + +class BackendAlreadyExistsException(StorageBackendException): + """Raised when storage backend already exists.""" + + pass + + +class BackendValidationException(StorageBackendException): + """Raised when storage backend configuration is invalid.""" + + pass + + +# ============================================================================= +# Data Models +# ============================================================================= + + +class StorageBackendConfig(BaseModel): + """Base configuration model for storage backends.""" + + name: str = Field(..., description="Backend name") + + +class StorageBackendInfo(BaseModel): + """Information about a deployed storage backend.""" + + name: str + backend_type: str + status: str + charm: str + config: Dict[str, Any] = {} diff --git a/sunbeam-python/sunbeam/storage/registry.py b/sunbeam-python/sunbeam/storage/registry.py new file mode 100644 index 000000000..867259987 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/registry.py @@ -0,0 +1,192 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +import importlib +import logging +import pathlib +from typing import Dict, List + +import click +from rich.console import Console +from rich.table import Table + +from sunbeam.core.deployment import Deployment +from sunbeam.storage.base import StorageBackendBase +from sunbeam.storage.models import StorageBackendInfo +from sunbeam.storage.service import StorageBackendService + +LOG = logging.getLogger(__name__) +console = Console() + +# Global registry for storage backends +_STORAGE_BACKENDS: Dict[str, StorageBackendBase] = {} + + +class StorageBackendRegistry: + """Registry for managing storage backends.""" + + def __init__(self): + self._backends: Dict[str, StorageBackendBase] = {} + self._loaded = False + + def _load_backends(self) -> None: + """Load all storage backends from the storage/backends directory.""" + if self._loaded: + return + + LOG.debug("Loading storage backends") + import sunbeam.storage.backends + + sunbeam_storage_backends = pathlib.Path( + sunbeam.storage.backends.__file__ + ).parent + + for path in sunbeam_storage_backends.iterdir(): + # Skip non-directories and special files + if not path.is_dir() or path.name.startswith("_") or path.name == "etc": + continue + + backend_name = path.name + backend_module_path = path / "backend.py" + + # Check if the backend.py file exists in the backend directory + if not backend_module_path.exists(): + LOG.debug(f"Skipping {backend_name}: no backend.py file found") + continue + + try: + LOG.debug(f"Loading storage backend: {backend_name}") + # Import the backend module from the backend subdirectory + mod = importlib.import_module( + f"sunbeam.storage.backends.{backend_name}.backend" + ) + + # Look for backend classes + for attr_name in dir(mod): + attr = getattr(mod, attr_name) + if ( + isinstance(attr, type) + and issubclass(attr, StorageBackendBase) + and attr != StorageBackendBase + ): + backend_instance = attr() + self._backends[backend_instance.name] = backend_instance + LOG.debug( + f"Registered storage backend: {backend_instance.name}" + ) + + except Exception as e: + LOG.warning(f"Failed to load storage backend {backend_name}: {e}") + + self._loaded = True + + def get_backend(self, name: str) -> StorageBackendBase: + """Get a storage backend by name.""" + self._load_backends() + if name not in self._backends: + raise ValueError(f"Storage backend '{name}' not found") + return self._backends[name] + + def list_backends(self) -> Dict[str, StorageBackendBase]: + """Get all available storage backends.""" + self._load_backends() + return self._backends.copy() + + def get_backends(self) -> Dict[str, StorageBackendBase]: + """Get all available storage backends (alias for list_backends).""" + return self.list_backends() + + def register_cli_commands( + self, storage_group: click.Group, deployment: Deployment + ) -> None: + """Register all backend commands with the storage CLI group. + + This now follows the provider pattern: create stable top-level groups + and let each backend self-register its subcommands under those groups. + The CLI UX remains the same, e.g.: + sunbeam storage add [...] + sunbeam storage remove + sunbeam storage list all + sunbeam storage config show + sunbeam storage config set key=value ... + sunbeam storage config reset key ... + sunbeam storage config options [name] + """ + self._load_backends() + + # Top-level subgroups + add_group = click.Group(name="add") + remove_group = click.Group(name="remove") + list_group = click.Group(name="list") + config_group = click.Group(name="config") + + # Config subgroups for backend-specific commands + config_show = click.Group(name="show") + config_set = click.Group(name="set") + config_reset = click.Group(name="reset") + config_options = click.Group(name="options") + + # Attach config subgroups to the config group + config_group.add_command(config_show) + config_group.add_command(config_set) + config_group.add_command(config_reset) + config_group.add_command(config_options) + + # List commands (keep generic 'all') + @click.command(name="all") + @click.pass_context + def list_all(ctx): + """List all storage backends.""" + service = StorageBackendService(deployment) + backends = service.list_backends() + self._display_backends_table(backends) + + list_group.add_command(list_all) + + # Delegate CLI registration to each backend + for backend in self._backends.values(): + try: + backend.register_add_cli(add_group) + backend.register_cli( + remove_group, + config_show, + config_set, + config_options, + deployment, + ) + except Exception as e: + backend_name = getattr(backend, "name", "unknown") + LOG.warning( + "Backend %s failed to register CLI: %s", + backend_name, + e, + ) + + # Mount groups under storage + storage_group.add_command(add_group) + storage_group.add_command(remove_group) + storage_group.add_command(list_group) + storage_group.add_command(config_group) + + def _display_backends_table(self, backends: List[StorageBackendInfo]) -> None: + """Display backends in a formatted table.""" + if not backends: + console.print("[yellow]No storage backends found[/yellow]") + return + + table = Table(title="Storage Backends") + table.add_column("Name", style="cyan") + table.add_column("Type", style="magenta") + table.add_column("Status", style="green") + table.add_column("Charm", style="blue") + + for backend in backends: + status_style = "green" if backend.status == "active" else "red" + table.add_row( + backend.name, + backend.backend_type, + f"[{status_style}]{backend.status}[/{status_style}]", + backend.charm, + ) + + console.print(table) diff --git a/sunbeam-python/sunbeam/storage/service.py b/sunbeam-python/sunbeam/storage/service.py new file mode 100644 index 000000000..31e491158 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/service.py @@ -0,0 +1,248 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Storage backend service layer.""" + +import logging +from typing import Any, Dict, List + +from rich.console import Console + +from sunbeam.clusterd.service import ConfigItemNotFoundException +from sunbeam.core.common import read_config +from sunbeam.core.deployment import Deployment +from sunbeam.core.juju import JujuHelper + +from .models import ( + BackendNotFoundException, + StorageBackendException, + StorageBackendInfo, +) + +LOG = logging.getLogger(__name__) +console = Console() + + +class StorageBackendService: + """Service layer for storage backend operations.""" + + def __init__(self, deployment: Deployment): + self.deployment = deployment + self.model = self._get_model_name() + # Use a consistent config key for all storage backends + self._tfvar_config_key = "TerraformVarsStorageBackends" + + def _get_model_name(self) -> str: + """Get the OpenStack machines model name.""" + model = self.deployment.openstack_machines_model + if not model.startswith("admin/"): + model = f"admin/{model}" + return model + + def list_backends(self) -> List[StorageBackendInfo]: + """List all Terraform-managed storage backends with dynamic status. + + Returns: + List of StorageBackendInfo objects for all Terraform-managed + storage backends with real-time status and charm information + """ + backends = [] + client = self.deployment.get_client() + jhelper = JujuHelper(self.deployment.juju_controller) + + # Get all available backend types from registry + from .registry import StorageBackendRegistry + + registry = StorageBackendRegistry() + available_backends = registry.get_backends() + + # Search each backend type's individual config key + for backend_type, backend_instance in available_backends.items(): + try: + # Each backend stores config in its + # own key: TerraformVarsStorageBackends{Type} + backend_config_key = backend_instance.tfvar_config_key + LOG.debug( + f"Searching for {backend_type} backends in config key: " + f"{backend_config_key}" + ) + + current_config = read_config(client, backend_config_key) + + # Look for backend-specific keys (e.g., "hitachi_backends", + # "purestorage_backends") + backend_key = f"{backend_type}_backends" + + if backend_key in current_config: + for backend_name, backend_config in current_config[ + backend_key + ].items(): + try: + # Get actual application name from Terraform config + app_name = backend_config.get( + "application_name", backend_name + ) + + # Query actual status and charm from Juju + status = self._get_application_status(jhelper, app_name) + charm_name = self._get_application_charm(jhelper, app_name) + + backend = StorageBackendInfo( + name=backend_name, + backend_type=backend_type, + status=status, + charm=charm_name, + config=backend_config.get("charm_config", {}), + ) + backends.append(backend) + LOG.debug(f"Found {backend_type} backend: {backend_name}") + except Exception as e: + LOG.warning( + f"Error processing {backend_type}" + f"backend {backend_name}: {e}" + ) + continue + else: + LOG.debug(f"No {backend_key} found in config for {backend_type}") + + except ConfigItemNotFoundException: + LOG.debug(f"No configuration found for {backend_type} backends") + continue + except Exception as e: + LOG.warning(f"Error reading {backend_type} backends from clusterd: {e}") + continue + + return backends + + def _get_application_status(self, jhelper: JujuHelper, app_name: str) -> str: + """Get application status from Juju. + + Args: + jhelper: JujuHelper instance for Juju operations + app_name: Name of the Juju application + + Returns: + Application status string or "unknown" if not found + """ + try: + # Get model status using JujuHelper.get_model_status() + model_status = jhelper.get_model_status( + self.deployment.openstack_machines_model + ) + + # Check if application exists in the model + if app_name in model_status.apps: + app_status = model_status.apps[app_name] + return app_status.app_status.current + + return "not-found" + except Exception as e: + LOG.debug(f"Failed to get status for application {app_name}: {e}") + return "unknown" + + def _get_application_charm(self, jhelper: JujuHelper, app_name: str) -> str: + """Get charm name from Juju. + + Args: + jhelper: JujuHelper instance for Juju operations + app_name: Name of the Juju application + + Returns: + Charm name or fallback name if not found + """ + try: + # Get model status using JujuHelper.get_model_status() + model_status = jhelper.get_model_status( + self.deployment.openstack_machines_model + ) + + # Check if application exists in the model + if app_name in model_status.apps: + app_status = model_status.apps[app_name] + charm_url = app_status.charm + return charm_url + + return "Not Found" + + except Exception as e: + LOG.debug(f"Failed to get charm for application {app_name}: {e}") + return "Unknown" + + def _load_backend_tfvars(self, backend_type: str) -> Dict[str, Any]: + """Safely load storage backend Terraform variables from clusterd. + + Args: + backend_type: The backend type (e.g., 'hitachi', 'purestorage') + + Returns: + Dict containing the backend configuration, empty dict if not found. + """ + try: + # Get the backend instance to access its config key + from .registry import StorageBackendRegistry + + registry = StorageBackendRegistry() + available_backends = registry.get_backends() + + if backend_type not in available_backends: + return {} + + backend_instance = available_backends[backend_type] + backend_config_key = backend_instance.tfvar_config_key + + client = self.deployment.get_client() + return read_config(client, backend_config_key) + except ConfigItemNotFoundException: + return {} + except Exception: + return {} + + def _iter_backend_items(self, tfvars: Dict[str, Any], backend_type: str): + """Yield (name, config) pairs for a given backend_type.""" + typed_key = f"{backend_type}_backends" + typed_map = tfvars.get(typed_key, {}) or {} + if isinstance(typed_map, dict): + for name, cfg in typed_map.items(): + yield name, cfg + + def _get_backend_entry( + self, backend_type: str, backend_name: str + ) -> Dict[str, Any] | None: + """Return the backend config entry if found, else None.""" + tfvars = self._load_backend_tfvars(backend_type) + for name, cfg in self._iter_backend_items(tfvars, backend_type): + if name == backend_name: + return cfg + return None + + def backend_exists(self, backend_name: str, backend_type: str) -> bool: + """Check if a backend exists in Terraform configuration.""" + return self._get_backend_entry(backend_type, backend_name) is not None + + def get_backend_config( + self, backend_name: str, backend_type: str + ) -> Dict[str, Any]: + """Get the current configuration of a storage backend.""" + try: + entry = self._get_backend_entry(backend_type, backend_name) + if not entry: + raise BackendNotFoundException(f"Backend '{backend_name}' not found") + + # Return the full backend configuration, not just charm_config + # This includes credentials that can be masked by the display logic + full_config = dict(entry) + + # Merge charm_config into the top level for backward compatibility + charm_config = entry.get("charm_config", {}) + full_config.update(charm_config) + + # Remove the charm_config key to avoid showing it as a separate field + full_config.pop("charm_config", None) + + return full_config + + except BackendNotFoundException: + raise + except Exception as e: + LOG.error(f"Failed to get config for backend '{backend_name}': {e}") + raise StorageBackendException(f"Failed to get backend config: {e}") from e diff --git a/sunbeam-python/sunbeam/storage/steps.py b/sunbeam-python/sunbeam/storage/steps.py new file mode 100644 index 000000000..2360cda3c --- /dev/null +++ b/sunbeam-python/sunbeam/storage/steps.py @@ -0,0 +1,642 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Base step classes for storage backend implementations. + +This module provides base step classes that facilitate the implementation +of storage backend steps. Backends can inherit from these base classes +to get common functionality while customizing specific behavior. +""" + +import logging +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any, Dict + +import tenacity +from rich.console import Console +from rich.status import Status + +from sunbeam.clusterd.client import Client +from sunbeam.clusterd.service import ConfigItemNotFoundException +from sunbeam.core.common import ( + BaseStep, + Result, + ResultType, + friendly_terraform_lock_retry_callback, + read_config, + update_config, +) +from sunbeam.core.deployment import Deployment +from sunbeam.core.juju import ( + ControllerNotFoundException, + ControllerNotReachableException, + JujuException, + JujuHelper, +) +from sunbeam.core.manifest import Manifest +from sunbeam.core.terraform import TerraformHelper, TerraformStateLockedException + +from .models import BackendNotFoundException, StorageBackendConfig + +if TYPE_CHECKING: + from .base import StorageBackendBase + +LOG = logging.getLogger(__name__) +console = Console() + + +class ValidateStoragePrerequisitesStep(BaseStep): + """Validate that Sunbeam is bootstrapped and storage role is deployed.""" + + def __init__(self, deployment: Deployment, client: Client, jhelper: JujuHelper): + super().__init__( + "Validate storage prerequisites", + "Checking Sunbeam bootstrap and storage role deployment", + ) + self.deployment = deployment + self.client = client + self.jhelper = jhelper + self.OPENSTACK_MACHINE_MODEL = self.deployment.openstack_machines_model + + def _check_juju_authentication(self) -> Result: + """Check if the current user is authenticated with Juju.""" + try: + # Use the existing JujuHelper to check authentication + # If we can list models, we're authenticated + models = self.jhelper.models() + LOG.debug( + f"Juju authentication check successful, found {len(models)} models" + ) + return Result(ResultType.COMPLETED) + + except ControllerNotFoundException: + return Result( + ResultType.FAILED, + "Juju controller not found. Please ensure Sunbeam is bootstrapped:\n" + "'sunbeam cluster bootstrap'", + ) + except ControllerNotReachableException: + return Result( + ResultType.FAILED, + "Juju controller not reachable. Please check network connectivity\n" + "or re-authenticate with 'sunbeam utils juju-login'", + ) + except JujuException as e: + # Check if it's an authentication-related error + error_msg = str(e).lower() + if any( + keyword in error_msg + for keyword in [ + "not logged in", + "authentication", + "unauthorized", + "permission denied", + "please enter password", + ] + ): + return Result( + ResultType.FAILED, + "Not authenticated with Juju controller. Please run:\n" + "'sunbeam utils juju-login'\n" + "or authenticate manually with 'juju login'", + ) + else: + return Result(ResultType.FAILED, f"Juju operation failed: {e}") + except Exception as e: + return Result( + ResultType.FAILED, f"Failed to check Juju authentication: {e}" + ) + + def run(self, status: Status | None = None) -> Result: + """Validate storage backend prerequisites.""" + try: + # 0. Check Juju authentication first + auth_result = self._check_juju_authentication() + if auth_result.result_type != ResultType.COMPLETED: + return auth_result + + # 1. Check if Sunbeam is bootstrapped + is_bootstrapped = self.client.cluster.check_sunbeam_bootstrapped() + if not is_bootstrapped: + return Result( + ResultType.FAILED, + "Deployment not bootstrapped. Please run\n" + "'sunbeam cluster bootstrap' first.", + ) + + # 2. Check if OpenStack model exists + if not self.jhelper.model_exists(self.OPENSTACK_MACHINE_MODEL): + return Result( + ResultType.FAILED, + f"OpenStack model '{self.OPENSTACK_MACHINE_MODEL}' not found. " + "Please deploy OpenStack first with\n" + "'sunbeam configure --openstack'.", + ) + + # 3. Check if storage role is deployed (at least one storage node) + storage_nodes = self.client.cluster.list_nodes_by_role("storage") + if not storage_nodes: + return Result( + ResultType.FAILED, + "No storage role found. Please add storage nodes to the cluster " + "before deploying storage backends.", + ) + + # 4. Check if cinder-volume application exists in OpenStack model + try: + cinder_volume_app = self.jhelper.get_application( + "cinder-volume", self.OPENSTACK_MACHINE_MODEL + ) + if not cinder_volume_app: + return Result( + ResultType.FAILED, + "cinder-volume application not found in OpenStack model. " + "Please deploy OpenStack storage services first.", + ) + except Exception as e: + LOG.debug(f"Failed to check cinder-volume application: {e}") + return Result( + ResultType.FAILED, + "Unable to verify cinder-volume application. " + "Please ensure OpenStack storage services are deployed.", + ) + + console.print("✓ All storage prerequisites validated successfully") + return Result(ResultType.COMPLETED) + + except Exception as e: + LOG.error(f"Failed to validate storage prerequisites: {e}") + return Result(ResultType.FAILED, str(e)) + + +class BaseStorageBackendDeployStep(BaseStep, ABC): + """Base class for storage backend deployment steps. + + Provides common deployment functionality that backends can inherit from + and customize as needed. Backends should override get_terraform_variables() + and can override other methods for custom behavior. + """ + + def __init__( + self, + deployment: Deployment, + client: Client, + tfhelper: TerraformHelper, + jhelper: JujuHelper, + manifest: Manifest, + backend_name: str, + backend_config: StorageBackendConfig, + backend_instance: "StorageBackendBase", + model: str, + ): + super().__init__( + f"Deploy {backend_instance.display_name} backend {backend_name}", + f"Deploying {backend_instance.display_name} storage backend {backend_name}", + ) + self.deployment = deployment + self.client = client + self.tfhelper = tfhelper + self.jhelper = jhelper + self.manifest = manifest + self.backend_name = backend_name + self.backend_config = backend_config + self.backend_instance = backend_instance + self.model = model + + @abstractmethod + def get_terraform_variables(self) -> Dict[str, Any]: + """Get Terraform variables for this backend deployment. + + Backends must implement this method to provide their specific + Terraform variables for deployment. + """ + pass + + @tenacity.retry( + wait=tenacity.wait_fixed(60), + stop=tenacity.stop_after_delay(300), + retry=tenacity.retry_if_exception_type(TerraformStateLockedException), + retry_error_callback=friendly_terraform_lock_retry_callback, + before_sleep=lambda retry_state: console.print( + f"Terraform state locked, retrying in 60 seconds... " + f"(attempt {retry_state.attempt_number}/5)" + ), + ) + def run(self, status: Status | None = None) -> Result: + """Deploy the storage backend using Terraform.""" + try: + # Ensure fresh Juju credentials and Terraform env before applying + try: + self.deployment.reload_tfhelpers() + except Exception as cred_err: + LOG.debug(f"Failed to reload credentials/env: {cred_err}") + + # Get Terraform variables for this backend (contains a single backend entry) + tf_vars = self.get_terraform_variables() + + # Merge with existing backends so we don't overwrite them + backend_key = f"{self.backend_instance.name}_backends" + try: + current_tfvars = read_config( + self.client, self.backend_instance.tfvar_config_key + ) + current_backends = ( + current_tfvars.get(backend_key, {}) if current_tfvars else {} + ) + except Exception: + current_backends = {} + + # The new backend map is at tf_vars[backend_key] + new_backends = tf_vars.get(backend_key, {}) + merged_backends = {**current_backends, **new_backends} + tf_vars[backend_key] = merged_backends + + # Update Terraform variables and apply with merged map + self.tfhelper.update_tfvars_and_apply_tf( + self.client, + self.manifest, + tfvar_config=self.backend_instance.tfvar_config_key, + override_tfvars=tf_vars, + ) + + console.print( + f"Successfully deployed {self.backend_instance.display_name} " + f"backend '{self.backend_name}'" + ) + return Result(ResultType.COMPLETED) + + except TerraformStateLockedException as e: + # Bubble up to trigger retry + raise e + except Exception as e: + LOG.error( + f"Failed to deploy {self.backend_instance.display_name} " + f"backend {self.backend_name}: {e}" + ) + return Result(ResultType.FAILED, str(e)) + + def get_application_timeout(self) -> int: + """Return application timeout in seconds. Override for custom timeout.""" + return 1200 # 20 minutes, same as cinder-volume + + def get_accepted_application_status(self) -> list[str]: + """Return accepted application status.""" + return ["active", "waiting"] + + +class BaseStorageBackendDestroyStep(BaseStep, ABC): + """Base class for storage backend destruction steps. + + Provides common destruction functionality that backends can inherit from + and customize as needed. Handles Terraform state cleanup and configuration + removal from clusterd. + """ + + def __init__( + self, + deployment: Deployment, + client: Client, + tfhelper: TerraformHelper, + jhelper: JujuHelper, + manifest: Manifest, + backend_name: str, + backend_instance: "StorageBackendBase", + model: str, + ): + super().__init__( + f"Destroy {backend_instance.display_name} backend {backend_name}", + f"Destroying {backend_instance.display_name} storage " + f"backend {backend_name}", + ) + self.deployment = deployment + self.client = client + self.tfhelper = tfhelper + self.jhelper = jhelper + self.manifest = manifest + self.backend_name = backend_name + self.backend_instance = backend_instance + self.model = model + + def should_destroy_all_resources(self) -> bool: + """Check if all resources should be destroyed (no backends left). + + Override this method if backend has custom logic for determining + when to destroy all resources vs just removing configuration. + """ + try: + current_config = read_config( + self.client, self.backend_instance.tfvar_config_key + ) + + backend_key = ( + f"{self.backend_instance.name}_backends" # e.g., "hitachi_backends" + ) + + if backend_key in current_config: + backends = current_config[backend_key] + else: + raise BackendNotFoundException( + f"Backend '{self.backend_name}' not found" + ) + + # Remove this backend from the count + backends_without_current = { + k: v for k, v in backends.items() if k != self.backend_name + } + return len(backends_without_current) == 0 + except ConfigItemNotFoundException: + return True + + @tenacity.retry( + wait=tenacity.wait_fixed(60), + stop=tenacity.stop_after_delay(300), + retry=tenacity.retry_if_exception_type(TerraformStateLockedException), + retry_error_callback=friendly_terraform_lock_retry_callback, + before_sleep=lambda retry_state: console.print( + f"Terraform state locked, retrying in 60 seconds... " + f"(attempt {retry_state.attempt_number}/5)" + ), + ) + def run(self, status: Status | None = None) -> Result: + """Run the destroy step atomically. + + This step removes the backend from the Terraform configuration + and applies the changes to destroy the associated resources. + The operation is atomic: either it succeeds completely or fails + without modifying the configuration. + """ + try: + # Ensure fresh Juju credentials and Terraform env before destroying/applying + try: + self.deployment.reload_tfhelpers() + except Exception as cred_err: + LOG.debug(f"Failed to reload credentials/env: {cred_err}") + + # First, read and validate the current configuration + try: + current_config = read_config( + self.client, self.backend_instance.tfvar_config_key + ) + except ConfigItemNotFoundException: + LOG.warning(f"No configuration found for backend {self.backend_name}") + raise BackendNotFoundException( + f"No Terraform configuration found for backend " + f"'{self.backend_name}'" + ) + + # Check if backend exists in configuration + backend_key = ( + f"{self.backend_instance.name}_backends" # e.g., "hitachi_backends" + ) + + if ( + backend_key not in current_config + or self.backend_name not in current_config[backend_key] + ): + LOG.warning(f"Backend {self.backend_name} not found in configuration") + raise BackendNotFoundException( + f"Backend '{self.backend_name}' not found in Terraform" + f"configuration. This may indicate a state inconsistency." + ) + + # Create a backup of the backend configuration before removal + backend_backup = current_config[backend_key][self.backend_name].copy() + + # Remove backend from configuration (in memory only) + del current_config[backend_key][self.backend_name] + + # Determine if we need to destroy all resources or just apply changes + destroy_all = self.should_destroy_all_resources() + + try: + if destroy_all: + # For complete destruction: first update config, then destroy + # If destroy fails, we can restore the config + update_config( + self.client, + self.backend_instance.tfvar_config_key, + current_config, + ) + + try: + self.tfhelper.destroy() + except Exception as destroy_error: + # Restore the backend configuration if destroy fails + LOG.error( + f"""Terraform destroy failed, + restoring configuration: {destroy_error}""" + ) + current_config[backend_key][self.backend_name] = backend_backup + update_config( + self.client, + self.backend_instance.tfvar_config_key, + current_config, + ) + raise destroy_error + else: + # For partial removal: update config and apply atomically + LOG.info( + f"Performing partial removal for backend {self.backend_name}" + ) + LOG.info( + f"Remaining backends after removal: " + f"{list(current_config[backend_key].keys())}" + ) + + # First update the configuration + update_config( + self.client, + self.backend_instance.tfvar_config_key, + current_config, + ) + LOG.info("Configuration updated, now running terraform apply...") + + try: + LOG.info("Starting terraform apply for partial removal") + # CRITICAL: Write the updated Terraform variables + # before applying + # This was missing and causing partial removal to fail! + LOG.info("Writing updated Terraform variables...") + + # Get the updated Terraform variables from the current config + tf_vars = current_config.copy() + LOG.info( + f"Writing Terraform variables with backends: " + f"{list(tf_vars.get(backend_key, {}).keys())}" + ) + + self.tfhelper.write_tfvars(tf_vars) + LOG.info("Terraform variables written, now applying...") + self.tfhelper.apply() + LOG.info( + "Terraform apply completed successfully for partial removal" + ) + except Exception as apply_error: + # Restore the backend configuration if apply fails + LOG.error( + f"Terraform apply failed, restoring configuration: " + f"{apply_error}" + ) + current_config[backend_key][self.backend_name] = backend_backup + update_config( + self.client, + self.backend_instance.tfvar_config_key, + current_config, + ) + raise apply_error + + except Exception as tf_error: + # Any Terraform operation failure should be propagated + raise tf_error + + console.print( + f"Successfully removed {self.backend_instance.display_name} " + f"backend '{self.backend_name}'" + ) + return Result(ResultType.COMPLETED) + + except TerraformStateLockedException as e: + # Bubble up to trigger retry + raise e + except Exception as e: + LOG.error( + f"Failed to destroy {self.backend_instance.display_name} " + f"backend {self.backend_name}: {e}" + ) + return Result(ResultType.FAILED, str(e)) + + def get_application_timeout(self) -> int: + """Return application timeout in seconds.""" + return 1200 # 20 minutes, same as cinder-volume + + +class BaseStorageBackendConfigUpdateStep(BaseStep, ABC): + """Base class for storage backend configuration update steps. + + Provides common configuration update functionality that backends can inherit from + and customize as needed. Handles configuration updates and reset operations. + """ + + def __init__( + self, + deployment: Deployment, + backend_instance: "StorageBackendBase", + backend_name: str, + config_updates: Dict[str, Any], + ): + super().__init__( + f"Update {backend_instance.display_name} backend config {backend_name}", + f"Updating {backend_instance.display_name} storage backend " + f"configuration for {backend_name}", + ) + self.deployment = deployment + self.backend_instance = backend_instance + self.backend_name = backend_name + self.config_updates = config_updates + self.client = deployment.get_client() + self.tfhelper = deployment.get_tfhelper(backend_instance.tfplan) + + def handle_update_operation(self, current_config: Dict[str, Any]) -> Dict[str, Any]: + """Handle configuration update operation. Override for custom update logic. + + Args: + current_config: Current backend configuration + + Returns: + Updated configuration with new values applied + """ + # Get backend config from new format only + backend_key = ( + f"{self.backend_instance.name}_backends" # e.g., "hitachi_backends" + ) + backend_config = current_config[backend_key][self.backend_name] + if "charm_config" not in backend_config: + backend_config["charm_config"] = {} + + # Apply configuration updates (excluding reset keys) with field mapping + updates = {k: v for k, v in self.config_updates.items() if k != "_reset_keys"} + + # Apply field mapping to convert internal field names to charm field names + field_mapping = self.backend_instance.get_field_mapping() + mapped_updates = {} + for key, value in updates.items(): + # Use field mapping if available, otherwise use the key as-is + charm_key = field_mapping.get(key, key) + mapped_updates[charm_key] = value + + backend_config["charm_config"].update(mapped_updates) + + return current_config + + @tenacity.retry( + wait=tenacity.wait_fixed(60), + stop=tenacity.stop_after_delay(300), + retry=tenacity.retry_if_exception_type(TerraformStateLockedException), + retry_error_callback=friendly_terraform_lock_retry_callback, + before_sleep=lambda retry_state: console.print( + f"Terraform state locked, retrying in 60 seconds... " + f"(attempt {retry_state.attempt_number}/5)" + ), + ) + def run(self, status: Status | None = None) -> Result: + """Update the storage backend configuration using Terraform.""" + # Read current configuration + try: + current_config = read_config( + self.client, self.backend_instance.tfvar_config_key + ) + + # Check new format (backend-specific keys only) + backend_key = ( + f"{self.backend_instance.name}_backends" # e.g., "hitachi_backends" + ) + + if ( + backend_key not in current_config + or self.backend_name not in current_config[backend_key] + ): + return Result( + ResultType.FAILED, f"Backend {self.backend_name} not found" + ) + + # Handle update operation + current_config = self.handle_update_operation(current_config) + operation_type = "update" + + # Save updated configuration and apply with updated tfvars + update_config( + self.client, self.backend_instance.tfvar_config_key, current_config + ) + + # Ensure fresh Juju credentials and Terraform env before applying + try: + self.deployment.reload_tfhelpers() + except Exception as cred_err: + LOG.debug(f"Failed to reload credentials/env: {cred_err}") + + # Write the updated tfvars and apply + self.tfhelper.write_tfvars(current_config) + self.tfhelper.apply() + + console.print( + f"Successfully {operation_type}d " + f"{self.backend_instance.display_name} backend " + f"'{self.backend_name}' configuration" + ) + return Result(ResultType.COMPLETED) + + except ConfigItemNotFoundException: + return Result( + ResultType.FAILED, + f"Configuration not found for backend {self.backend_name}", + ) + + except TerraformStateLockedException as e: + # Bubble up to trigger retry + raise e + except Exception as e: + LOG.error( + f"Failed to update {self.backend_instance.display_name} " + f"backend {self.backend_name} configuration: {e}" + ) + return Result(ResultType.FAILED, str(e)) diff --git a/sunbeam-python/tests/unit/sunbeam/storage/backends/__init__.py b/sunbeam-python/tests/unit/sunbeam/storage/backends/__init__.py new file mode 100644 index 000000000..9c90cc6d9 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/storage/backends/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for storage backends.""" diff --git a/sunbeam-python/tests/unit/sunbeam/storage/backends/hitachi/__init__.py b/sunbeam-python/tests/unit/sunbeam/storage/backends/hitachi/__init__.py new file mode 100644 index 000000000..f2acd69a0 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/storage/backends/hitachi/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for Hitachi storage backend.""" diff --git a/sunbeam-python/tests/unit/sunbeam/storage/backends/hitachi/test_cli.py b/sunbeam-python/tests/unit/sunbeam/storage/backends/hitachi/test_cli.py new file mode 100644 index 000000000..238e15718 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/storage/backends/hitachi/test_cli.py @@ -0,0 +1,367 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for Hitachi storage backend CLI commands.""" + +import unittest.mock as mock + +import click +from click.testing import CliRunner + +from sunbeam.storage.backends.hitachi.backend import HitachiBackend, HitachiConfig +from sunbeam.storage.backends.hitachi.cli import HitachiCLI + + +class TestHitachiCLI: + """Test cases for Hitachi storage backend CLI commands.""" + + def setup_method(self): + """Set up test fixtures.""" + self.runner = CliRunner() + self.mock_deployment = mock.MagicMock() + self.backend = HitachiBackend() + self.cli = HitachiCLI(self.backend) + + def test_register_add_cli(self): + """Test that register_add_cli creates a command properly.""" + mock_add_group = mock.MagicMock() + + # Test that register_add_cli doesn't raise an exception + self.cli.register_add_cli(mock_add_group) + + # Verify that a command was added to the group + mock_add_group.add_command.assert_called_once() + + # Get the command that was added + added_command = mock_add_group.add_command.call_args[0][0] + assert isinstance(added_command, click.Command) + assert added_command.name == "hitachi" + + def test_load_config_file_yaml(self): + """Test loading YAML config file.""" + import tempfile + + import yaml + + config_data = { + "name": "test-backend", + "hitachi_storage_id": "12345", + "san_ip": "192.168.1.100", + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + yaml.dump(config_data, f) + f.flush() + + from pathlib import Path + + result = self.cli._load_config_file(Path(f.name)) + + assert result == config_data + + def test_add_missing_required(self): + """Test that Click exits non-zero with missing required option.""" + # Create a mock add group and register the command + add_group = click.Group(name="add") + self.cli.register_add_cli(add_group) + + result = self.runner.invoke( + add_group, + [ + "hitachi", + "--name", + "test-hitachi", + # Missing required --hitachi-storage-id + "--hitachi-pools", + "pool1,pool2", + "--san-ip", + "192.168.1.100", + ], + obj=self.mock_deployment, + ) + + assert result.exit_code != 0 + assert "Missing option" in result.output or "required" in result.output.lower() + + def test_register_cli(self): + """Test that register_cli creates management commands properly.""" + mock_remove = mock.MagicMock() + mock_config_show = mock.MagicMock() + mock_config_set = mock.MagicMock() + mock_config_options = mock.MagicMock() + + # Test that register_cli doesn't raise an exception + self.cli.register_cli( + mock_remove, + mock_config_show, + mock_config_set, + mock_config_options, + self.mock_deployment, + ) + + # Verify that commands were added to each group + mock_remove.add_command.assert_called_once() + mock_config_show.add_command.assert_called_once() + mock_config_set.add_command.assert_called_once() + mock_config_options.add_command.assert_called_once() + + def test_cli_methods_exist(self): + """Test that all required CLI methods exist.""" + assert hasattr(self.cli, "register_add_cli") + assert hasattr(self.cli, "register_cli") + assert hasattr(self.cli, "_load_config_file") + + # Test they are callable + assert callable(self.cli.register_add_cli) + assert callable(self.cli.register_cli) + assert callable(self.cli._load_config_file) + + def test_load_config_file_json(self): + """Test loading JSON config file.""" + import json + import tempfile + + config_data = { + "name": "test-backend", + "hitachi_storage_id": "12345", + "san_ip": "192.168.1.100", + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(config_data, f) + f.flush() + + from pathlib import Path + + result = self.cli._load_config_file(Path(f.name)) + + assert result == config_data + + def test_load_config_file_empty_path(self): + """Test loading config file with empty path returns empty dict.""" + result = self.cli._load_config_file(None) + assert result == {} + + def test_cli_backend_reference(self): + """Test that CLI maintains reference to backend.""" + assert self.cli.backend is not None + assert isinstance(self.cli.backend, HitachiBackend) + assert self.cli.backend.name == "hitachi" + + def test_add_with_chap_auth(self): + """Test addition with CHAP authentication enabled.""" + with mock.patch.object(self.backend, "add_backend") as mock_add_backend: + mock_add_backend.return_value = None + + # Create a mock add group and register the command + add_group = click.Group(name="add") + self.cli.register_add_cli(add_group) + + result = self.runner.invoke( + add_group, + [ + "hitachi", + "--name", + "test-hitachi", + "--hitachi-storage-id", + "12345", + "--hitachi-pools", + "pool1,pool2", + "--san-ip", + "192.168.1.100", + "--san-username", + "sanuser", + "--san-password", + "sanpass", + "--protocol", + "iSCSI", + "--chap-username", + "chapuser", + "--chap-password", + "chappass", + "--use-chap-auth", + "true", + ], + obj=self.mock_deployment, + ) + + assert result.exit_code == 0 + mock_add_backend.assert_called_once() + + # Verify the config passed to add_backend + call_args = mock_add_backend.call_args + config = call_args[0][2] # Third argument is the config + assert config.use_chap_auth is True + assert config.chap_username == "chapuser" + assert config.chap_password == "chappass" + + def test_add_backend_exception(self): + """Test that backend exceptions are handled gracefully.""" + with mock.patch.object(self.backend, "add_backend") as mock_add_backend: + mock_add_backend.side_effect = Exception("Backend error") + + # Create a mock add group and register the command + add_group = click.Group(name="add") + self.cli.register_add_cli(add_group) + + result = self.runner.invoke( + add_group, + [ + "hitachi", + "--name", + "test-hitachi", + "--hitachi-storage-id", + "12345", + "--hitachi-pools", + "pool1,pool2", + "--san-ip", + "192.168.1.100", + "--san-username", + "sanuser", + "--san-password", + "sanpass", + "--protocol", + "iSCSI", + ], + obj=self.mock_deployment, + ) + + assert result.exit_code != 0 + # The exception is raised but not caught by CLI, so no output is generated + # The exception object itself contains the error message + assert result.exception is not None + assert "Backend error" in str(result.exception) + + def test_cli_initialization(self): + """Test that HitachiCLI can be initialized with a backend.""" + backend = HitachiBackend() + cli = HitachiCLI(backend) + assert cli.backend == backend + + def test_backend_delegation_works(self): + """Test that CLI properly delegates to backend methods.""" + # Test that CLI can access backend properties + assert self.cli.backend.name == "hitachi" + assert self.cli.backend.display_name == "Hitachi VSP Storage" + assert self.cli.backend.config_class == HitachiConfig + + def test_register_add_cli_creates_command_with_correct_name(self): + """Test that register_add_cli creates a command with the backend name.""" + mock_add_group = mock.MagicMock() + + self.cli.register_add_cli(mock_add_group) + + # Verify command was added with correct name + mock_add_group.add_command.assert_called_once() + added_command = mock_add_group.add_command.call_args[0][0] + assert added_command.name == self.backend.name + + def test_register_cli_creates_all_management_commands(self): + """Test that register_cli creates all expected management commands.""" + mock_remove = mock.MagicMock() + mock_config_show = mock.MagicMock() + mock_config_set = mock.MagicMock() + mock_config_options = mock.MagicMock() + + self.cli.register_cli( + mock_remove, + mock_config_show, + mock_config_set, + mock_config_options, + self.mock_deployment, + ) + + # Verify all commands were created with correct names + for mock_group in [ + mock_remove, + mock_config_show, + mock_config_set, + mock_config_options, + ]: + mock_group.add_command.assert_called_once() + added_command = mock_group.add_command.call_args[0][0] + assert added_command.name == self.backend.name + + def test_remove_ok(self): + """Test successful removal of Hitachi backend.""" + with ( + mock.patch.object(self.backend, "_get_service") as mock_get_service, + mock.patch.object(self.backend, "remove_backend") as mock_remove_backend, + ): + mock_service = mock.MagicMock() + mock_service.backend_exists.return_value = True + mock_get_service.return_value = mock_service + mock_remove_backend.return_value = None + + # Create a mock remove group and register the command + remove_group = click.Group(name="remove") + config_show = click.Group(name="config_show") + config_set = click.Group(name="config_set") + config_options = click.Group(name="config_options") + self.cli.register_cli( + remove_group, + config_show, + config_set, + config_options, + self.mock_deployment, + ) + + result = self.runner.invoke( + remove_group, + ["hitachi", "test-hitachi", "--yes"], + obj=self.mock_deployment, + ) + + assert result.exit_code == 0 + mock_get_service.assert_called_once_with(self.mock_deployment) + mock_service.backend_exists.assert_called_once_with( + "test-hitachi", "hitachi" + ) + mock_remove_backend.assert_called_once_with( + self.mock_deployment, "test-hitachi", mock.ANY + ) + + def test_remove_service_exception(self): + """Test handling of service exceptions during removal.""" + with ( + mock.patch.object(self.backend, "_get_service") as mock_get_service, + mock.patch.object(self.backend, "remove_backend") as mock_remove_backend, + ): + mock_service = mock.MagicMock() + mock_service.backend_exists.return_value = True + mock_get_service.return_value = mock_service + mock_remove_backend.side_effect = Exception("Service error") + + # Create a mock remove group and register the command + remove_group = click.Group(name="remove") + config_show = click.Group(name="config_show") + config_set = click.Group(name="config_set") + config_options = click.Group(name="config_options") + self.cli.register_cli( + remove_group, + config_show, + config_set, + config_options, + self.mock_deployment, + ) + + result = self.runner.invoke( + remove_group, + ["hitachi", "test-hitachi", "--yes"], + obj=self.mock_deployment, + ) + + assert result.exit_code != 0 + assert "Service" in result.output and "error" in result.output + + def test_cli_class_has_required_methods(self): + """Test that HitachiCLI class has all required methods.""" + required_methods = [ + "register_add_cli", + "register_cli", + "_load_config_file", + ] + + for method_name in required_methods: + assert hasattr(self.cli, method_name) + assert callable(getattr(self.cli, method_name)) diff --git a/sunbeam-python/tests/unit/sunbeam/storage/conftest.py b/sunbeam-python/tests/unit/sunbeam/storage/conftest.py new file mode 100644 index 000000000..00b2c0996 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/storage/conftest.py @@ -0,0 +1,275 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Test fixtures for storage backend tests.""" + +from unittest.mock import Mock, PropertyMock + +import pytest + +from sunbeam.clusterd.client import Client +from sunbeam.core.manifest import Manifest +from sunbeam.core.terraform import TerraformHelper +from sunbeam.storage.base import StorageBackendBase +from sunbeam.storage.models import StorageBackendConfig + + +@pytest.fixture +def mock_deployment(): + """Mock deployment object.""" + deployment = Mock() + deployment.openstack_machines_model = "openstack" # Match actual service behavior + + # Mock the client with cluster attribute + mock_client = Mock() + mock_cluster = Mock() + mock_client.cluster = mock_cluster + deployment.get_client.return_value = mock_client + + # Mock juju_controller + mock_controller = Mock() + mock_controller.name = "test-controller" + type(deployment).juju_controller = PropertyMock(return_value=mock_controller) + + # Mock get_tfhelper + deployment.get_tfhelper.return_value = Mock(spec=TerraformHelper) + + return deployment + + +@pytest.fixture +def mock_client(): + """Mock clusterd client.""" + client = Mock(spec=Client) + client.cluster = Mock() + return client + + +@pytest.fixture +def mock_tfhelper(): + """Mock Terraform helper.""" + return Mock(spec=TerraformHelper) + + +@pytest.fixture +def mock_jhelper(): + """Mock Juju helper.""" + return Mock() + + +@pytest.fixture +def mock_manifest(): + """Mock manifest object.""" + return Mock(spec=Manifest) + + +@pytest.fixture +def sample_backend_config(): + """Sample backend configuration for testing.""" + return { + "model": "openstack", + "hitachi_backends": { + "test-backend": { + "backend_type": "hitachi", + "charm_name": "cinder-volume-hitachi", + "charm_channel": "latest/edge", + "backend_config": { + "hitachi-storage-id": "123456", + "hitachi-pools": ["pool1", "pool2"], + "san-ip": "192.168.1.100", + }, + "backend_endpoint": "cinder-volume", + "units": 1, + "additional_integrations": {}, + } + }, + } + + +@pytest.fixture +def sample_clusterd_config(): + """Sample clusterd configuration for testing.""" + return { + "TerraformVarsStorageBackends": { + "hitachi_backends": { + "test-backend": { + "backend_type": "hitachi", + "charm_name": "cinder-volume-hitachi", + "charm_channel": "latest/edge", + "charm_config": { + "hitachi-storage-id": "123456", + "hitachi-pools": ["pool1", "pool2"], + "san-ip": "192.168.1.100", + }, + "backend_endpoint": "cinder-volume", + "units": 1, + "additional_integrations": {}, + } + } + } + } + + +@pytest.fixture +def mock_storage_backend(): + """Mock storage backend for testing.""" + + class MockStorageBackend(StorageBackendBase): + """Mock storage backend for testing.""" + + name = "mock" + display_name = "Mock Storage Backend" + charm_name = "mock-charm" + + def __init__(self): + super().__init__() + self.tfplan = "mock-backend-plan" + self.tfplan_dir = "deploy-mock-backend" + + @property + def config_class(self): + return StorageBackendConfig + + def get_terraform_variables( + self, backend_name: str, config: StorageBackendConfig, model: str + ): + return { + "model": model, + "mock_backends": { + backend_name: { + "backend_type": self.name, + "charm_name": self.charm_name, + "charm_channel": "latest/stable", + "backend_config": config.model_dump(), + "backend_endpoint": "storage-backend", + "units": 1, + "additional_integrations": {}, + } + }, + } + + def create_deploy_step( + self, + deployment, + client, + tfhelper, + jhelper, + manifest, + backend_name, + backend_config, + model, + ): + """Create a mock deploy step.""" + from sunbeam.storage.steps import BaseStorageBackendDeployStep + + class MockDeployStep(BaseStorageBackendDeployStep): + def get_terraform_variables(self): + return self.backend_instance.get_terraform_variables( + self.backend_name, self.backend_config, self.model + ) + + return MockDeployStep( + deployment, + client, + tfhelper, + jhelper, + manifest, + backend_name, + backend_config, + self, + model, + ) + + def create_destroy_step( + self, deployment, client, tfhelper, jhelper, manifest, backend_name, model + ): + """Create a mock destroy step.""" + from sunbeam.storage.steps import BaseStorageBackendDestroyStep + + return BaseStorageBackendDestroyStep( + deployment, + client, + tfhelper, + jhelper, + manifest, + backend_name, + self, + model, + ) + + def create_config_update_step( + self, + deployment, + client, + tfhelper, + jhelper, + manifest, + backend_name, + backend_config, + model, + updates, + ): + """Create a mock config update step.""" + from sunbeam.storage.steps import BaseStorageBackendConfigUpdateStep + + return BaseStorageBackendConfigUpdateStep( + deployment, + client, + tfhelper, + jhelper, + manifest, + backend_name, + backend_config, + self, + model, + updates, + ) + + def create_update_config_step(self, deployment, backend_name, config_updates): + """Create a configuration update step for this backend.""" + # Mock implementation - return a simple mock step + + from sunbeam.core.common import BaseStep + + class MockUpdateConfigStep(BaseStep): + def run(self): + from sunbeam.core.common import ResultType + + return ResultType.COMPLETED + + return MockUpdateConfigStep() + + def commands(self): + """Return mock commands for testing.""" + return { + "add": [{"name": "mock", "command": Mock()}], + "remove": [{"name": "mock", "command": Mock()}], + "list": [{"name": "mock", "command": Mock()}], + "config": [{"name": "mock", "command": Mock()}], + } + + def register_add_cli(self, add): + """Mock CLI registration.""" + pass + + def register_cli( + self, + remove, + config_show, + config_set, + config_reset, + config_options, + deployment, + ): + """Mock CLI registration.""" + pass + + def prompt_for_config(self, backend_name: str) -> StorageBackendConfig: + """Mock prompt for configuration.""" + return StorageBackendConfig(name=backend_name) + + def _get_default_config(self) -> StorageBackendConfig: + """Get a default configuration instance for comparison.""" + return StorageBackendConfig(name="default") + + return MockStorageBackend() diff --git a/sunbeam-python/tests/unit/sunbeam/storage/test_hitachi.py b/sunbeam-python/tests/unit/sunbeam/storage/test_hitachi.py new file mode 100644 index 000000000..d200a2e82 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/storage/test_hitachi.py @@ -0,0 +1,575 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for Hitachi storage backend implementation.""" + +from unittest.mock import patch + +import click +import pytest +from pydantic import ValidationError + +from sunbeam.storage.backends.hitachi.backend import ( + HitachiBackend, + HitachiConfig, + HitachiDeployStep, + HitachiDestroyStep, + HitachiUpdateConfigStep, +) +from sunbeam.storage.models import StorageBackendConfig + + +class TestHitachiConfig: + """Test cases for HitachiConfig model.""" + + def test_valid_config_minimal(self): + """Test creating valid minimal Hitachi configuration.""" + config = HitachiConfig( + name="hitachi-backend-1", + hitachi_storage_id="123456", + hitachi_pools="pool1,pool2", + san_ip="192.168.1.100", + protocol="FC", + san_username="testuser", + san_password="testpassword", + ) + + assert config.name == "hitachi-backend-1" + assert config.hitachi_storage_id == "123456" + assert config.hitachi_pools == "pool1,pool2" + assert config.san_ip == "192.168.1.100" + assert config.protocol == "FC" # Default value + assert config.san_username == "testuser" + assert config.san_password == "testpassword" + + def test_valid_config_full(self): + """Test creating valid full Hitachi configuration.""" + config = HitachiConfig( + name="hitachi-backend-1", + hitachi_storage_id="123456", + hitachi_pools="pool1,pool2", + san_ip="192.168.1.100", + protocol="iSCSI", + san_username="testuser", + san_password="testpassword", + ) + + assert config.name == "hitachi-backend-1" + assert config.hitachi_storage_id == "123456" + assert config.hitachi_pools == "pool1,pool2" + assert config.san_ip == "192.168.1.100" + assert config.protocol == "iSCSI" + assert config.san_username == "testuser" + assert config.san_password == "testpassword" + + def test_config_with_iscsi_protocol(self): + """Test configuration with iSCSI protocol.""" + config = HitachiConfig( + name="hitachi-iscsi", + hitachi_storage_id="123456", + hitachi_pools="pool1", + san_ip="192.168.1.100", + protocol="iSCSI", + san_username="testuser", + san_password="testpassword", + ) + + assert config.protocol == "iSCSI" + assert config.san_username == "testuser" + assert config.san_password == "testpassword" + + def test_config_validation_missing_required_fields(self): + """Test validation errors for missing required fields.""" + # Missing hitachi_storage_id + with pytest.raises(ValidationError): + HitachiConfig(name="test", hitachi_pools="pool1", san_ip="192.168.1.100") + + # Missing hitachi_pools + with pytest.raises(ValidationError): + HitachiConfig( + name="test", hitachi_storage_id="123456", san_ip="192.168.1.100" + ) + + # Missing san_ip + with pytest.raises(ValidationError): + HitachiConfig( + name="test", hitachi_storage_id="123456", hitachi_pools="pool1" + ) + + def test_ip_validation_valid_ip(self): + """Test valid IP address validation.""" + config = HitachiConfig( + name="test", + hitachi_storage_id="123456", + hitachi_pools="pool1", + san_ip="192.168.1.100", + protocol="FC", + san_username="testuser", + san_password="testpassword", + ) + assert config.san_ip == "192.168.1.100" + + def test_ip_validation_valid_fqdn(self): + """Test valid FQDN validation.""" + config = HitachiConfig( + name="test", + hitachi_storage_id="123456", + hitachi_pools="pool1", + san_ip="storage.example.com", + protocol="FC", + san_username="testuser", + san_password="testpassword", + ) + assert config.san_ip == "storage.example.com" + + def test_protocol_validation(self): + """Test protocol field validation.""" + # Valid protocols + for protocol in ["FC", "iSCSI"]: + config = HitachiConfig( + name="test", + hitachi_storage_id="123456", + hitachi_pools="pool1", + san_ip="192.168.1.100", + protocol=protocol, + san_username="testuser", + san_password="testpassword", + ) + assert config.protocol == protocol + + def test_config_serialization(self): + """Test configuration serialization.""" + config = HitachiConfig( + name="test", + hitachi_storage_id="123456", + hitachi_pools="pool1", + san_ip="192.168.1.100", + protocol="FC", + san_username="testuser", + san_password="testpassword", + ) + + data = config.model_dump() + assert data["name"] == "test" + assert data["hitachi_storage_id"] == "123456" + assert data["hitachi_pools"] == "pool1" + assert data["san_ip"] == "192.168.1.100" + assert data["protocol"] == "FC" + assert data["san_username"] == "testuser" + assert data["san_password"] == "testpassword" + + def test_config_inheritance(self): + """Test that HitachiConfig inherits from StorageBackendConfig.""" + config = HitachiConfig( + name="test", + hitachi_storage_id="123456", + hitachi_pools="pool1", + san_ip="192.168.1.100", + protocol="FC", + san_username="testuser", + san_password="testpassword", + ) + + assert isinstance(config, StorageBackendConfig) + + +class TestHitachiBackend: + """Test cases for HitachiBackend class.""" + + def test_init(self): + """Test backend initialization.""" + backend = HitachiBackend() + + assert backend.name == "hitachi" + assert backend.display_name == "Hitachi VSP Storage" + assert backend.charm_name == "cinder-volume-hitachi" + assert backend.tfplan == "hitachi-backend-plan" + assert backend.tfplan_dir == "deploy-hitachi-backend" + assert ( + backend.charm_channel == "latest/edge" + ) # this have to be updated after the charm progress to stable + assert backend.charm_revision == 2 + assert backend.charm_base == "ubuntu@24.04" + assert backend.backend_endpoint == "cinder-volume" + assert backend.units == 1 + assert backend.additional_integrations == [] + + def test_config_class(self): + """Test configuration class retrieval.""" + backend = HitachiBackend() + config_class = backend.config_class + assert config_class == HitachiConfig + + def test_get_field_mapping(self): + """Test field mapping for charm configuration.""" + backend = HitachiBackend() + mapping = backend.get_field_mapping() + + # Test some key mappings + assert mapping["hitachi_storage_id"] == "hitachi-storage-id" + assert mapping["hitachi_pools"] == "hitachi-pools" + assert mapping["san_ip"] == "san-ip" + assert mapping["protocol"] == "protocol" + assert mapping["use_chap_auth"] == "use-chap-auth" + + def test_get_terraform_variables(self): + """Test Terraform variables generation.""" + backend = HitachiBackend() + config = HitachiConfig( + name="hitachi-backend-1", + hitachi_storage_id="123456", + hitachi_pools="pool1,pool2", + san_ip="192.168.1.100", + protocol="FC", + san_username="testuser", + san_password="testpassword", + ) + + variables = backend.get_terraform_variables( + "hitachi-backend-1", config, "openstack" + ) + + assert "machine_model" in variables + assert "hitachi_backends" in variables + assert variables["machine_model"] == "openstack" + assert "hitachi-backend-1" in variables["hitachi_backends"] + + backend_config = variables["hitachi_backends"]["hitachi-backend-1"] + assert "charm_config" in backend_config + + # Verify charm config contains the expected fields (excluding credentials) + charm_config = backend_config["charm_config"] + assert charm_config["hitachi-storage-id"] == "123456" + assert charm_config["hitachi-pools"] == "pool1,pool2" + assert charm_config["san-ip"] == "192.168.1.100" + # Protocol field is excluded when it matches default value + assert "protocol" not in charm_config + # Credentials should not be in charm config - they go in secrets + assert "san-username" not in charm_config + assert "san-password" not in charm_config + + def test_should_include_config_value(self): + """Test configuration value inclusion logic.""" + backend = HitachiBackend() + + # Non-default string value should be included + assert backend._should_include_config_value("san-ip", "192.168.1.100", "") + + # Default string value should not be included + assert not backend._should_include_config_value( + "san-login", "maintenance", "maintenance" + ) + + # Empty string should not be included + assert not backend._should_include_config_value("san-ip", "", "") + + # Non-empty list should be included + assert backend._should_include_config_value("hitachi-pools", ["pool1"], []) + + # Empty list should not be included + assert not backend._should_include_config_value("hitachi-pools", [], []) + + # None values should not be included + assert not backend._should_include_config_value("optional-field", None, None) + + def test_create_deploy_step( + self, mock_deployment, mock_client, mock_tfhelper, mock_jhelper, mock_manifest + ): + """Test deployment step creation.""" + backend = HitachiBackend() + config = HitachiConfig( + name="test", + hitachi_storage_id="123456", + hitachi_pools="pool1", + san_ip="192.168.1.100", + protocol="FC", + san_username="testuser", + san_password="testpassword", + ) + + step = backend.create_deploy_step( + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + "test-backend", + config, + "openstack", + ) + + assert isinstance(step, HitachiDeployStep) + assert step.backend_name == "test-backend" + assert step.backend_config == config + assert step.model == "openstack" + + def test_create_destroy_step( + self, mock_deployment, mock_client, mock_tfhelper, mock_jhelper, mock_manifest + ): + """Test destruction step creation.""" + backend = HitachiBackend() + + step = backend.create_destroy_step( + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + "test-backend", + "openstack", + ) + + assert isinstance(step, HitachiDestroyStep) + assert step.backend_name == "test-backend" + assert step.model == "openstack" + + def test_create_update_config_step(self, mock_deployment): + """Test configuration update step creation.""" + backend = HitachiBackend() + config_updates = {"key1": "value1", "key2": "value2"} + + step = backend.create_update_config_step( + mock_deployment, "test-backend", config_updates + ) + + assert isinstance(step, HitachiUpdateConfigStep) + assert step.backend_name == "test-backend" + assert step.config_updates == config_updates + + @patch("click.confirm") + @patch("click.prompt") + def test_prompt_for_config(self, mock_prompt, mock_confirm, mock_deployment): + """Test configuration prompting.""" + backend = HitachiBackend() + + # Mock user inputs for all prompts in _prompt_for_config + mock_prompt.side_effect = [ + "123456", # hitachi_storage_id (Array serial number) + "pool1,pool2", # hitachi_pools (Storage pools) + "FC", # protocol + "192.168.1.100", # san_ip (Management IP/FQDN) + "maintenance", # san_username (SAN username) + "secret123", # san_password (SAN password) + "test-backend", # volume_backend_name + ] + + # Mock confirmation prompts (all False to avoid optional config) + mock_confirm.side_effect = [ + False, # configure_mirror (Configure mirror/replication) + ] + + config = backend.prompt_for_config("test-backend") + + assert isinstance(config, HitachiConfig) + assert config.name == "test-backend" + assert config.hitachi_storage_id == "123456" + assert config.hitachi_pools == "pool1,pool2" + assert config.san_ip == "192.168.1.100" + # Protocol should use default value + assert config.protocol == "FC" + + def test_validate_ip_or_fqdn_valid_ip(self): + """Test IP validation with valid IP.""" + # Should not raise exception + HitachiBackend._validate_ip_or_fqdn("192.168.1.100") + + def test_validate_ip_or_fqdn_valid_fqdn(self): + """Test IP validation with valid FQDN.""" + # Should not raise exception + HitachiBackend._validate_ip_or_fqdn("storage.example.com") + + def test_validate_ip_or_fqdn_invalid(self): + """Test IP validation with invalid input.""" + with pytest.raises(click.BadParameter): + HitachiBackend._validate_ip_or_fqdn("..invalid..") + + +class TestHitachiSteps: + """Test cases for Hitachi-specific step implementations.""" + + def test_hitachi_deploy_step_init( + self, mock_deployment, mock_client, mock_tfhelper, mock_jhelper, mock_manifest + ): + """Test HitachiDeployStep initialization.""" + backend_instance = HitachiBackend() + backend_name = "hitachi-backend-1" + model = "openstack" + + config = HitachiConfig( + name=backend_name, + hitachi_storage_id="123456", + hitachi_pools="pool1,pool2", + san_ip="192.168.1.100", + protocol="FC", + san_username="testuser", + san_password="testpassword", + ) + + step = HitachiDeployStep( + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + backend_name, + config, + backend_instance, + model, + ) + + assert step.backend_name == backend_name + assert step.backend_config == config + assert step.backend_instance == backend_instance + assert step.model == model + + def test_hitachi_deploy_step_get_terraform_variables( + self, mock_deployment, mock_client, mock_tfhelper, mock_jhelper, mock_manifest + ): + """Test Terraform variables generation in deploy step.""" + backend_instance = HitachiBackend() + backend_name = "hitachi-backend-1" + model = "openstack" + + config = HitachiConfig( + name=backend_name, + hitachi_storage_id="123456", + hitachi_pools="pool1,pool2", + san_ip="192.168.1.100", + protocol="FC", + san_username="testuser", + san_password="testpassword", + ) + + step = HitachiDeployStep( + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + backend_name, + config, + backend_instance, + model, + ) + + variables = step.get_terraform_variables() + + assert "machine_model" in variables + assert "hitachi_backends" in variables + assert variables["machine_model"] == "openstack" + assert "hitachi-backend-1" in variables["hitachi_backends"] + + backend_config = variables["hitachi_backends"]["hitachi-backend-1"] + assert "charm_config" in backend_config + + def test_hitachi_destroy_step_init( + self, mock_deployment, mock_client, mock_tfhelper, mock_jhelper, mock_manifest + ): + """Test HitachiDestroyStep initialization.""" + backend_instance = HitachiBackend() + backend_name = "hitachi-backend-1" + model = "openstack" + + step = HitachiDestroyStep( + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + backend_name, + backend_instance, + model, + ) + + assert step.backend_name == backend_name + assert step.backend_instance == backend_instance + assert step.model == model + + def test_hitachi_update_config_step_init(self, mock_deployment): + """Test HitachiUpdateConfigStep initialization.""" + backend_instance = HitachiBackend() + backend_name = "hitachi-backend-1" + config_updates = {"san-ip": "192.168.1.101"} + + step = HitachiUpdateConfigStep( + mock_deployment, backend_instance, backend_name, config_updates + ) + + assert step.backend_name == backend_name + assert step.backend_instance == backend_instance + assert step.config_updates == config_updates + + def test_hitachi_deploy_step_creation( + self, mock_deployment, mock_client, mock_tfhelper, mock_jhelper, mock_manifest + ): + """Test Hitachi deploy step creation.""" + backend_instance = HitachiBackend() + backend_name = "hitachi-backend-1" + model = "openstack" + + config = HitachiConfig( + name=backend_name, + hitachi_storage_id="123456", + hitachi_pools="pool1", + san_ip="192.168.1.100", + protocol="FC", + san_username="testuser", + san_password="testpassword", + ) + + step = HitachiDeployStep( + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + backend_name, + config, + backend_instance, + model, + ) + + # Test that step was created successfully + assert step is not None + assert step.backend_name == backend_name + assert step.backend_instance == backend_instance + + def test_hitachi_destroy_step_creation( + self, mock_deployment, mock_client, mock_tfhelper, mock_jhelper, mock_manifest + ): + """Test Hitachi destroy step creation.""" + backend_instance = HitachiBackend() + backend_name = "hitachi-backend-1" + model = "openstack" + + step = HitachiDestroyStep( + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + backend_name, + backend_instance, + model, + ) + + # Test that step was created successfully + assert step is not None + assert step.backend_name == backend_name + assert step.backend_instance == backend_instance + + def test_hitachi_update_config_step_creation(self, mock_deployment): + """Test Hitachi update config step creation.""" + backend_instance = HitachiBackend() + backend_name = "hitachi-backend-1" + config_updates = {"san-ip": "192.168.1.101"} + + step = HitachiUpdateConfigStep( + mock_deployment, backend_instance, backend_name, config_updates + ) + + # Test that step was created successfully + assert step is not None + assert step.backend_name == backend_name + assert step.backend_instance == backend_instance + assert step.config_updates == config_updates diff --git a/sunbeam-python/tests/unit/sunbeam/storage/test_models.py b/sunbeam-python/tests/unit/sunbeam/storage/test_models.py new file mode 100644 index 000000000..e56fea157 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/storage/test_models.py @@ -0,0 +1,170 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for storage backend models.""" + +import pytest +from pydantic import ValidationError + +from sunbeam.storage.models import ( + BackendAlreadyExistsException, + BackendNotFoundException, + BackendValidationException, + StorageBackendConfig, + StorageBackendException, + StorageBackendInfo, +) + + +class TestStorageBackendConfig: + """Test cases for StorageBackendConfig model.""" + + def test_valid_config(self): + """Test creating valid configuration.""" + config = StorageBackendConfig(name="test-backend") + assert config.name == "test-backend" + + def test_invalid_config_missing_name(self): + """Test validation error for missing name.""" + with pytest.raises(ValidationError): + StorageBackendConfig() + + def test_config_serialization(self): + """Test configuration serialization.""" + config = StorageBackendConfig(name="test-backend") + data = config.model_dump() + assert data == {"name": "test-backend"} + + def test_config_from_dict(self): + """Test creating configuration from dictionary.""" + data = {"name": "test-backend"} + config = StorageBackendConfig(**data) + assert config.name == "test-backend" + + +class TestStorageBackendInfo: + """Test cases for StorageBackendInfo model.""" + + def test_valid_info(self): + """Test creating valid backend info.""" + info = StorageBackendInfo( + name="test-backend", + backend_type="hitachi", + status="active", + charm="cinder-volume-hitachi", + config={"key": "value"}, + ) + assert info.name == "test-backend" + assert info.backend_type == "hitachi" + assert info.status == "active" + assert info.charm == "cinder-volume-hitachi" + assert info.config == {"key": "value"} + + def test_info_with_defaults(self): + """Test creating info with default values.""" + info = StorageBackendInfo( + name="test-backend", + backend_type="hitachi", + status="active", + charm="cinder-volume-hitachi", + ) + assert info.config == {} + + def test_info_serialization(self): + """Test backend info serialization.""" + info = StorageBackendInfo( + name="test-backend", + backend_type="hitachi", + status="active", + charm="cinder-volume-hitachi", + config={"key": "value"}, + ) + data = info.model_dump() + expected = { + "name": "test-backend", + "backend_type": "hitachi", + "status": "active", + "charm": "cinder-volume-hitachi", + "config": {"key": "value"}, + } + assert data == expected + + def test_info_from_dict(self): + """Test creating backend info from dictionary.""" + data = { + "name": "test-backend", + "backend_type": "hitachi", + "status": "active", + "charm": "cinder-volume-hitachi", + "config": {"key": "value"}, + } + info = StorageBackendInfo(**data) + assert info.name == "test-backend" + assert info.backend_type == "hitachi" + assert info.status == "active" + assert info.charm == "cinder-volume-hitachi" + assert info.config == {"key": "value"} + + +class TestStorageBackendExceptions: + """Test cases for storage backend exceptions.""" + + def test_storage_backend_exception(self): + """Test base storage backend exception.""" + exc = StorageBackendException("Test error") + assert str(exc) == "Test error" + assert isinstance(exc, Exception) + + def test_backend_not_found_exception(self): + """Test backend not found exception.""" + exc = BackendNotFoundException("Backend not found") + assert str(exc) == "Backend not found" + assert isinstance(exc, StorageBackendException) + + def test_backend_already_exists_exception(self): + """Test backend already exists exception.""" + exc = BackendAlreadyExistsException("Backend already exists") + assert str(exc) == "Backend already exists" + assert isinstance(exc, StorageBackendException) + + def test_backend_validation_exception(self): + """Test backend validation exception.""" + exc = BackendValidationException("Validation failed") + assert str(exc) == "Validation failed" + assert isinstance(exc, StorageBackendException) + + def test_exception_inheritance(self): + """Test exception inheritance hierarchy.""" + # All custom exceptions should inherit from StorageBackendException + exceptions = [ + BackendNotFoundException("test"), + BackendAlreadyExistsException("test"), + BackendValidationException("test"), + ] + + for exc in exceptions: + assert isinstance(exc, StorageBackendException) + assert isinstance(exc, Exception) + + def test_exception_with_no_message(self): + """Test exceptions with no message.""" + exc = StorageBackendException() + assert isinstance(exc, Exception) + + exc = BackendNotFoundException() + assert isinstance(exc, StorageBackendException) + + def test_exception_chaining(self): + """Test exception chaining.""" + original = ValueError("Original error") + chained = None + try: + raise original + except ValueError as e: + try: + raise StorageBackendException("Chained error") from e + except StorageBackendException as chained_exc: + chained = chained_exc + + assert str(chained) == "Chained error" + assert chained.__cause__ is original diff --git a/sunbeam-python/tests/unit/sunbeam/storage/test_registry.py b/sunbeam-python/tests/unit/sunbeam/storage/test_registry.py new file mode 100644 index 000000000..9db18185f --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/storage/test_registry.py @@ -0,0 +1,504 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for storage backend registry.""" + +from unittest.mock import Mock, patch + +import pytest + +from sunbeam.storage.base import StorageBackendBase +from sunbeam.storage.models import StorageBackendConfig +from sunbeam.storage.registry import StorageBackendRegistry + + +class MockStorageBackend(StorageBackendBase): + """Mock storage backend for testing.""" + + name = "mock" + display_name = "Mock Storage" + description = "Mock storage backend for testing" + terraform_plan_location = "/path/to/mock/terraform" + + def __init__(self, deployment=None): + super().__init__() + self.deployment = deployment + + def _get_default_config(self) -> dict: + """Get default configuration for mock backend.""" + return {"mock_key": "default_value"} + + def prompt_for_config(self) -> dict: + """Mock prompt for config.""" + return {"mock_key": "mock_value"} + + @property + def tfvar_config_key(self): + """Config key for storing Terraform variables in clusterd.""" + return f"TerraformVars{self.name.title()}Backend" + + @property + def config_class(self): + return StorageBackendConfig + + def create_deploy_step( + self, + deployment, + client, + tfhelper, + jhelper, + manifest, + backend_name, + backend_config, + model, + ): + """Create a mock deployment step.""" + # Use test-specific mock step if available; otherwise base class + try: + from sunbeam.storage.steps import MockDeployStep # type: ignore + + return MockDeployStep( + deployment, + client, + tfhelper, + jhelper, + manifest, + backend_name, + backend_config, + self, + model, + ) + except Exception: + from sunbeam.storage.steps import BaseStorageBackendDeployStep + + return BaseStorageBackendDeployStep( + deployment, + client, + tfhelper, + jhelper, + manifest, + backend_name, + backend_config, + self, + model, + ) + + def create_destroy_step( + self, deployment, client, tfhelper, jhelper, manifest, backend_name, model + ): + """Create a mock destruction step.""" + # Use test-specific mock step if available; otherwise base class + try: + from sunbeam.storage.steps import MockDestroyStep # type: ignore + + return MockDestroyStep( + deployment, + client, + tfhelper, + jhelper, + manifest, + backend_name, + self, + model, + ) + except Exception: + from sunbeam.storage.steps import BaseStorageBackendDestroyStep + + return BaseStorageBackendDestroyStep( + deployment, + client, + tfhelper, + jhelper, + manifest, + backend_name, + self, + model, + ) + + def create_update_config_step(self, deployment, backend_name, config_updates): + """Create a mock configuration update step.""" + from sunbeam.storage.steps import BaseStorageBackendConfigUpdateStep + + return BaseStorageBackendConfigUpdateStep( + deployment, self, backend_name, config_updates + ) + + def prompt_for_config(self, backend_name: str) -> StorageBackendConfig: # noqa: F811 + """Mock prompt for configuration.""" + return StorageBackendConfig(name=backend_name) + + def register_add_cli(self, add): + """Mock CLI registration.""" + pass + + def register_cli( + self, remove, config_show, config_set, config_reset, config_options, deployment + ): + """Mock CLI registration.""" + pass + + def get_terraform_variables( + self, backend_name: str, config: StorageBackendConfig, model: str + ): + return { + "model": model, + "mock_backends": { + backend_name: { + "backend_type": self.name, + "charm_name": self.charm_name, + "charm_channel": "latest/stable", + "backend_config": config.model_dump(), + "backend_endpoint": "storage-backend", + "units": 1, + "additional_integrations": {}, + } + }, + } + + def commands(self): + """Return mock commands for testing.""" + return { + "add": [{"name": "mock", "command": Mock()}], + "remove": [{"name": "mock", "command": Mock()}], + "list": [{"name": "mock", "command": Mock()}], + "config": [{"name": "mock", "command": Mock()}], + } + + +class TestStorageBackendRegistry: + """Test cases for StorageBackendRegistry class.""" + + def test_init(self): + """Test registry initialization.""" + registry = StorageBackendRegistry() + + assert registry._backends == {} + assert hasattr(registry, "_loaded") + assert registry._loaded is False + + def test_load_backends_success(self): + """Test successful backend loading by directly adding a backend.""" + registry = StorageBackendRegistry() + + # Directly add a mock backend to test registry functionality + mock_backend = MockStorageBackend() + registry._backends["mock"] = mock_backend + registry._loaded = True + + assert registry._loaded is True + assert "mock" in registry._backends + assert isinstance(registry._backends["mock"], MockStorageBackend) + + @patch("sunbeam.storage.registry.pathlib.Path") + def test_load_backends_no_backends_dir(self, mock_path): + """Test loading when backends directory doesn't exist.""" + registry = StorageBackendRegistry() + + # Mock the pathlib.Path chain: + # pathlib.Path(sunbeam.storage.backends.__file__).parent + mock_backends_dir = Mock() + mock_backends_dir.iterdir.return_value = [] # Empty for no backends case + + mock_path.return_value.parent = mock_backends_dir + + registry._load_backends() + + assert registry._loaded is True + assert registry._backends == {} + + @patch("sunbeam.storage.registry.pathlib.Path") + def test_load_backends_empty_dir(self, mock_path): + """Test backend loading with empty backends directory.""" + registry = StorageBackendRegistry() + + # Mock the pathlib.Path chain: + # pathlib.Path(sunbeam.storage.backends.__file__).parent + mock_backends_dir = Mock() + mock_backends_dir.iterdir.return_value = [] + + mock_path.return_value.parent = mock_backends_dir + + registry._load_backends() + + assert registry._loaded is True + assert registry._backends == {} + + def test_get_backend_success(self): + """Test successful backend retrieval.""" + registry = StorageBackendRegistry() + mock_backend = MockStorageBackend() + registry._backends = {"mock": mock_backend} + registry._loaded = True + + backend = registry.get_backend("mock") + assert backend == mock_backend + + def test_get_backend_not_found(self): + """Test backend retrieval for non-existent backend.""" + registry = StorageBackendRegistry() + registry._backends = {} + registry._loaded = True + + with pytest.raises(ValueError, match="Storage backend 'nonexistent' not found"): + registry.get_backend("nonexistent") + + def test_get_backend_auto_load(self): + """Test that get_backend automatically loads backends if not loaded.""" + registry = StorageBackendRegistry() + + with patch.object(registry, "_load_backends") as mock_load: + mock_backend = MockStorageBackend() + registry._backends = {"mock": mock_backend} + registry._loaded = True + + backend = registry.get_backend("mock") + mock_load.assert_called_once() + assert backend == mock_backend + + def test_list_backends(self): + """Test backend listing.""" + registry = StorageBackendRegistry() + mock_backend1 = MockStorageBackend() + mock_backend2 = MockStorageBackend() + mock_backend2.name = "mock2" + + registry._backends = {"mock": mock_backend1, "mock2": mock_backend2} + registry._loaded = True + + backends = registry.list_backends() + assert len(backends) == 2 + assert "mock" in backends + assert "mock2" in backends + assert backends["mock"] == mock_backend1 + assert backends["mock2"] == mock_backend2 + + def test_list_backends_auto_load(self): + """Test that list_backends automatically loads backends if not loaded.""" + registry = StorageBackendRegistry() + + with patch.object(registry, "_load_backends") as mock_load: + registry._backends = {"mock": MockStorageBackend()} + registry._loaded = True + + backends = registry.list_backends() + mock_load.assert_called_once() + assert len(backends) == 1 + + def test_register_add_commands_via_backend(self, mock_deployment): + """Test that add commands are registered via backend.register_add_cli().""" + registry = StorageBackendRegistry() + mock_backend = MockStorageBackend() + registry._backends = {"mock": mock_backend} + registry._loaded = True + + mock_storage_group = Mock() + + with patch.object(mock_backend, "register_add_cli") as mock_register_add: + registry.register_cli_commands(mock_storage_group, mock_deployment) + + # Verify that the backend's register_add_cli method was called + mock_register_add.assert_called_once() + # Verify that groups were added to the storage group + assert mock_storage_group.add_command.call_count >= 4 + + def test_register_remove_commands_via_backend(self, mock_deployment): + """Test that remove commands are registered via backend.register_cli().""" + registry = StorageBackendRegistry() + mock_backend = MockStorageBackend() + registry._backends = {"mock": mock_backend} + registry._loaded = True + + mock_storage_group = Mock() + + with patch.object(mock_backend, "register_cli") as mock_register_cli: + registry.register_cli_commands(mock_storage_group, mock_deployment) + + # Verify that the backend's register_cli method was called + mock_register_cli.assert_called_once() + # Verify that groups were added to the storage group + assert mock_storage_group.add_command.call_count >= 4 + + def test_register_list_commands_includes_all(self, mock_deployment): + """Test that list commands include the 'all' command.""" + registry = StorageBackendRegistry() + mock_backend = MockStorageBackend() + registry._backends = {"mock": mock_backend} + registry._loaded = True + + mock_storage_group = Mock() + + with patch("sunbeam.storage.registry.StorageBackendService"): + registry.register_cli_commands(mock_storage_group, mock_deployment) + + # Verify that groups were added to the storage group + assert mock_storage_group.add_command.call_count >= 4 + + def test_register_config_commands_includes_subgroups(self, mock_deployment): + """Test that config commands include show, set, reset, options subgroups.""" + registry = StorageBackendRegistry() + mock_backend = MockStorageBackend() + registry._backends = {"mock": mock_backend} + registry._loaded = True + + mock_storage_group = Mock() + + with patch.object(mock_backend, "register_cli") as mock_register_cli: + registry.register_cli_commands(mock_storage_group, mock_deployment) + + # Verify that the backend's register_cli method was called with config + # subgroups + mock_register_cli.assert_called_once() + # The call should include the config subgroups as arguments + call_args = mock_register_cli.call_args[0] + assert ( + len(call_args) >= 5 + ) # remove_group, config_show, config_set, config_reset, config_options + + def test_register_commands_all_groups(self, mock_deployment): + """Test registration of all command groups.""" + registry = StorageBackendRegistry() + mock_backend = MockStorageBackend() + registry._backends = {"mock": mock_backend} + registry._loaded = True + + mock_cli = Mock() + + with ( + patch.object(mock_backend, "register_add_cli") as mock_add, + patch.object(mock_backend, "register_cli") as mock_register_cli, + ): + registry.register_cli_commands(mock_cli, mock_deployment) + + # Verify that backend CLI registration methods were called + mock_add.assert_called_once() + mock_register_cli.assert_called_once() + + # Verify that the CLI groups were added to the storage group + assert ( + mock_cli.add_command.call_count >= 4 + ) # add, remove, list, config groups + + def test_backend_discovery_error_handling(self): + """Test error handling during backend discovery.""" + registry = StorageBackendRegistry() + + with ( + patch("sunbeam.storage.registry.importlib.import_module"), + patch("sunbeam.storage.registry.pathlib.Path") as mock_path, + ): + mock_backends_dir = Mock() + mock_backends_dir.iterdir.side_effect = Exception("Directory read error") + + mock_path.return_value.parent = mock_backends_dir + + # Registry should handle directory iteration gracefully + registry._load_backends() + + # Should still mark as loaded even with directory errors + assert registry._loaded is True + assert registry._backends == {} + + def test_module_loading_error_handling(self): + """Test error handling during module loading.""" + registry = StorageBackendRegistry() + + with ( + patch("sunbeam.storage.registry.pathlib.Path") as mock_path, + patch( + "sunbeam.storage.registry.importlib.import_module" + ) as mock_import_module, + ): + mock_backends_dir = Mock() + mock_backends_dir.exists.return_value = True + mock_backends_dir.is_dir.return_value = True + + mock_backend_dir = Mock() + mock_backend_dir.name = "mock" + mock_backend_dir.is_dir.return_value = True + + # Set up path operations for mock_backend_dir + mock_backend_module_path = Mock() + mock_backend_module_path.exists.return_value = True + mock_backend_dir.__truediv__ = Mock(return_value=mock_backend_module_path) + + mock_backends_dir.iterdir.return_value = [mock_backend_dir] + + mock_path.return_value.parent = mock_backends_dir + + mock_import_module.side_effect = Exception("Module loading error") + + # Should not raise exception, just log error and continue + registry._load_backends() + + assert registry._loaded is True + assert registry._backends == {} + + def test_singleton_behavior(self): + """Test that registry instances are independent (not singleton).""" + registry1 = StorageBackendRegistry() + registry2 = StorageBackendRegistry() + + # Should be different instances + assert registry1 is not registry2 + assert registry1._backends is not registry2._backends + + def test_backend_validation(self): + """Test backend validation during loading.""" + registry = StorageBackendRegistry() + + # Mock a class that doesn't inherit from StorageBackendBase + class InvalidBackend: + name = "invalid" + + # Directly test validation by adding backends + # Valid backend should be accepted + valid_backend = MockStorageBackend() + registry._backends["mock"] = valid_backend + + # Test that we can access the valid backend + assert len(registry._backends) == 1 + assert "mock" in registry._backends + assert isinstance(registry._backends["mock"], MockStorageBackend) + + def test_command_registration_with_no_backends(self, mock_deployment): + """Test command registration when no backends are loaded.""" + registry = StorageBackendRegistry() + registry._backends = {} + registry._loaded = True + + mock_cli = Mock() + + # Should not raise errors even with no backends + with patch("sunbeam.storage.registry.StorageBackendService"): + registry.register_cli_commands(mock_cli, mock_deployment) + + # Should still create the basic command groups + assert mock_cli.add_command.call_count >= 4 + + def test_command_registration_with_backend_errors(self, mock_deployment): + """Test command registration when backend registration fails.""" + registry = StorageBackendRegistry() + mock_backend = MockStorageBackend() + registry._backends = {"mock": mock_backend} + registry._loaded = True + + mock_cli = Mock() + + with ( + patch.object( + mock_backend, + "register_add_cli", + side_effect=Exception("Registration error"), + ), + patch.object( + mock_backend, + "register_cli", + side_effect=Exception("Registration error"), + ), + patch("sunbeam.storage.registry.StorageBackendService"), + ): + # Should not raise errors even if backend registration fails + registry.register_cli_commands(mock_cli, mock_deployment) + + # Should still create the basic command groups + assert mock_cli.add_command.call_count >= 4 diff --git a/sunbeam-python/tests/unit/sunbeam/storage/test_service.py b/sunbeam-python/tests/unit/sunbeam/storage/test_service.py new file mode 100644 index 000000000..73c52a884 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/storage/test_service.py @@ -0,0 +1,206 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for storage backend service layer.""" + +import pytest + +from sunbeam.clusterd.service import ConfigItemNotFoundException +from sunbeam.storage.models import ( + BackendNotFoundException, +) +from sunbeam.storage.service import StorageBackendService + + +class TestStorageBackendService: + """Test cases for StorageBackendService class.""" + + def test_init(self, mock_deployment): + """Test service initialization.""" + service = StorageBackendService(mock_deployment) + + assert service.deployment == mock_deployment + assert service.model == "admin/openstack" + assert service._tfvar_config_key == "TerraformVarsStorageBackends" + + def test_get_model_name_with_admin_prefix(self, mock_deployment): + """Test model name retrieval when model already has admin prefix.""" + mock_deployment.openstack_machines_model = "admin/openstack" + service = StorageBackendService(mock_deployment) + + assert service.model == "admin/openstack" + + def test_get_model_name_without_admin_prefix(self, mock_deployment): + """Test model name retrieval when missing admin prefix.""" + mock_deployment.openstack_machines_model = "openstack" + service = StorageBackendService(mock_deployment) + + assert service.model == "admin/openstack" + + def test_list_backends_success(self, mock_deployment, sample_clusterd_config): + """Test successful backend listing.""" + import json + from unittest.mock import Mock, patch + + mock_client = mock_deployment.get_client.return_value + # read_config expects JSON string, not dict + mock_client.cluster.get_config.return_value = json.dumps( + sample_clusterd_config["TerraformVarsStorageBackends"] + ) + + # Mock JujuHelper and model status for dynamic status resolution + with patch("sunbeam.storage.service.JujuHelper") as mock_juju_helper: + # Create mock status objects + mock_app_status = Mock() + mock_app_status.app_status.current = "active" + mock_app_status.charm = "ch:amd64/jammy/cinder-volume-hitachi-123" + + mock_model_status = Mock() + # App name is set to backend_name directly in Terraform + mock_model_status.apps = {"test-backend": mock_app_status} + + mock_juju_instance = Mock() + mock_juju_instance.get_model_status.return_value = mock_model_status + mock_juju_helper.return_value = mock_juju_instance + + service = StorageBackendService(mock_deployment) + backends = service.list_backends() + + assert len(backends) == 1 + assert backends[0].name == "test-backend" + assert backends[0].backend_type == "hitachi" + assert backends[0].status == "active" + assert backends[0].charm == "ch:amd64/jammy/cinder-volume-hitachi-123" + + def test_list_backends_no_config(self, mock_deployment): + """Test backend listing when no configuration exists.""" + mock_client = mock_deployment.get_client.return_value + mock_client.cluster.get_config.side_effect = ConfigItemNotFoundException("test") + + service = StorageBackendService(mock_deployment) + backends = service.list_backends() + + assert backends == [] + + def test_list_backends_empty_config(self, mock_deployment): + """Test backend listing with empty configuration.""" + mock_client = mock_deployment.get_client.return_value + mock_client.cluster.get_config.return_value = { + "TerraformVarsStorageBackends": {} + } + + service = StorageBackendService(mock_deployment) + backends = service.list_backends() + + assert backends == [] + + def test_list_backends_no_backends_key(self, mock_deployment): + """Test backend listing when backends key is missing.""" + mock_client = mock_deployment.get_client.return_value + mock_client.cluster.get_config.return_value = { + "TerraformVarsStorageBackends": {"model": "openstack"} + } + + service = StorageBackendService(mock_deployment) + backends = service.list_backends() + + assert backends == [] + + def test_backend_exists_true(self, mock_deployment, sample_clusterd_config): + """Test backend existence check when backend exists.""" + import json + + mock_client = mock_deployment.get_client.return_value + mock_client.cluster.get_config.return_value = json.dumps( + sample_clusterd_config["TerraformVarsStorageBackends"] + ) + + service = StorageBackendService(mock_deployment) + exists = service.backend_exists("test-backend", "hitachi") + + assert exists is True + + def test_backend_exists_false(self, mock_deployment, sample_clusterd_config): + """Test backend existence check when backend doesn't exist.""" + import json + + mock_client = mock_deployment.get_client.return_value + mock_client.cluster.get_config.return_value = json.dumps( + sample_clusterd_config["TerraformVarsStorageBackends"] + ) + + service = StorageBackendService(mock_deployment) + result = service.backend_exists("nonexistent-backend", "hitachi") + + assert result is False + + def test_backend_exists_no_config(self, mock_deployment): + """Test backend existence check when no configuration exists.""" + from sunbeam.clusterd.service import ConfigItemNotFoundException + + mock_client = mock_deployment.get_client.return_value + mock_client.cluster.get_config.side_effect = ConfigItemNotFoundException( + "Config not found" + ) + + service = StorageBackendService(mock_deployment) + result = service.backend_exists("test-backend", "hitachi") + + assert result is False + + def test_get_backend_config_success(self, mock_deployment, sample_clusterd_config): + """Test successful backend config retrieval.""" + import json + + mock_client = mock_deployment.get_client.return_value + mock_client.cluster.get_config.return_value = json.dumps( + sample_clusterd_config["TerraformVarsStorageBackends"] + ) + + service = StorageBackendService(mock_deployment) + config = service.get_backend_config("test-backend", "hitachi") + + assert isinstance(config, dict) + assert "hitachi-storage-id" in config + assert config["hitachi-storage-id"] == "123456" + + def test_get_backend_config_not_found( + self, mock_deployment, sample_clusterd_config + ): + """Test backend config retrieval for non-existent backend.""" + import json + + mock_client = mock_deployment.get_client.return_value + mock_client.cluster.get_config.return_value = json.dumps( + sample_clusterd_config["TerraformVarsStorageBackends"] + ) + + service = StorageBackendService(mock_deployment) + + with pytest.raises(BackendNotFoundException): + service.get_backend_config("nonexistent-backend", "hitachi") + + def test_get_backend_config_no_config(self, mock_deployment): + """Test backend config retrieval when no configuration exists.""" + from sunbeam.clusterd.service import ConfigItemNotFoundException + + mock_client = mock_deployment.get_client.return_value + mock_client.cluster.get_config.side_effect = ConfigItemNotFoundException( + "Config not found" + ) + + service = StorageBackendService(mock_deployment) + + with pytest.raises(BackendNotFoundException): + service.get_backend_config("test-backend", "hitachi") + + def test_error_handling_client_exception(self, mock_deployment): + """Test error handling when client raises exception.""" + mock_client = mock_deployment.get_client.return_value + mock_client.cluster.get_config.side_effect = Exception("Client error") + + service = StorageBackendService(mock_deployment) + + # The service logs the error but returns empty list instead of raising exception + backends = service.list_backends() + assert backends == [] diff --git a/sunbeam-python/tests/unit/sunbeam/storage/test_storage_base.py b/sunbeam-python/tests/unit/sunbeam/storage/test_storage_base.py new file mode 100644 index 000000000..e4e3edde6 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/storage/test_storage_base.py @@ -0,0 +1,296 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for storage backend base classes.""" + +from pathlib import Path +from unittest.mock import Mock, patch + +from sunbeam.storage.base import StorageBackendBase +from sunbeam.storage.models import StorageBackendConfig +from sunbeam.storage.service import StorageBackendService + + +class MockStorageBackend(StorageBackendBase): + """Mock storage backend for testing.""" + + name = "mock" + display_name = "Mock Storage Backend" + charm_name = "mock-charm" + + def __init__(self): + super().__init__() + + @property + def tfvar_config_key(self): + """Config key for storing Terraform variables in clusterd.""" + return f"TerraformVars{self.name.title()}Backend" + + @property + def config_class(self): + return StorageBackendConfig + + def get_terraform_variables( + self, backend_name: str, config: StorageBackendConfig, model: str + ): + return { + "model": model, + "mock_backends": { + backend_name: { + "backend_type": self.name, + "charm_name": self.charm_name, + "charm_channel": self.charm_channel, + "backend_config": config.model_dump(), + "backend_endpoint": self.backend_endpoint, + "units": self.units, + "additional_integrations": self.additional_integrations, + } + }, + } + + def commands(self, conditions=None): + """Return command mapping for this backend.""" + return {} + + def prompt_for_config(self, backend_name: str): + """Mock implementation of prompt_for_config.""" + return StorageBackendConfig(name=backend_name) + + def create_deploy_step( + self, + deployment, + client, + tfhelper, + jhelper, + manifest, + backend_name, + config, + model, + ): + """Create a deployment step for this backend.""" + from sunbeam.storage.steps import BaseStorageBackendDeployStep + + class MockDeployStep(BaseStorageBackendDeployStep): + def get_terraform_variables(self): + return self.backend_instance.get_terraform_variables( + self.backend_name, self.backend_config, self.model + ) + + return MockDeployStep( + deployment, + client, + tfhelper, + jhelper, + manifest, + backend_name, + config, + self, + model, + ) + + def create_destroy_step( + self, deployment, client, tfhelper, jhelper, manifest, backend_name, model + ): + """Create a destruction step for this backend.""" + from sunbeam.storage.steps import BaseStorageBackendDestroyStep + + class MockDestroyStep(BaseStorageBackendDestroyStep): + def get_terraform_variables(self): + return self.backend_instance.get_terraform_variables( + self.backend_name, + StorageBackendConfig(name=self.backend_name), + self.model, + ) + + return MockDestroyStep( + deployment, client, tfhelper, jhelper, manifest, backend_name, self, model + ) + + def create_update_config_step(self, deployment, backend_name, config_updates): + """Create a configuration update step for this backend.""" + from sunbeam.storage.steps import BaseStorageBackendConfigUpdateStep + + return BaseStorageBackendConfigUpdateStep( + deployment, self, backend_name, config_updates + ) + + def register_add_cli(self, add): + """Mock CLI registration.""" + pass + + def register_cli( + self, remove, config_show, config_set, config_reset, config_options, deployment + ): + """Mock CLI registration.""" + pass + + def _get_default_config(self) -> StorageBackendConfig: + """Get a default configuration instance for comparison.""" + return StorageBackendConfig(name="default") + + +class TestStorageBackendBase: + """Test cases for StorageBackendBase class.""" + + def test_init(self): + """Test backend initialization.""" + backend = MockStorageBackend() + + assert backend.name == "mock" + assert backend.display_name == "Mock Storage Backend" + assert backend.charm_name == "mock-charm" + assert backend.tfplan == "storage-backend-plan" + assert backend.tfplan_dir == "deploy-storage-backend" + assert backend.charm_channel == "stable" + assert backend.charm_base == "ubuntu@22.04" + assert backend.backend_endpoint == "cinder-volume" + assert backend.units == 1 + assert backend.additional_integrations == [] + + def test_config_class(self): + """Test configuration class retrieval.""" + backend = MockStorageBackend() + config_class = backend.config_class + assert config_class == StorageBackendConfig + + def test_get_terraform_variables(self): + """Test Terraform variables generation.""" + backend = MockStorageBackend() + config = StorageBackendConfig(name="test-backend") + + variables = backend.get_terraform_variables("test-backend", config, "openstack") + + assert "model" in variables + assert "mock_backends" in variables + assert variables["model"] == "openstack" + assert "test-backend" in variables["mock_backends"] + + backend_config = variables["mock_backends"]["test-backend"] + assert backend_config["backend_type"] == "mock" + assert backend_config["charm_name"] == "mock-charm" + assert backend_config["charm_channel"] == "stable" + assert backend_config["backend_endpoint"] == "cinder-volume" + assert backend_config["units"] == 1 + + # todo: fix this implementation. the get service is not part of the base class + @patch("sunbeam.storage.base.StorageBackendService") + def test_get_service(self, mock_service_class, mock_deployment): + """Test service creation and caching.""" + backend = MockStorageBackend() + mock_service = Mock(spec=StorageBackendService) + mock_service_class.return_value = mock_service + + # First call should create service + service1 = backend._get_service(mock_deployment) + assert service1 == mock_service + mock_service_class.assert_called_once_with(mock_deployment) + + # Second call should return cached service + service2 = backend._get_service(mock_deployment) + assert service2 == mock_service + # Should not call constructor again + assert mock_service_class.call_count == 1 + + def test_prompt_for_config(self, mock_deployment): + """Test configuration prompting (base implementation).""" + backend = MockStorageBackend() + + # Base implementation should return empty config + config = backend.prompt_for_config("test-backend") + assert isinstance(config, StorageBackendConfig) + assert config.name == "test-backend" + + def test_abstract_methods_not_implemented(self): + """Test that abstract methods raise NotImplementedError in base class.""" + # This test verifies that StorageBackendBase is properly abstract + # We can't instantiate it directly, but we can test that our mock + # implements required methods + backend = MockStorageBackend() + + # These methods should be implemented in the mock + assert hasattr(backend, "config_class") + assert hasattr(backend, "get_terraform_variables") + assert callable(backend.config_class) + assert callable(backend.get_terraform_variables) + + def test_version_property(self): + """Test version property.""" + backend = MockStorageBackend() + assert hasattr(backend, "version") + # Version should be set in base class + assert str(backend.version) == "0.0.1" + + def test_tf_plan_location_property(self): + """Test Terraform plan location property.""" + backend = MockStorageBackend() + assert backend.tf_plan_location == "FEATURE_REPO" + + def test_user_manifest_property(self): + """Test user manifest property.""" + backend = MockStorageBackend() + assert backend.user_manifest is None + + @patch("sunbeam.storage.base.Path") + def test_tfplan_path_property(self, mock_path): + """Test Terraform plan path property.""" + backend = MockStorageBackend() + mock_path.return_value = Path("/test/path") + + # This would test the tfplan_path property if it exists + # The actual implementation may vary + assert hasattr(backend, "tfplan_dir") + assert backend.tfplan_dir == "deploy-storage-backend" + + def test_backend_attributes(self): + """Test that all required backend attributes are set.""" + backend = MockStorageBackend() + + # Test required string attributes + required_attrs = ["name", "display_name", "charm_name", "tfplan", "tfplan_dir"] + for attr in required_attrs: + assert hasattr(backend, attr) + assert isinstance(getattr(backend, attr), str) + assert getattr(backend, attr) != "" + + # Test optional attributes + optional_attrs = [ + "charm_channel", + "charm_revision", + "charm_base", + "backend_endpoint", + ] + for attr in optional_attrs: + assert hasattr(backend, attr) + + # Test integer attributes + assert hasattr(backend, "units") + assert isinstance(backend.units, int) + assert backend.units > 0 + + # Test list attributes + assert hasattr(backend, "additional_integrations") + assert isinstance(backend.additional_integrations, list) + + def test_service_caching_with_different_deployments(self, mock_deployment): + """Test service caching behavior with different deployment objects.""" + backend = MockStorageBackend() + + with patch("sunbeam.storage.base.StorageBackendService") as mock_service_class: + mock_service1 = Mock(spec=StorageBackendService) + mock_service_class.return_value = mock_service1 + + # First deployment + service1 = backend._get_service(mock_deployment) + assert service1 == mock_service1 + + # Different deployment object should return same cached service + mock_deployment2 = Mock() + service2 = backend._get_service(mock_deployment2) + assert service2 == mock_service1 # Same service is cached and reused + + # Same deployment should return cached service + service1_cached = backend._get_service(mock_deployment) + assert service1_cached == mock_service1 + + # Should have called constructor only once (service is cached) + assert mock_service_class.call_count == 1 diff --git a/sunbeam-python/tests/unit/sunbeam/storage/test_storage_steps.py b/sunbeam-python/tests/unit/sunbeam/storage/test_storage_steps.py new file mode 100644 index 000000000..a171cc6d2 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/storage/test_storage_steps.py @@ -0,0 +1,594 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for Terraform-based storage backend step classes.""" + +from unittest.mock import Mock, patch + +from sunbeam.core.common import ResultType +from sunbeam.storage.base import StorageBackendBase +from sunbeam.storage.models import StorageBackendConfig +from sunbeam.storage.steps import ( + BaseStorageBackendConfigUpdateStep, + BaseStorageBackendDeployStep, + BaseStorageBackendDestroyStep, +) + + +class MockStorageBackend(StorageBackendBase): + """Mock storage backend for testing.""" + + name = "mock" + display_name = "Mock Storage" + description = "Mock storage backend for testing" + terraform_plan_location = "/path/to/mock/terraform" + charm_name = "mock-charm" + + def __init__(self, deployment=None): + super().__init__() + self.deployment = deployment + + def _get_default_config(self) -> dict: + """Get default configuration for mock backend.""" + return {"mock_key": "default_value"} + + def prompt_for_config(self) -> dict: + """Mock prompt for config.""" + return {"mock_key": "mock_value"} + + @property + def tfvar_config_key(self): + """Config key for storing Terraform variables in clusterd.""" + return f"TerraformVars{self.name.title()}Backend" + + def register_add_cli(self, add): + """Mock CLI registration.""" + pass + + def register_cli( + self, remove, config_show, config_set, config_reset, config_options, deployment + ): + """Mock CLI registration.""" + pass + + def get_terraform_variables(self, backend_name, config, model): + """Mock Terraform variables.""" + return { + "model": model, + "mock_backends": { + backend_name: { + "backend_type": self.name, + "charm_name": self.charm_name, + "charm_channel": "latest/stable", + "backend_config": config.model_dump(), + "backend_endpoint": "storage-backend", + "units": 1, + "additional_integrations": {}, + } + }, + } + + def create_deploy_step( + self, + deployment, + client, + tfhelper, + jhelper, + manifest, + backend_name, + backend_config, + model, + ): + """Create a mock deployment step.""" + return MockDeployStep( + deployment, + client, + tfhelper, + jhelper, + manifest, + backend_name, + backend_config, + self, + model, + ) + + def create_destroy_step( + self, deployment, client, tfhelper, jhelper, manifest, backend_name, model + ): + """Create a mock destruction step.""" + return MockDestroyStep( + deployment, client, tfhelper, jhelper, manifest, backend_name, self, model + ) + + def create_update_config_step(self, deployment, backend_name, config_updates): + """Create a mock configuration update step.""" + return BaseStorageBackendConfigUpdateStep( + deployment, self, backend_name, config_updates + ) + + def prompt_for_config(self, backend_name: str) -> StorageBackendConfig: # noqa: F811 + """Mock prompt for configuration.""" + return StorageBackendConfig(name=backend_name) + + def get_field_mapping(self): + """Return field mapping for mock backend.""" + return {} + + def commands(self): + """Return commands for mock backend.""" + return [] + + +class MockDeployStep(BaseStorageBackendDeployStep): + """Concrete implementation of BaseStorageBackendDeployStep for testing.""" + + def get_terraform_variables(self) -> dict: + """Get terraform variables for deployment.""" + return self.backend_instance.get_terraform_variables( + self.backend_name, self.backend_config, self.model + ) + + +class MockDestroyStep(BaseStorageBackendDestroyStep): + """Concrete implementation of BaseStorageBackendDestroyStep for testing.""" + + pass + + +class TestBaseStorageBackendDeployStep: + """Test cases for BaseStorageBackendDeployStep.""" + + def test_init( + self, mock_deployment, mock_client, mock_tfhelper, mock_jhelper, mock_manifest + ): + """Test step initialization.""" + backend_instance = MockStorageBackend() + backend_name = "test-backend" + backend_config = StorageBackendConfig(name=backend_name) + model = "openstack" + + step = MockDeployStep( + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + backend_name, + backend_config, + backend_instance, + model, + ) + + assert step.deployment == mock_deployment + assert step.client == mock_client + assert step.tfhelper == mock_tfhelper + assert step.jhelper == mock_jhelper + assert step.manifest == mock_manifest + assert step.backend_name == backend_name + assert step.backend_config == backend_config + assert step.backend_instance == backend_instance + assert step.model == model + assert "Deploy Mock Storage backend" in step.name + assert "test-backend" in step.description + + @patch("sunbeam.storage.steps.read_config") + @patch("sunbeam.storage.steps.update_config") + def test_run_success( + self, + mock_update_config, + mock_read_config, + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + ): + """Test successful deployment run.""" + # Mock existing config + mock_read_config.return_value = {"existing": "config"} + + # Mock Terraform operations + mock_tfhelper.update_tfvars_and_apply_tf.return_value = None + + backend_instance = MockStorageBackend() + backend_name = "test-backend" + backend_config = StorageBackendConfig(name=backend_name) + model = "openstack" + + step = MockDeployStep( + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + backend_name, + backend_config, + backend_instance, + model, + ) + + result = step.run() + + assert result.result_type == ResultType.COMPLETED + + # Verify Terraform was called with correct variables + mock_tfhelper.update_tfvars_and_apply_tf.assert_called_once() + call_args = mock_tfhelper.update_tfvars_and_apply_tf.call_args + tfvars = call_args[1]["override_tfvars"] + + assert "model" in tfvars + assert "mock_backends" in tfvars + assert tfvars["model"] == "openstack" + + def test_get_application_timeout( + self, mock_deployment, mock_client, mock_tfhelper, mock_jhelper, mock_manifest + ): + """Test application timeout retrieval.""" + backend_instance = MockStorageBackend() + backend_name = "test-backend" + backend_config = StorageBackendConfig(name=backend_name) + model = "openstack" + + step = MockDeployStep( + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + backend_name, + backend_config, + backend_instance, + model, + ) + + timeout = step.get_application_timeout() + assert timeout == 1200 # Default timeout + + +class TestBaseStorageBackendDestroyStep: + """Test cases for BaseStorageBackendDestroyStep.""" + + def test_init( + self, mock_deployment, mock_client, mock_tfhelper, mock_jhelper, mock_manifest + ): + """Test step initialization.""" + backend_instance = MockStorageBackend() + backend_name = "test-backend" + model = "openstack" + + step = BaseStorageBackendDestroyStep( + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + backend_name, + backend_instance, + model, + ) + + assert step.deployment == mock_deployment + assert step.client == mock_client + assert step.tfhelper == mock_tfhelper + assert step.jhelper == mock_jhelper + assert step.manifest == mock_manifest + assert step.backend_name == backend_name + assert step.backend_instance == backend_instance + assert step.model == model + assert "Destroy Mock Storage backend" in step.name + assert "test-backend" in step.description + + @patch("sunbeam.storage.steps.read_config") + @patch("sunbeam.storage.steps.update_config") + def test_run_success_partial_destroy( + self, + mock_update_config, + mock_read_config, + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + ): + """Test successful destruction run with partial destroy.""" + # Mock existing config with multiple backends + mock_read_config.return_value = { + "mock_backends": { + "test-backend": {"backend_type": "mock"}, + "other-backend": {"backend_type": "hitachi"}, + } + } + + # Mock Terraform operations + mock_tfhelper.apply.return_value = None + + backend_instance = MockStorageBackend() + backend_name = "test-backend" + model = "openstack" + + step = BaseStorageBackendDestroyStep( + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + backend_name, + backend_instance, + model, + ) + + # Mock should_destroy_all_resources to return False + with patch.object(step, "should_destroy_all_resources", return_value=False): + result = step.run() + + assert result.result_type == ResultType.COMPLETED + + # Verify Terraform was called + mock_tfhelper.apply.assert_called_once() + + @patch("sunbeam.storage.steps.read_config") + def test_run_success_full_destroy( + self, + mock_read_config, + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + ): + """Test successful destruction run with full destroy.""" + # Mock existing config with only one backend + mock_read_config.return_value = { + "mock_backends": {"test-backend": {"backend_type": "mock"}} + } + + # Mock Terraform operations + mock_tfhelper.destroy.return_value = None + + backend_instance = MockStorageBackend() + backend_name = "test-backend" + model = "openstack" + + step = BaseStorageBackendDestroyStep( + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + backend_name, + backend_instance, + model, + ) + + # Mock should_destroy_all_resources to return True + with patch.object(step, "should_destroy_all_resources", return_value=True): + result = step.run() + + assert result.result_type == ResultType.COMPLETED + + # Verify Terraform destroy was called + mock_tfhelper.destroy.assert_called_once() + + @patch("sunbeam.storage.steps.read_config") + def test_should_destroy_all_resources_true( + self, + mock_read_config, + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + ): + """Test should_destroy_all_resources logic when only one backend exists.""" + # Mock config with only one backend + mock_read_config.return_value = { + "mock_backends": {"test-backend": {"backend_type": "mock"}} + } + + backend_instance = MockStorageBackend() + backend_name = "test-backend" + model = "openstack" + + step = BaseStorageBackendDestroyStep( + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + backend_name, + backend_instance, + model, + ) + + result = step.should_destroy_all_resources() + assert result is True + + @patch("sunbeam.storage.steps.read_config") + def test_should_destroy_all_resources_false( + self, + mock_read_config, + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + ): + """Test should_destroy_all_resources logic when multiple backends exist.""" + # Mock config with multiple backends + mock_read_config.return_value = { + "mock_backends": { + "test-backend": {"backend_type": "mock"}, + "other-backend": {"backend_type": "hitachi"}, + } + } + + backend_instance = MockStorageBackend() + backend_name = "test-backend" + model = "openstack" + + step = BaseStorageBackendDestroyStep( + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + backend_name, + backend_instance, + model, + ) + + result = step.should_destroy_all_resources() + assert result is False + + def test_get_application_timeout( + self, mock_deployment, mock_client, mock_tfhelper, mock_jhelper, mock_manifest + ): + """Test application timeout retrieval.""" + backend_instance = MockStorageBackend() + backend_name = "test-backend" + model = "openstack" + + step = BaseStorageBackendDestroyStep( + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + backend_name, + backend_instance, + model, + ) + + timeout = step.get_application_timeout() + assert timeout == 1200 # Default timeout + + +class TestBaseStorageBackendConfigUpdateStep: + """Test cases for BaseStorageBackendConfigUpdateStep.""" + + def test_init(self, mock_deployment): + """Test step initialization.""" + backend_instance = MockStorageBackend() + backend_name = "test-backend" + config_updates = {"key1": "value1", "key2": "value2"} + + # Mock deployment methods + mock_client = Mock() + mock_tfhelper = Mock() + mock_deployment.get_client.return_value = mock_client + mock_deployment.get_tfhelper.return_value = mock_tfhelper + + step = BaseStorageBackendConfigUpdateStep( + mock_deployment, backend_instance, backend_name, config_updates + ) + + assert step.deployment == mock_deployment + assert step.backend_instance == backend_instance + assert step.backend_name == backend_name + assert step.config_updates == config_updates + assert step.client == mock_client + assert step.tfhelper == mock_tfhelper + assert "Update Mock Storage backend" in step.name + assert "test-backend" in step.description + + @patch("sunbeam.storage.steps.read_config") + @patch("sunbeam.storage.steps.update_config") + def test_run_update_operation( + self, mock_update_config, mock_read_config, mock_deployment + ): + """Test configuration update operation.""" + # Mock existing config + mock_read_config.return_value = { + "mock_backends": { + "test-backend": {"backend_config": {"existing_key": "existing_value"}} + } + } + + backend_instance = MockStorageBackend() + backend_name = "test-backend" + config_updates = {"key1": "value1", "key2": "value2"} + + # Mock deployment methods + mock_client = Mock() + mock_tfhelper = Mock() + mock_tfhelper.write_tfvars.return_value = None + mock_tfhelper.apply.return_value = None + mock_deployment.get_client.return_value = mock_client + mock_deployment.get_tfhelper.return_value = mock_tfhelper + + step = BaseStorageBackendConfigUpdateStep( + mock_deployment, backend_instance, backend_name, config_updates + ) + + result = step.run() + + assert result.result_type == ResultType.COMPLETED + + # Verify Terraform was called + mock_tfhelper.apply.assert_called_once() + + @patch("sunbeam.storage.steps.read_config") + @patch("sunbeam.storage.steps.update_config") + def test_run_reset_operation( + self, mock_update_config, mock_read_config, mock_deployment + ): + """Test configuration reset operation.""" + # Mock existing config + mock_read_config.return_value = { + "mock_backends": { + "test-backend": {"backend_config": {"key1": "value1", "key2": "value2"}} + } + } + + backend_instance = MockStorageBackend() + backend_name = "test-backend" + config_updates = {"_reset_keys": ["key1"]} + + # Mock deployment methods + mock_client = Mock() + mock_tfhelper = Mock() + mock_tfhelper.write_tfvars.return_value = None + mock_tfhelper.apply.return_value = None + mock_deployment.get_client.return_value = mock_client + mock_deployment.get_tfhelper.return_value = mock_tfhelper + + step = BaseStorageBackendConfigUpdateStep( + mock_deployment, backend_instance, backend_name, config_updates + ) + + result = step.run() + + assert result.result_type == ResultType.COMPLETED + + # Verify Terraform was called + mock_tfhelper.apply.assert_called_once() + + def test_handle_update_operation(self, mock_deployment): + """Test update operation handling.""" + backend_instance = MockStorageBackend() + backend_name = "test-backend" + config_updates = {"key1": "value1", "key2": "value2"} + + step = BaseStorageBackendConfigUpdateStep( + mock_deployment, backend_instance, backend_name, config_updates + ) + + current_config = { + "mock_backends": { + "test-backend": {"charm_config": {"existing_key": "existing_value"}} + } + } + updated_config = step.handle_update_operation(current_config) + + expected_config = { + "mock_backends": { + "test-backend": { + "charm_config": { + "existing_key": "existing_value", + "key1": "value1", + "key2": "value2", + } + } + } + } + assert updated_config == expected_config