diff --git a/cloud/etc/deploy-storage/main.tf b/cloud/etc/deploy-storage/main.tf new file mode 100644 index 000000000..51ceae154 --- /dev/null +++ b/cloud/etc/deploy-storage/main.tf @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +terraform { + required_providers { + juju = { + source = "juju/juju" + version = "= 0.23.1" + } + } +} + +provider "juju" {} + +data "juju_model" "model" { + name = var.model +} + +module "backends" { + for_each = var.backends + + source = "./modules/backend" + + model = data.juju_model.model.uuid + + name = each.key + principal_application = each.value.principal_application + charm_name = each.value.charm_name + charm_base = each.value.charm_base + charm_channel = each.value.charm_channel + charm_revision = each.value.charm_revision + charm_config = each.value.charm_config + endpoint_bindings = each.value.endpoint_bindings + secrets = each.value.secrets +} diff --git a/cloud/etc/deploy-storage/modules/backend/main.tf b/cloud/etc/deploy-storage/modules/backend/main.tf new file mode 100644 index 000000000..6d59d05ec --- /dev/null +++ b/cloud/etc/deploy-storage/modules/backend/main.tf @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +terraform { + required_providers { + juju = { + source = "juju/juju" + } + } +} + +data "juju_model" "model" { + name = var.model +} + +data "juju_application" "cinder-volume" { + name = var.principal_application + model = data.juju_model.model.name +} + +resource "juju_secret" "secret" { + model = data.juju_model.model.name + name = "${var.name}-config-secret" + value = { + for k, v in var.secrets : v => var.charm_config[k] + } +} + +resource "juju_access_secret" "secret-access" { + model = juju_secret.secret.model + secret_id = juju_secret.secret.secret_id + applications = [juju_application.storage-backend.name] +} + +locals { + charm_config = merge( + { volume-backend-name = var.name }, + var.charm_config, + { for k, v in var.secrets : k => juju_secret.secret.secret_uri } + ) +} + +# Deploy Storage backend charms +resource "juju_application" "storage-backend" { + name = var.name + model = data.juju_model.model.uuid + units = 1 + + charm { + name = var.charm_name + channel = var.charm_channel + revision = var.charm_revision + base = var.charm_base + } + + config = local.charm_config + + endpoint_bindings = var.endpoint_bindings +} + +# Integrate Storage backends with cinder-volume +resource "juju_integration" "storage-backend-to-cinder-volume" { + model = data.juju_model.model.name + + application { + name = juju_application.storage-backend.name + endpoint = "cinder-volume" + } + + application { + name = data.juju_application.cinder-volume.name + endpoint = "cinder-volume" + } +} diff --git a/cloud/etc/deploy-storage/modules/backend/variables.tf b/cloud/etc/deploy-storage/modules/backend/variables.tf new file mode 100644 index 000000000..2f2f3f10c --- /dev/null +++ b/cloud/etc/deploy-storage/modules/backend/variables.tf @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +variable "model" { + description = "Name of the machine model to deploy to" + type = string +} + +variable "principal_application" { + description = "Name of the principal application to integrate with" + type = string + default = "cinder-volume" +} + +variable "charm_name" { + description = "Name of the Storage charm" + type = string +} + +variable "charm_base" { + description = "Base for the Storage charm" + type = string + default = "ubuntu@24.04" +} + +variable "charm_channel" { + description = "Operator channel for Storage backend deployment" + type = string + default = "latest/edge" +} + +variable "charm_revision" { + description = "Operator channel revision for Storage backend deployment" + type = number + default = null +} + +variable "name" { + description = "Name of the backend" + type = string +} + +variable "endpoint_bindings" { + description = "Endpoint bindings for the applications" + type = set(map(string)) + default = null +} + +variable "charm_config" { + description = "Operator config for the Storage backend deployment" + type = map(string) + default = {} +} + +variable "secrets" { + description = "Map of secret names to create. The key is the config option name, the value is key to use in the secret dict for the value." + type = map(string) + default = {} +} diff --git a/cloud/etc/deploy-storage/outputs.tf b/cloud/etc/deploy-storage/outputs.tf new file mode 100644 index 000000000..12519b28d --- /dev/null +++ b/cloud/etc/deploy-storage/outputs.tf @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 diff --git a/cloud/etc/deploy-storage/variables.tf b/cloud/etc/deploy-storage/variables.tf new file mode 100644 index 000000000..493261af0 --- /dev/null +++ b/cloud/etc/deploy-storage/variables.tf @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +variable "model" { + description = "UUID of the machine model to deploy to" + type = string +} + +variable "backends" { + description = "Map of storage backend configurations" + type = map(object({ + principal_application = string + charm_name = string + charm_base = string + charm_channel = string + charm_revision = number + charm_config = map(string) + endpoint_bindings = set(map(string)) + secrets = map(string) + })) + default = {} +} diff --git a/sunbeam-microcluster/api/apitypes/storage_backends.go b/sunbeam-microcluster/api/apitypes/storage_backends.go new file mode 100644 index 000000000..92c12a3a7 --- /dev/null +++ b/sunbeam-microcluster/api/apitypes/storage_backends.go @@ -0,0 +1,19 @@ +// Package apitypes provides shared types and structs. +package apitypes + +// StorageBackends holds list of StorageBackend type +type StorageBackends []StorageBackend + +// StorageBackend structure to hold storage backend details like name and type +type StorageBackend struct { + // Name of the storage backend + Name string `json:"name" yaml:"name"` + // Type of the storage backend + Type string `json:"type" yaml:"type"` + // Config holds backend specific configuration as a json blob + Config string `json:"config" yaml:"config"` + // Name of the principal application this storage backend is associated with + Principal string `json:"principal" yaml:"principal"` + // ModelUUID is the juju model UUID where this storage backend is deployed + ModelUUID string `json:"model-uuid" yaml:"model-uuid"` +} diff --git a/sunbeam-microcluster/api/servers.go b/sunbeam-microcluster/api/servers.go index df657d8d2..7e5ad33c6 100644 --- a/sunbeam-microcluster/api/servers.go +++ b/sunbeam-microcluster/api/servers.go @@ -29,6 +29,8 @@ var Servers = map[string]rest.Server{ manifestsCmd, manifestCmd, statusCmd, + storageBackendsCmd, + storageBackendCmd, }, }, { diff --git a/sunbeam-microcluster/api/storage_backends.go b/sunbeam-microcluster/api/storage_backends.go new file mode 100644 index 000000000..2a96738fd --- /dev/null +++ b/sunbeam-microcluster/api/storage_backends.go @@ -0,0 +1,117 @@ +package api + +import ( + "encoding/json" + "net/http" + "net/url" + + "github.com/canonical/lxd/lxd/response" + "github.com/canonical/lxd/shared/api" + "github.com/canonical/microcluster/v2/rest" + "github.com/canonical/microcluster/v2/state" + "github.com/gorilla/mux" + + "github.com/canonical/snap-openstack/sunbeam-microcluster/access" + "github.com/canonical/snap-openstack/sunbeam-microcluster/api/apitypes" + "github.com/canonical/snap-openstack/sunbeam-microcluster/sunbeam" +) + +// /1.0/storage-backend endpoint. +var storageBackendsCmd = rest.Endpoint{ + Path: "storage-backend", + + Get: access.ClusterCATrustedEndpoint(cmdStorageBackendsGetAll, true), + Post: access.ClusterCATrustedEndpoint(cmdStorageBackendsPost, true), +} + +// /1.0/storage-backend/ endpoint. +var storageBackendCmd = rest.Endpoint{ + Path: "storage-backend/{backendname}", + + Get: access.ClusterCATrustedEndpoint(cmdStorageBackendGet, true), + Delete: access.ClusterCATrustedEndpoint(cmdStorageBackendDelete, true), + Put: access.ClusterCATrustedEndpoint(cmdStorageBackendPut, true), +} + +func cmdStorageBackendsGetAll(s state.State, r *http.Request) response.Response { + + storageBackends, err := sunbeam.ListStorageBackends(r.Context(), s) + if err != nil { + return response.InternalError(err) + } + + return response.SyncResponse(true, storageBackends) +} + +func cmdStorageBackendsPost(s state.State, r *http.Request) response.Response { + var req apitypes.StorageBackend + + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + return response.InternalError(err) + } + + err = sunbeam.AddStorageBackend(r.Context(), s, req.Name, req.Type, req.Principal, req.ModelUUID, req.Config) + if err != nil { + return response.InternalError(err) + } + + return response.EmptySyncResponse +} + +func cmdStorageBackendGet(s state.State, r *http.Request) response.Response { + var backendName string + backendName, err := url.PathUnescape(mux.Vars(r)["backendname"]) + if err != nil { + return response.InternalError(err) + } + backend, err := sunbeam.GetStorageBackend(r.Context(), s, backendName) + if err != nil { + if err, ok := err.(api.StatusError); ok { + if err.Status() == http.StatusNotFound { + return response.NotFound(err) + } + } + return response.InternalError(err) + } + + return response.SyncResponse(true, backend) +} + +func cmdStorageBackendDelete(s state.State, r *http.Request) response.Response { + backendName, err := url.PathUnescape(mux.Vars(r)["backendname"]) + if err != nil { + return response.SmartError(err) + } + err = sunbeam.DeleteStorageBackend(r.Context(), s, backendName) + if err != nil { + if err, ok := err.(api.StatusError); ok { + if err.Status() == http.StatusNotFound { + return response.NotFound(err) + } + } + return response.InternalError(err) + } + + return response.EmptySyncResponse +} + +func cmdStorageBackendPut(s state.State, r *http.Request) response.Response { + backendName, err := url.PathUnescape(mux.Vars(r)["backendname"]) + if err != nil { + return response.SmartError(err) + } + + var req apitypes.StorageBackend + err = json.NewDecoder(r.Body).Decode(&req) + if err != nil { + return response.InternalError(err) + } + + err = sunbeam.UpdateStorageBackend(r.Context(), s, backendName, req.Type, req.Config, req.Principal, req.ModelUUID) + if err != nil { + return response.InternalError(err) + } + + return response.EmptySyncResponse +} diff --git a/sunbeam-microcluster/database/schema.go b/sunbeam-microcluster/database/schema.go index 75235b591..1c9924100 100644 --- a/sunbeam-microcluster/database/schema.go +++ b/sunbeam-microcluster/database/schema.go @@ -16,6 +16,7 @@ var SchemaExtensions = []schema.Update{ JujuUserSchemaUpdate, ManifestsSchemaUpdate, AddSystemIDToNodes, + StorageBackendSchemaUpdate, } // NodesSchemaUpdate is schema for table nodes @@ -96,3 +97,21 @@ ALTER TABLE nodes ADD COLUMN system_id TEXT default ''; return err } + +// StorageBackendSchemaUpdate is schema for table storage_backends +func StorageBackendSchemaUpdate(_ context.Context, tx *sql.Tx) error { + stmt := ` +CREATE TABLE storage_backends ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT NOT NULL, + type TEXT NOT NULL, + principal TEXT, + model_uuid TEXT, + config TEXT, + UNIQUE(name) +); + ` + + _, err := tx.Exec(stmt) + return err +} diff --git a/sunbeam-microcluster/database/storage_backend.go b/sunbeam-microcluster/database/storage_backend.go new file mode 100644 index 000000000..1e4d02392 --- /dev/null +++ b/sunbeam-microcluster/database/storage_backend.go @@ -0,0 +1,40 @@ +package database + +//go:generate -command mapper lxd-generate db mapper -t storage_backend.mapper.go +//go:generate mapper reset +// +//go:generate mapper stmt -d github.com/canonical/microcluster/v2/cluster -e StorageBackend objects table=storage_backends +//go:generate mapper stmt -d github.com/canonical/microcluster/v2/cluster -e StorageBackend objects-by-Name table=storage_backends +//go:generate mapper stmt -d github.com/canonical/microcluster/v2/cluster -e StorageBackend objects-by-Type table=storage_backends +//go:generate mapper stmt -d github.com/canonical/microcluster/v2/cluster -e StorageBackend objects-by-Principal table=storage_backends +//go:generate mapper stmt -d github.com/canonical/microcluster/v2/cluster -e StorageBackend objects-by-ModelUUID table=storage_backends +//go:generate mapper stmt -d github.com/canonical/microcluster/v2/cluster -e StorageBackend id table=storage_backends +//go:generate mapper stmt -d github.com/canonical/microcluster/v2/cluster -e StorageBackend create table=storage_backends +//go:generate mapper stmt -d github.com/canonical/microcluster/v2/cluster -e StorageBackend delete-by-Name table=storage_backends +//go:generate mapper stmt -d github.com/canonical/microcluster/v2/cluster -e StorageBackend update table=storage_backends +// +//go:generate mapper method -i -d github.com/canonical/microcluster/v2/cluster -e StorageBackend GetMany +//go:generate mapper method -i -d github.com/canonical/microcluster/v2/cluster -e StorageBackend GetOne +//go:generate mapper method -i -d github.com/canonical/microcluster/v2/cluster -e StorageBackend ID +//go:generate mapper method -i -d github.com/canonical/microcluster/v2/cluster -e StorageBackend Exists +//go:generate mapper method -i -d github.com/canonical/microcluster/v2/cluster -e StorageBackend Create +//go:generate mapper method -i -d github.com/canonical/microcluster/v2/cluster -e StorageBackend DeleteOne-by-Name +//go:generate mapper method -i -d github.com/canonical/microcluster/v2/cluster -e StorageBackend Update + +// StorageBackend is used to track StorageBackend information. +type StorageBackend struct { + ID int + Name string `db:"primary=yes"` + Type string + Config string + Principal string + ModelUUID string +} + +// StorageBackendFilter is a required struct for use with lxd-generate. It is used for filtering fields on database fetches. +type StorageBackendFilter struct { + Name *string + Type *string + Principal *string + ModelUUID *string +} diff --git a/sunbeam-microcluster/database/storage_backend.mapper.go b/sunbeam-microcluster/database/storage_backend.mapper.go new file mode 100644 index 000000000..231c0758e --- /dev/null +++ b/sunbeam-microcluster/database/storage_backend.mapper.go @@ -0,0 +1,422 @@ +package database + +// The code below was generated by lxd-generate - DO NOT EDIT! + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net/http" + "strings" + + "github.com/canonical/lxd/lxd/db/query" + "github.com/canonical/lxd/shared/api" + "github.com/canonical/microcluster/v2/cluster" +) + +var _ = api.ServerEnvironment{} + +var storageBackendObjects = cluster.RegisterStmt(` +SELECT storage_backends.id, storage_backends.name, storage_backends.type, storage_backends.config, storage_backends.principal, storage_backends.model_uuid + FROM storage_backends + ORDER BY storage_backends.name +`) + +var storageBackendObjectsByName = cluster.RegisterStmt(` +SELECT storage_backends.id, storage_backends.name, storage_backends.type, storage_backends.config, storage_backends.principal, storage_backends.model_uuid + FROM storage_backends + WHERE ( storage_backends.name = ? ) + ORDER BY storage_backends.name +`) + +var storageBackendObjectsByType = cluster.RegisterStmt(` +SELECT storage_backends.id, storage_backends.name, storage_backends.type, storage_backends.config, storage_backends.principal, storage_backends.model_uuid + FROM storage_backends + WHERE ( storage_backends.type = ? ) + ORDER BY storage_backends.name +`) + +var storageBackendObjectsByPrincipal = cluster.RegisterStmt(` +SELECT storage_backends.id, storage_backends.name, storage_backends.type, storage_backends.config, storage_backends.principal, storage_backends.model_uuid + FROM storage_backends + WHERE ( storage_backends.principal = ? ) + ORDER BY storage_backends.name +`) + +var storageBackendObjectsByModelUUID = cluster.RegisterStmt(` +SELECT storage_backends.id, storage_backends.name, storage_backends.type, storage_backends.config, storage_backends.principal, storage_backends.model_uuid + FROM storage_backends + WHERE ( storage_backends.model_uuid = ? ) + ORDER BY storage_backends.name +`) + +var storageBackendID = cluster.RegisterStmt(` +SELECT storage_backends.id FROM storage_backends + WHERE storage_backends.name = ? +`) + +var storageBackendCreate = cluster.RegisterStmt(` +INSERT INTO storage_backends (name, type, config, principal, model_uuid) + VALUES (?, ?, ?, ?, ?) +`) + +var storageBackendDeleteByName = cluster.RegisterStmt(` +DELETE FROM storage_backends WHERE name = ? +`) + +var storageBackendUpdate = cluster.RegisterStmt(` +UPDATE storage_backends + SET name = ?, type = ?, config = ?, principal = ?, model_uuid = ? + WHERE id = ? +`) + +// storageBackendColumns returns a string of column names to be used with a SELECT statement for the entity. +// Use this function when building statements to retrieve database entries matching the StorageBackend entity. +func storageBackendColumns() string { + return "storage_backends.id, storage_backends.name, storage_backends.type, storage_backends.config, storage_backends.principal, storage_backends.model_uuid" +} + +// getStorageBackends can be used to run handwritten sql.Stmts to return a slice of objects. +func getStorageBackends(ctx context.Context, stmt *sql.Stmt, args ...any) ([]StorageBackend, error) { + objects := make([]StorageBackend, 0) + + dest := func(scan func(dest ...any) error) error { + s := StorageBackend{} + err := scan(&s.ID, &s.Name, &s.Type, &s.Config, &s.Principal, &s.ModelUUID) + if err != nil { + return err + } + + objects = append(objects, s) + + return nil + } + + err := query.SelectObjects(ctx, stmt, dest, args...) + if err != nil { + return nil, fmt.Errorf("Failed to fetch from \"storage_backends\" table: %w", err) + } + + return objects, nil +} + +// getStorageBackendsRaw can be used to run handwritten query strings to return a slice of objects. +func getStorageBackendsRaw(ctx context.Context, tx *sql.Tx, sql string, args ...any) ([]StorageBackend, error) { + objects := make([]StorageBackend, 0) + + dest := func(scan func(dest ...any) error) error { + s := StorageBackend{} + err := scan(&s.ID, &s.Name, &s.Type, &s.Config, &s.Principal, &s.ModelUUID) + if err != nil { + return err + } + + objects = append(objects, s) + + return nil + } + + err := query.Scan(ctx, tx, sql, dest, args...) + if err != nil { + return nil, fmt.Errorf("Failed to fetch from \"storage_backends\" table: %w", err) + } + + return objects, nil +} + +// GetStorageBackends returns all available StorageBackends. +// generator: StorageBackend GetMany +func GetStorageBackends(ctx context.Context, tx *sql.Tx, filters ...StorageBackendFilter) ([]StorageBackend, error) { + var err error + + // Result slice. + objects := make([]StorageBackend, 0) + + // Pick the prepared statement and arguments to use based on active criteria. + var sqlStmt *sql.Stmt + args := []any{} + queryParts := [2]string{} + + if len(filters) == 0 { + sqlStmt, err = cluster.Stmt(tx, storageBackendObjects) + if err != nil { + return nil, fmt.Errorf("Failed to get \"storageBackendObjects\" prepared statement: %w", err) + } + } + + for i, filter := range filters { + if filter.Type != nil && filter.Name == nil && filter.Principal == nil && filter.ModelUUID == nil { + args = append(args, []any{filter.Type}...) + if len(filters) == 1 { + sqlStmt, err = cluster.Stmt(tx, storageBackendObjectsByType) + if err != nil { + return nil, fmt.Errorf("Failed to get \"storageBackendObjectsByType\" prepared statement: %w", err) + } + + break + } + + query, err := cluster.StmtString(storageBackendObjectsByType) + if err != nil { + return nil, fmt.Errorf("Failed to get \"storageBackendObjects\" prepared statement: %w", err) + } + + parts := strings.SplitN(query, "ORDER BY", 2) + if i == 0 { + copy(queryParts[:], parts) + continue + } + + _, where, _ := strings.Cut(parts[0], "WHERE") + queryParts[0] += "OR" + where + } else if filter.Principal != nil && filter.Name == nil && filter.Type == nil && filter.ModelUUID == nil { + args = append(args, []any{filter.Principal}...) + if len(filters) == 1 { + sqlStmt, err = cluster.Stmt(tx, storageBackendObjectsByPrincipal) + if err != nil { + return nil, fmt.Errorf("Failed to get \"storageBackendObjectsByPrincipal\" prepared statement: %w", err) + } + + break + } + + query, err := cluster.StmtString(storageBackendObjectsByPrincipal) + if err != nil { + return nil, fmt.Errorf("Failed to get \"storageBackendObjects\" prepared statement: %w", err) + } + + parts := strings.SplitN(query, "ORDER BY", 2) + if i == 0 { + copy(queryParts[:], parts) + continue + } + + _, where, _ := strings.Cut(parts[0], "WHERE") + queryParts[0] += "OR" + where + } else if filter.Name != nil && filter.Type == nil && filter.Principal == nil && filter.ModelUUID == nil { + args = append(args, []any{filter.Name}...) + if len(filters) == 1 { + sqlStmt, err = cluster.Stmt(tx, storageBackendObjectsByName) + if err != nil { + return nil, fmt.Errorf("Failed to get \"storageBackendObjectsByName\" prepared statement: %w", err) + } + + break + } + + query, err := cluster.StmtString(storageBackendObjectsByName) + if err != nil { + return nil, fmt.Errorf("Failed to get \"storageBackendObjects\" prepared statement: %w", err) + } + + parts := strings.SplitN(query, "ORDER BY", 2) + if i == 0 { + copy(queryParts[:], parts) + continue + } + + _, where, _ := strings.Cut(parts[0], "WHERE") + queryParts[0] += "OR" + where + } else if filter.ModelUUID != nil && filter.Name == nil && filter.Type == nil && filter.Principal == nil { + args = append(args, []any{filter.ModelUUID}...) + if len(filters) == 1 { + sqlStmt, err = cluster.Stmt(tx, storageBackendObjectsByModelUUID) + if err != nil { + return nil, fmt.Errorf("Failed to get \"storageBackendObjectsByModelUUID\" prepared statement: %w", err) + } + + break + } + + query, err := cluster.StmtString(storageBackendObjectsByModelUUID) + if err != nil { + return nil, fmt.Errorf("Failed to get \"storageBackendObjects\" prepared statement: %w", err) + } + + parts := strings.SplitN(query, "ORDER BY", 2) + if i == 0 { + copy(queryParts[:], parts) + continue + } + + _, where, _ := strings.Cut(parts[0], "WHERE") + queryParts[0] += "OR" + where + } else if filter.Name == nil && filter.Type == nil && filter.Principal == nil && filter.ModelUUID == nil { + return nil, fmt.Errorf("Cannot filter on empty StorageBackendFilter") + } else { + return nil, fmt.Errorf("No statement exists for the given Filter") + } + } + + // Select. + if sqlStmt != nil { + objects, err = getStorageBackends(ctx, sqlStmt, args...) + } else { + queryStr := strings.Join(queryParts[:], "ORDER BY") + objects, err = getStorageBackendsRaw(ctx, tx, queryStr, args...) + } + + if err != nil { + return nil, fmt.Errorf("Failed to fetch from \"storage_backends\" table: %w", err) + } + + return objects, nil +} + +// GetStorageBackend returns the StorageBackend with the given key. +// generator: StorageBackend GetOne +func GetStorageBackend(ctx context.Context, tx *sql.Tx, name string) (*StorageBackend, error) { + filter := StorageBackendFilter{} + filter.Name = &name + + objects, err := GetStorageBackends(ctx, tx, filter) + if err != nil { + return nil, fmt.Errorf("Failed to fetch from \"storage_backends\" table: %w", err) + } + + switch len(objects) { + case 0: + return nil, api.StatusErrorf(http.StatusNotFound, "StorageBackend not found") + case 1: + return &objects[0], nil + default: + return nil, fmt.Errorf("More than one \"storage_backends\" entry matches") + } +} + +// GetStorageBackendID return the ID of the StorageBackend with the given key. +// generator: StorageBackend ID +func GetStorageBackendID(ctx context.Context, tx *sql.Tx, name string) (int64, error) { + stmt, err := cluster.Stmt(tx, storageBackendID) + if err != nil { + return -1, fmt.Errorf("Failed to get \"storageBackendID\" prepared statement: %w", err) + } + + row := stmt.QueryRowContext(ctx, name) + var id int64 + err = row.Scan(&id) + if errors.Is(err, sql.ErrNoRows) { + return -1, api.StatusErrorf(http.StatusNotFound, "StorageBackend not found") + } + + if err != nil { + return -1, fmt.Errorf("Failed to get \"storage_backends\" ID: %w", err) + } + + return id, nil +} + +// StorageBackendExists checks if a StorageBackend with the given key exists. +// generator: StorageBackend Exists +func StorageBackendExists(ctx context.Context, tx *sql.Tx, name string) (bool, error) { + _, err := GetStorageBackendID(ctx, tx, name) + if err != nil { + if api.StatusErrorCheck(err, http.StatusNotFound) { + return false, nil + } + + return false, err + } + + return true, nil +} + +// CreateStorageBackend adds a new StorageBackend to the database. +// generator: StorageBackend Create +func CreateStorageBackend(ctx context.Context, tx *sql.Tx, object StorageBackend) (int64, error) { + // Check if a StorageBackend with the same key exists. + exists, err := StorageBackendExists(ctx, tx, object.Name) + if err != nil { + return -1, fmt.Errorf("Failed to check for duplicates: %w", err) + } + + if exists { + return -1, api.StatusErrorf(http.StatusConflict, "This \"storage_backends\" entry already exists") + } + + args := make([]any, 5) + + // Populate the statement arguments. + args[0] = object.Name + args[1] = object.Type + args[2] = object.Config + args[3] = object.Principal + args[4] = object.ModelUUID + + // Prepared statement to use. + stmt, err := cluster.Stmt(tx, storageBackendCreate) + if err != nil { + return -1, fmt.Errorf("Failed to get \"storageBackendCreate\" prepared statement: %w", err) + } + + // Execute the statement. + result, err := stmt.Exec(args...) + if err != nil { + return -1, fmt.Errorf("Failed to create \"storage_backends\" entry: %w", err) + } + + id, err := result.LastInsertId() + if err != nil { + return -1, fmt.Errorf("Failed to fetch \"storage_backends\" entry ID: %w", err) + } + + return id, nil +} + +// DeleteStorageBackend deletes the StorageBackend matching the given key parameters. +// generator: StorageBackend DeleteOne-by-Name +func DeleteStorageBackend(_ context.Context, tx *sql.Tx, name string) error { + stmt, err := cluster.Stmt(tx, storageBackendDeleteByName) + if err != nil { + return fmt.Errorf("Failed to get \"storageBackendDeleteByName\" prepared statement: %w", err) + } + + result, err := stmt.Exec(name) + if err != nil { + return fmt.Errorf("Delete \"storage_backends\": %w", err) + } + + n, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("Fetch affected rows: %w", err) + } + + if n == 0 { + return api.StatusErrorf(http.StatusNotFound, "StorageBackend not found") + } else if n > 1 { + return fmt.Errorf("Query deleted %d StorageBackend rows instead of 1", n) + } + + return nil +} + +// UpdateStorageBackend updates the StorageBackend matching the given key parameters. +// generator: StorageBackend Update +func UpdateStorageBackend(ctx context.Context, tx *sql.Tx, name string, object StorageBackend) error { + id, err := GetStorageBackendID(ctx, tx, name) + if err != nil { + return err + } + + stmt, err := cluster.Stmt(tx, storageBackendUpdate) + if err != nil { + return fmt.Errorf("Failed to get \"storageBackendUpdate\" prepared statement: %w", err) + } + + result, err := stmt.Exec(object.Name, object.Type, object.Config, object.Principal, object.ModelUUID, id) + if err != nil { + return fmt.Errorf("Update \"storage_backends\" entry failed: %w", err) + } + + n, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("Fetch affected rows: %w", err) + } + + if n != 1 { + return fmt.Errorf("Query updated %d rows instead of 1", n) + } + + return nil +} diff --git a/sunbeam-microcluster/sunbeam/storage_backend.go b/sunbeam-microcluster/sunbeam/storage_backend.go new file mode 100644 index 000000000..9f4cdf79d --- /dev/null +++ b/sunbeam-microcluster/sunbeam/storage_backend.go @@ -0,0 +1,127 @@ +package sunbeam + +import ( + "context" + "database/sql" + "fmt" + + "github.com/canonical/microcluster/v2/state" + + "github.com/canonical/snap-openstack/sunbeam-microcluster/api/apitypes" + "github.com/canonical/snap-openstack/sunbeam-microcluster/database" +) + +// ListStorageBackends return all the storage backends, filterable by role (Optional) +func ListStorageBackends(ctx context.Context, s state.State) (apitypes.StorageBackends, error) { + backends := apitypes.StorageBackends{} + + // Get the storage backends from the database. + err := s.Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error { + records, err := database.GetStorageBackends(ctx, tx) + if err != nil { + return fmt.Errorf("Failed to fetch storage backends: %w", err) + } + + for _, backend := range records { + + backends = append(backends, apitypes.StorageBackend{ + Name: backend.Name, + Type: backend.Type, + Principal: backend.Principal, + ModelUUID: backend.ModelUUID, + Config: backend.Config, + }) + } + + return nil + }) + if err != nil { + return nil, err + } + + return backends, nil + +} + +// GetStorageBackend returns a StorageBackend with the given name +func GetStorageBackend(ctx context.Context, s state.State, name string) (apitypes.StorageBackend, error) { + backend := apitypes.StorageBackend{} + err := s.Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error { + record, err := database.GetStorageBackend(ctx, tx, name) + if err != nil { + return err + } + + backend.Name = record.Name + backend.Type = record.Type + backend.Principal = record.Principal + backend.ModelUUID = record.ModelUUID + backend.Config = record.Config + + return nil + }) + if err != nil { + return apitypes.StorageBackend{}, err + } + return backend, nil +} + +// AddStorageBackend adds a storage backend to the database +func AddStorageBackend(ctx context.Context, s state.State, name string, backendType string, principal string, modelUUID string, config string) error { + // Add storage backend to the database. + return s.Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error { + _, err := database.CreateStorageBackend(ctx, tx, database.StorageBackend{ + Name: name, + Type: backendType, + Principal: principal, + ModelUUID: modelUUID, + Config: config, + }) + if err != nil { + return fmt.Errorf("Failed to record storage backend: %w", err) + } + + return nil + }) +} + +// UpdateStorageBackend updates a storage backend record in the database +func UpdateStorageBackend(ctx context.Context, s state.State, name string, backendType string, principal string, modelUUID string, config string) error { + // Update storage backend to the database. + err := s.Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error { + backend, err := database.GetStorageBackend(ctx, tx, name) + if err != nil { + return fmt.Errorf("Failed to retrieve storage backend details: %w", err) + } + + if backendType == "" { + backendType = backend.Type + } + if principal == "" { + principal = backend.Principal + } + if modelUUID == "" { + modelUUID = backend.ModelUUID + } + if config == "" { + config = backend.Config + } + + err = database.UpdateStorageBackend(ctx, tx, name, database.StorageBackend{Name: name, Type: backendType, Principal: principal, ModelUUID: modelUUID, Config: config}) + if err != nil { + return fmt.Errorf("Failed to update record storage backend: %w", err) + } + + return nil + }) + + return err +} + +// DeleteStorageBackend deletes a storage backend from database +func DeleteStorageBackend(ctx context.Context, s state.State, name string) error { + return s.Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error { + return database.DeleteStorageBackend(ctx, tx, name) + }) + +} diff --git a/sunbeam-python/sunbeam/clusterd/cluster.py b/sunbeam-python/sunbeam/clusterd/cluster.py index 51990f3f5..7503b4fb0 100644 --- a/sunbeam-python/sunbeam/clusterd/cluster.py +++ b/sunbeam-python/sunbeam/clusterd/cluster.py @@ -9,7 +9,7 @@ from requests import codes from requests.models import HTTPError -from sunbeam.clusterd import service +from sunbeam.clusterd import models, service LOG = logging.getLogger(__name__) @@ -239,6 +239,58 @@ def get_status(self) -> dict[str, dict]: for member in members } + def get_storage_backends(self) -> models.StorageBackends: + """List all storage backends.""" + backends = self._get("/1.0/storage-backend") + return models.StorageBackends(root=backends.get("metadata", [])) + + def get_storage_backend(self, name: str) -> models.StorageBackend: + """Get storage backend by name.""" + backend = self._get(f"/1.0/storage-backend/{name}") + return models.StorageBackend(**backend.get("metadata", {})) + + def add_storage_backend( + self, + name: str, + backend_type: str, + config: dict[str, Any], + principal: str, + model_uuid: str, + ) -> None: + """Add a new storage backend.""" + data = { + "name": name, + "type": backend_type, + "config": json.dumps(config), + "principal": principal, + "model-uuid": model_uuid, + } + self._post("/1.0/storage-backend", data=json.dumps(data)) + + def delete_storage_backend(self, name: str) -> None: + """Delete storage backend by name.""" + self._delete(f"/1.0/storage-backend/{name}") + + def update_storage_backend( + self, + name: str, + backend_type: str | None = None, + config: dict[str, Any] | None = None, + principal: str | None = None, + model_uuid: str | None = None, + ) -> None: + """Update an existing storage backend.""" + data: dict[str, Any] = {} + if backend_type is not None: + data["type"] = backend_type + if config is not None: + data["config"] = json.dumps(config) + if principal is not None: + data["principal"] = principal + if model_uuid is not None: + data["model-uuid"] = model_uuid + self._put(f"/1.0/storage-backend/{name}", data=json.dumps(data)) + class ClusterService(MicroClusterService, ExtendedAPIService): """Lists and manages cluster.""" diff --git a/sunbeam-python/sunbeam/clusterd/models.py b/sunbeam-python/sunbeam/clusterd/models.py new file mode 100644 index 000000000..3e41228e0 --- /dev/null +++ b/sunbeam-python/sunbeam/clusterd/models.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Clusterd models for Sunbeam.""" + +import typing + +import pydantic + +from sunbeam import utils + + +class StorageBackend(pydantic.BaseModel): + """Storage backend model.""" + + model_config = pydantic.ConfigDict( + alias_generator=pydantic.AliasGenerator( + validation_alias=utils.to_kebab, + serialization_alias=utils.to_kebab, + ), + ) + name: str + type: str + config: pydantic.Json[dict[str, typing.Any]] + principal: str + model_uuid: str + + +class StorageBackends(pydantic.RootModel[list[StorageBackend]]): + """Storage backends model.""" diff --git a/sunbeam-python/sunbeam/clusterd/service.py b/sunbeam-python/sunbeam/clusterd/service.py index 8ae9e6fa7..cf360a89d 100644 --- a/sunbeam-python/sunbeam/clusterd/service.py +++ b/sunbeam-python/sunbeam/clusterd/service.py @@ -92,6 +92,14 @@ class URLNotFoundException(RemoteException): pass +class StorageBackendException(RemoteException): + """Base exception for storage backend operations.""" + + +class StorageBackendNotFoundException(StorageBackendException): + """Raised when storage backend is not found.""" + + class BaseService(ABC): """BaseService is the base service class for sunbeam clusterd services.""" @@ -203,6 +211,8 @@ def _request(self, method, path, **kwargs): # noqa: C901 too complex raise ConfigItemNotFoundException("ConfigItem not found") elif "ManifestItem not found" in error: raise ManifestItemNotFoundException("ManifestItem not found") + elif "StorageBackend not found" in error: + raise StorageBackendNotFoundException("Storage backend not found") raise e return response.json() diff --git a/sunbeam-python/sunbeam/core/common.py b/sunbeam-python/sunbeam/core/common.py index 85f09dafc..d0519ef47 100644 --- a/sunbeam-python/sunbeam/core/common.py +++ b/sunbeam-python/sunbeam/core/common.py @@ -21,6 +21,7 @@ from tenacity import RetryCallState from sunbeam.clusterd.client import Client +from sunbeam.errors import SunbeamException # noqa F401 LOG = logging.getLogger(__name__) RAM_16_GB_IN_KB = 16 * 1000 * 1000 @@ -516,12 +517,6 @@ def convert_proxy_to_model_configs(proxy_settings: dict) -> dict: } -class SunbeamException(Exception): - """Base exception for sunbeam.""" - - pass - - class RiskLevel(str, enum.Enum): STABLE = "stable" CANDIDATE = "candidate" @@ -651,3 +646,45 @@ def convert_retry_failure_as_result(retry_state: RetryCallState) -> Result: return Result(ResultType.FAILED, str(retry_state.outcome.exception())) else: return Result(ResultType.FAILED) + + +def friendly_terraform_lock_retry_callback(retry_state: RetryCallState) -> Result: + """Friendly retry callback for Terraform state lock exceptions. + + Shows user-friendly messages during lock retries + instead of verbose Terraform output. + """ + from sunbeam.core.terraform import TerraformStateLockedException + + if retry_state.outcome is not None: + exception = retry_state.outcome.exception() + if isinstance(exception, TerraformStateLockedException): + # Extract lock ID from the error message if possible + lock_id = "unknown" + error_str = str(exception) + if "ID:" in error_str: + try: + # Extract lock ID from Terraform output + lines = error_str.split("\n") + for line in lines: + if "ID:" in line: + lock_id = line.split("ID:")[1].strip() + break + except Exception: + LOG.debug( + "Failed to extract lock ID from Terraform output: %s", + error_str, + ) + pass + + return Result( + ResultType.FAILED, + f"Terraform state is locked (ID: {lock_id}). " + f"This usually resolves automatically. " + f"If it persists, use 'sunbeam plans unlock ' to " + f"clear stale locks.", + ) + else: + return Result(ResultType.FAILED, str(exception)) + else: + return Result(ResultType.FAILED, "Operation failed after retries") diff --git a/sunbeam-python/sunbeam/core/deployment.py b/sunbeam-python/sunbeam/core/deployment.py index bff4d8fa0..155e7c712 100644 --- a/sunbeam-python/sunbeam/core/deployment.py +++ b/sunbeam-python/sunbeam/core/deployment.py @@ -31,6 +31,9 @@ FeatureGroupManifest, FeatureManifest, Manifest, + StorageBackendManifests, + StorageInstanceManifest, + StorageManifest, embedded_manifest_path, ) from sunbeam.core.proxy import patch_process_env, should_bypass @@ -40,9 +43,11 @@ if TYPE_CHECKING: from sunbeam.feature_manager import FeatureManager from sunbeam.features.interface.v1.base import BaseFeature + from sunbeam.storage.manager import StorageBackendManager else: FeatureManager = object BaseFeature = object + StorageBackendManager = object LOG = logging.getLogger(__name__) PROXY_CONFIG_KEY = "ProxySettings" @@ -98,6 +103,7 @@ class Deployment(pydantic.BaseModel): _manifest: Manifest | None = pydantic.PrivateAttr(default=None) _tfhelpers: dict[str, TerraformHelper] = pydantic.PrivateAttr(default={}) _feature_manager: FeatureManager | None = pydantic.PrivateAttr(default=None) + _storage_manager: StorageBackendManager | None = pydantic.PrivateAttr(default=None) @property def openstack_machines_model(self) -> str: @@ -188,6 +194,15 @@ def get_feature_manager(self) -> "FeatureManager": return self._feature_manager + def get_storage_manager(self) -> "StorageBackendManager": + """Return the storage backend manager for the deployment.""" + from sunbeam.storage.manager import StorageBackendManager + + if self._storage_manager is None: + self._storage_manager = StorageBackendManager() + + return self._storage_manager + def get_proxy_settings(self) -> dict: """Fetch proxy settings from clusterd, if not available use defaults.""" proxy = {} @@ -269,12 +284,44 @@ def _parse_feature( return feature_manifests + def parse_storage_manifest( + self, storage_manifest_data: dict[str, dict] + ) -> StorageManifest: + """Parse storage manifest data.""" + if not storage_manifest_data: + return StorageManifest(root={}) + backends = self.get_storage_manager().backends() + + storage_manifest: StorageManifest = StorageManifest(root={}) + for backend_type, backends_dict in storage_manifest_data.items(): + backend = backends[backend_type] + backend_config_type = backend.config_type() + backends_config = storage_manifest.root.setdefault( + backend_type, StorageBackendManifests(root={}) + ) + for name, manifest_dict in backends_dict.items(): + manifest_dict_config = manifest_dict.pop("config", None) + backends_config.root[name] = StorageInstanceManifest.model_validate( + manifest_dict, by_alias=True + ) + if manifest_dict_config: + backends_config.root[ + name + ].config = backend_config_type.model_validate( + manifest_dict_config, by_alias=True + ) + + return storage_manifest + def parse_manifest(self, manifest_data: dict) -> Manifest: """Parse manifest data.""" features = manifest_data.pop("features", {}) + storage = manifest_data.pop("storage", {}) manifest = Manifest.model_validate(manifest_data) if features: manifest.features = self.parse_feature_manifest(features) + if storage: + manifest.storage = self.parse_storage_manifest(storage) return manifest def get_manifest(self, manifest_file: pathlib.Path | None = None) -> Manifest: diff --git a/sunbeam-python/sunbeam/core/manifest.py b/sunbeam-python/sunbeam/core/manifest.py index 4e615581b..fcad149ed 100644 --- a/sunbeam-python/sunbeam/core/manifest.py +++ b/sunbeam-python/sunbeam/core/manifest.py @@ -145,6 +145,26 @@ class FeatureConfig(pydantic.BaseModel): pass +class StorageBackendConfig(pydantic.BaseModel): + """Base configuration model for storage backends.""" + + model_config = pydantic.ConfigDict( + alias_generator=pydantic.AliasGenerator( + validation_alias=utils.to_kebab, + serialization_alias=utils.to_kebab, + ), + ) + + volume_backend_name: typing.Annotated[ + str | None, + Field(description="Name that Cinder will report for this backend"), + ] = None + backend_availability_zone: typing.Annotated[ + str | None, + Field(description="Availability zone to associate with this backend"), + ] = None + + def _default_software_config() -> SoftwareConfig: snap = Snap() return SoftwareConfig( @@ -287,15 +307,18 @@ def merge(self, other: "CoreManifest") -> "CoreManifest": return type(self)(config=config, software=software) -class FeatureManifest(pydantic.BaseModel): - config: pydantic.SerializeAsAny[FeatureConfig] | None = None +T = typing.TypeVar("T", bound=pydantic.BaseModel) + + +class _AddonManifest(pydantic.BaseModel, typing.Generic[T]): + config: pydantic.SerializeAsAny[T] | None = None software: SoftwareConfig = SoftwareConfig() - def merge(self, other: "FeatureManifest") -> "FeatureManifest": - """Merge the feature manifest with the provided manifest.""" + def merge(self, other: "typing.Self") -> "typing.Self": + """Merge the addon manifest with the provided manifest.""" if self.config and other.config: if type(self.config) is not type(other.config): - raise ValueError("Feature config types do not match") + raise ValueError("Config types do not match") config = type(self.config).model_validate( utils.merge_dict( self.config.model_dump(by_alias=True), @@ -312,6 +335,30 @@ def merge(self, other: "FeatureManifest") -> "FeatureManifest": return type(self)(config=config, software=software) +class FeatureManifest(_AddonManifest[FeatureConfig]): + pass + + +class StorageInstanceManifest(_AddonManifest[StorageBackendConfig]): + pass + + +class StorageBackendManifests(pydantic.RootModel[dict[str, StorageInstanceManifest]]): + """Storage backend manifests. + + Key: Instance name + Value: Storage backend manifest + """ + + +class StorageManifest(pydantic.RootModel[dict[str, StorageBackendManifests]]): + """Storage manifest containing all storage backends. + + Key: Storage type + Value: Storage backend manifests + """ + + class FeatureGroupManifest(pydantic.RootModel[dict[str, FeatureManifest]]): def merge(self, other: "FeatureGroupManifest") -> "FeatureGroupManifest": """Merge the feature group manifest with the provided manifest.""" @@ -336,6 +383,7 @@ def validate_againt_default(self, default_manifest: "FeatureGroupManifest") -> N class Manifest(pydantic.BaseModel): core: CoreManifest = pydantic.Field(default_factory=CoreManifest) features: dict[str, FeatureManifest | FeatureGroupManifest] = {} + storage: StorageManifest = StorageManifest(root={}) def get_features(self) -> typing.Generator[tuple[str, FeatureManifest], None, None]: """Return all the features.""" @@ -370,6 +418,8 @@ def from_file(cls, file: Path) -> "Manifest": def merge(self, other: "Manifest") -> "Manifest": """Merge the manifest with the provided manifest.""" core = self.core.merge(other.core) + # Storage has no defaults, and will be fully replaced + storage = other.storage features: dict[str, FeatureManifest | FeatureGroupManifest] = {} for feature, feature_or_group_manifest in self.features.items(): if other_manifest := other.features.get(feature): @@ -384,7 +434,7 @@ def merge(self, other: "Manifest") -> "Manifest": else: features[feature] = feature_or_group_manifest - return type(self)(core=core, features=features) + return type(self)(core=core, features=features, storage=storage) def validate_against_default(self, default_manifest: "Manifest") -> None: """Validate the manifest against the default manifest.""" diff --git a/sunbeam-python/sunbeam/errors.py b/sunbeam-python/sunbeam/errors.py new file mode 100644 index 000000000..f944c42e0 --- /dev/null +++ b/sunbeam-python/sunbeam/errors.py @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + + +class SunbeamException(Exception): + """Base exception for sunbeam.""" diff --git a/sunbeam-python/sunbeam/main.py b/sunbeam-python/sunbeam/main.py index 0ebc6f30f..48cc8ff5b 100644 --- a/sunbeam-python/sunbeam/main.py +++ b/sunbeam-python/sunbeam/main.py @@ -160,6 +160,9 @@ def main(): juju.add_command(juju_cmds.register_controller) juju.add_command(juju_cmds.unregister_controller) + # Register storage backend commands + deployment.get_storage_manager().register(cli, deployment) + # Register the features after all groups,commands are registered deployment.get_feature_manager().register(cli, deployment) diff --git a/sunbeam-python/sunbeam/storage/__init__.py b/sunbeam-python/sunbeam/storage/__init__.py new file mode 100644 index 000000000..005b6db0d --- /dev/null +++ b/sunbeam-python/sunbeam/storage/__init__.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Sunbeam Storage Backends. + +This module provides a pluggable storage backend system for Sunbeam. +""" + +# Backends are loaded dynamically by the registry - no hardcoded imports needed diff --git a/sunbeam-python/sunbeam/storage/backends/__init__.py b/sunbeam-python/sunbeam/storage/backends/__init__.py new file mode 100644 index 000000000..f4e0ac8a7 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/__init__.py @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Sunbeam Storage Backend Implementations. + +This package contains implementations of various storage backends for Sunbeam. +""" diff --git a/sunbeam-python/sunbeam/storage/backends/dellsc/__init__.py b/sunbeam-python/sunbeam/storage/backends/dellsc/__init__.py new file mode 100644 index 000000000..68f03f984 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/dellsc/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Dell Storage Center backend for Sunbeam storage.""" diff --git a/sunbeam-python/sunbeam/storage/backends/dellsc/backend.py b/sunbeam-python/sunbeam/storage/backends/dellsc/backend.py new file mode 100644 index 000000000..743cf14bd --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/dellsc/backend.py @@ -0,0 +1,154 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Dell Storage Center storage backend implementation using base step classes.""" + +import logging +from typing import Annotated, Literal + +from pydantic import Field +from rich.console import Console + +from sunbeam.core.manifest import StorageBackendConfig +from sunbeam.storage.base import StorageBackendBase +from sunbeam.storage.models import SecretDictField + +LOG = logging.getLogger(__name__) +console = Console() + + +class DellSCConfig(StorageBackendConfig): + """Static configuration model for Dell Storage Center storage backend. + + This model includes all configuration options supported by the + cinder-volume-dellsc charm as defined in charmcraft.yaml. + """ + + # Mandatory connection parameters + san_ip: Annotated[ + str, Field(description="Dell Storage Center management IP or hostname") + ] + san_username: Annotated[ + str, + Field(description="SAN management username"), + SecretDictField(field="primary-username"), + ] + san_password: Annotated[ + str, + Field(description="SAN management password"), + SecretDictField(field="primary-password"), + ] + dell_sc_ssn: Annotated[ + int | None, Field(description="Storage Center System Serial Number") + ] = None + protocol: Annotated[ + Literal["fc", "iscsi"] | None, + Field(description="Front-end protocol (fc or iscsi)"), + ] = None + + # Backend configuration + volume_backend_name: Annotated[ + str | None, Field(description="Name that Cinder will report for this backend") + ] = None + backend_availability_zone: Annotated[ + str | None, + Field(description="Availability zone to associate with this backend"), + ] = None + + # Dell Storage Center specific options + dell_sc_api_port: Annotated[ + int | None, Field(description="Dell Storage Center API port") + ] = None + dell_sc_server_folder: Annotated[ + str | None, Field(description="Server folder name on Dell SC") + ] = None + dell_sc_volume_folder: Annotated[ + str | None, Field(description="Volume folder name on Dell SC") + ] = None + dell_server_os: Annotated[ + str | None, Field(description="Server OS type for Dell SC") + ] = None + dell_sc_verify_cert: Annotated[ + bool | None, Field(description="Verify SSL certificate for Dell SC API") + ] = None + + # Provisioning options + san_thin_provision: Annotated[ + bool | None, Field(description="Enable thin provisioning") + ] = None + + # Domain and network filtering + excluded_domain_ips: Annotated[ + str | None, Field(description="Comma-separated list of excluded domain IPs") + ] = None + included_domain_ips: Annotated[ + str | None, Field(description="Comma-separated list of included domain IPs") + ] = None + + # Dual DSM configuration + secondary_san_ip: Annotated[ + str | None, Field(description="Secondary Dell Storage Center management IP") + ] = None + secondary_san_username: Annotated[ + str | None, + Field(description="Secondary SAN management username"), + SecretDictField(field="secondary-username"), + ] = None + secondary_san_password: Annotated[ + str | None, + Field(description="Secondary SAN management password"), + SecretDictField(field="secondary-password"), + ] = None + secondary_sc_api_port: Annotated[ + int | None, Field(description="Secondary Dell Storage Center API port") + ] = None + + # API timeout configuration + dell_api_async_rest_timeout: Annotated[ + int | None, Field(description="Async REST API timeout in seconds") + ] = None + dell_api_sync_rest_timeout: Annotated[ + int | None, Field(description="Sync REST API timeout in seconds") + ] = None + + # SSH connection settings + ssh_conn_timeout: Annotated[ + int | None, Field(description="SSH connection timeout in seconds") + ] = None + ssh_max_pool_conn: Annotated[ + int | None, Field(description="Maximum SSH pool connections") + ] = None + ssh_min_pool_conn: Annotated[ + int | None, Field(description="Minimum SSH pool connections") + ] = None + + +class DellSCBackend(StorageBackendBase): + """Dell Storage Center storage backend implementation.""" + + backend_type = "dellsc" + display_name = "Dell Storage Center" + + @property + def charm_name(self) -> str: + """Return the charm name for this backend.""" + return "cinder-volume-dellsc" + + @property + def charm_channel(self) -> str: + """Return the charm channel for this backend.""" + return "latest/edge" + + @property + def charm_revision(self) -> str | None: + """Return the charm revision for this backend.""" + return None + + @property + def charm_base(self) -> str: + """Return the charm base for this backend.""" + return "ubuntu@24.04" + + def config_type(self) -> type[StorageBackendConfig]: + """Return the configuration class for Dell Storage Center backend.""" + return DellSCConfig diff --git a/sunbeam-python/sunbeam/storage/backends/hitachi/__init__.py b/sunbeam-python/sunbeam/storage/backends/hitachi/__init__.py new file mode 100644 index 000000000..c661a9f84 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/hitachi/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Hitachi backend for Sunbeam storage.""" diff --git a/sunbeam-python/sunbeam/storage/backends/hitachi/backend.py b/sunbeam-python/sunbeam/storage/backends/hitachi/backend.py new file mode 100644 index 000000000..f41f8a3ec --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/hitachi/backend.py @@ -0,0 +1,303 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Hitachi VSP storage backend implementation using base step classes.""" + +import logging +from typing import Annotated, Literal + +from pydantic import Field +from rich.console import Console + +from sunbeam.core.manifest import StorageBackendConfig +from sunbeam.storage.base import StorageBackendBase +from sunbeam.storage.models import SecretDictField + +LOG = logging.getLogger(__name__) +console = Console() + + +class HitachiConfig(StorageBackendConfig): + """Static configuration model for Hitachi VSP storage backend. + + This model includes all configuration options supported by the + cinder-volume-hitachi charm as defined in charmcraft.yaml. + """ + + # Mandatory connection parameters + hitachi_storage_id: Annotated[ + str, Field(description="Storage system product number/serial") + ] + hitachi_pools: Annotated[ + str, Field(description="Comma-separated list of DP pool names/IDs") + ] + san_ip: Annotated[str, Field(description="Hitachi VSP management IP or hostname")] + san_username: Annotated[ + str, + Field(description="SAN management username"), + SecretDictField(field="san-username"), + ] + san_password: Annotated[ + str, + Field(description="SAN management password"), + SecretDictField(field="san-password"), + ] + protocol: Annotated[ + Literal["FC", "iSCSI"], Field(description="Front-end protocol (FC or iSCSI)") + ] + + # Backend configuration + volume_backend_name: Annotated[ + str | None, Field(description="Name that Cinder will report for this backend") + ] = None + backend_availability_zone: Annotated[ + str | None, + Field(description="Availability zone to associate with this backend"), + ] = None + + # Optional host-group / zoning controls + hitachi_target_ports: Annotated[ + str | None, Field(description="Comma-separated front-end port labels") + ] = None + hitachi_compute_target_ports: Annotated[ + str | None, Field(description="Comma-separated compute-node port IDs") + ] = None + hitachi_ldev_range: Annotated[ + str | None, Field(description="LDEV range usable by the driver") + ] = None + hitachi_zoning_request: Annotated[ + bool | None, Field(description="Request FC zone-manager to create zoning") + ] = None + + # Copy & replication tuning + hitachi_copy_speed: Annotated[ + int | None, Field(description="Copy bandwidth throttle (1-15)") + ] = None + hitachi_copy_check_interval: Annotated[ + int | None, Field(description="Seconds between sync copy-status polls") + ] = None + hitachi_async_copy_check_interval: Annotated[ + int | None, Field(description="Seconds between async copy-status polls") + ] = None + + # iSCSI authentication + use_chap_auth: Annotated[ + bool | None, Field(description="Use CHAP authentication for iSCSI") + ] = None + + # Array ranges and controls + hitachi_discard_zero_page: Annotated[ + bool | None, Field(description="Enable zero-page reclamation in DP-VOLs") + ] = None + hitachi_exec_retry_interval: Annotated[ + int | None, Field(description="Seconds to wait before retrying REST API call") + ] = None + hitachi_extend_timeout: Annotated[ + int | None, Field(description="Max seconds to wait for volume extension") + ] = None + hitachi_group_create: Annotated[ + bool | None, + Field(description="Automatically create host groups or iSCSI targets"), + ] = None + hitachi_group_delete: Annotated[ + bool | None, Field(description="Automatically delete unused host groups") + ] = None + hitachi_group_name_format: Annotated[ + str | None, Field(description="Python format string for naming host groups") + ] = None + hitachi_host_mode_options: Annotated[ + str | None, Field(description="Comma-separated host mode options") + ] = None + hitachi_lock_timeout: Annotated[ + int | None, Field(description="Max seconds for array login/unlock operations") + ] = None + hitachi_lun_retry_interval: Annotated[ + int | None, Field(description="Seconds before retrying LUN mapping") + ] = None + hitachi_lun_timeout: Annotated[ + int | None, Field(description="Max seconds to wait for LUN mapping") + ] = None + hitachi_port_scheduler: Annotated[ + bool | None, Field(description="Enable round-robin WWN registration") + ] = None + + # Mirror/replication settings + hitachi_mirror_compute_target_ports: Annotated[ + str | None, Field(description="Compute-node port names for GAD") + ] = None + hitachi_mirror_ldev_range: Annotated[ + str | None, Field(description="LDEV range for secondary storage") + ] = None + hitachi_mirror_pair_target_number: Annotated[ + int | None, Field(description="Host group number for GAD on secondary") + ] = None + hitachi_mirror_pool: Annotated[ + str | None, Field(description="DP pool name/ID on secondary storage") + ] = None + hitachi_mirror_rest_api_ip: Annotated[ + str | None, Field(description="REST API IP on secondary storage") + ] = None + hitachi_mirror_rest_api_port: Annotated[ + int | None, Field(description="REST API port on secondary storage") + ] = None + hitachi_mirror_rest_pair_target_ports: Annotated[ + str | None, Field(description="Pair-target port names for GAD") + ] = None + hitachi_mirror_snap_pool: Annotated[ + str | None, Field(description="Snapshot pool on secondary storage") + ] = None + hitachi_mirror_ssl_cert_path: Annotated[ + str | None, Field(description="CA_BUNDLE for secondary REST endpoint") + ] = None + hitachi_mirror_ssl_cert_verify: Annotated[ + bool | None, Field(description="Validate SSL cert of secondary REST") + ] = None + hitachi_mirror_storage_id: Annotated[ + str | None, Field(description="Product number of secondary storage") + ] = None + hitachi_mirror_target_ports: Annotated[ + str | None, Field(description="Controller node port IDs for GAD") + ] = None + hitachi_mirror_use_chap_auth: Annotated[ + bool | None, Field(description="Use CHAP auth for GAD on secondary") + ] = None + + # Replication settings + hitachi_pair_target_number: Annotated[ + int | None, Field(description="Host group number for primary replication") + ] = None + hitachi_path_group_id: Annotated[ + int | None, Field(description="Path group ID for remote replication") + ] = None + hitachi_quorum_disk_id: Annotated[ + int | None, Field(description="Quorum disk ID for Global-Active Device") + ] = None + hitachi_replication_copy_speed: Annotated[ + int | None, Field(description="Copy speed for remote replication") + ] = None + hitachi_replication_number: Annotated[ + int | None, Field(description="Instance number for REST API on replication") + ] = None + hitachi_replication_status_check_long_interval: Annotated[ + int | None, Field(description="Poll interval after initial check") + ] = None + hitachi_replication_status_check_short_interval: Annotated[ + int | None, Field(description="Initial poll interval") + ] = None + hitachi_replication_status_check_timeout: Annotated[ + int | None, Field(description="Max seconds for status change") + ] = None + + # REST API settings + hitachi_rest_another_ldev_mapped_retry_timeout: Annotated[ + int | None, Field(description="Retry seconds when LDEV allocation fails") + ] = None + hitachi_rest_connect_timeout: Annotated[ + int | None, Field(description="Max seconds to establish REST connection") + ] = None + hitachi_rest_disable_io_wait: Annotated[ + bool | None, Field(description="Detach volumes without waiting for I/O drain") + ] = None + hitachi_rest_get_api_response_timeout: Annotated[ + int | None, Field(description="Max seconds for sync REST GET") + ] = None + hitachi_rest_job_api_response_timeout: Annotated[ + int | None, Field(description="Max seconds for async REST PUT/DELETE") + ] = None + hitachi_rest_keep_session_loop_interval: Annotated[ + int | None, Field(description="Seconds between keep-alive loops") + ] = None + hitachi_rest_pair_target_ports: Annotated[ + str | None, Field(description="Pair-target port names for REST operations") + ] = None + hitachi_rest_server_busy_timeout: Annotated[ + int | None, Field(description="Max seconds when REST API returns busy") + ] = None + hitachi_rest_tcp_keepalive: Annotated[ + bool | None, Field(description="Enable TCP keepalive for REST connections") + ] = None + hitachi_rest_tcp_keepcnt: Annotated[ + int | None, Field(description="Number of TCP keepalive probes") + ] = None + hitachi_rest_tcp_keepidle: Annotated[ + int | None, Field(description="Seconds before sending first TCP keepalive") + ] = None + hitachi_rest_tcp_keepintvl: Annotated[ + int | None, Field(description="Seconds between TCP keepalive probes") + ] = None + hitachi_rest_timeout: Annotated[ + int | None, Field(description="Max seconds for each REST API call") + ] = None + hitachi_restore_timeout: Annotated[ + int | None, Field(description="Max seconds to wait for restore operation") + ] = None + + # Snapshot settings + hitachi_snap_pool: Annotated[ + str | None, Field(description="Pool name/ID for snapshots") + ] = None + hitachi_state_transition_timeout: Annotated[ + int | None, Field(description="Max seconds for volume state transition") + ] = None + + chap_username: Annotated[ + str | None, + Field(description="CHAP username for secret creation"), + SecretDictField(field="chap-username"), + ] = None + chap_password: Annotated[ + str | None, + Field(description="CHAP password for secret creation"), + SecretDictField(field="chap-password"), + ] = None + hitachi_mirror_chap_username: Annotated[ + str | None, + Field(description="Mirror CHAP username for secret creation"), + SecretDictField(field="mirror-chap-username"), + ] = None + hitachi_mirror_chap_password: Annotated[ + str | None, + Field(description="Mirror CHAP password for secret creation"), + SecretDictField(field="mirror-chap-password"), + ] = None + hitachi_mirror_rest_username: Annotated[ + str | None, + Field(description="Mirror REST username for secret creation"), + SecretDictField(field="mirror-rest-username"), + ] = None + hitachi_mirror_rest_password: Annotated[ + str | None, + Field(description="Mirror REST password for secret creation"), + SecretDictField(field="mirror-rest-password"), + ] = None + + +class HitachiBackend(StorageBackendBase): + """Hitachi storage backend implementation.""" + + backend_type = "hitachi" + display_name = "Hitachi VSP Storage" + + @property + def charm_name(self) -> str: + """Return the charm name for this backend.""" + return "cinder-volume-hitachi" + + @property + def charm_channel(self) -> str: + """Return the charm channel for this backend.""" + return "latest/edge" + + @property + def charm_revision(self) -> str | None: + """Return the charm revision for this backend.""" + return None + + @property + def charm_base(self) -> str: + """Return the charm base for this backend.""" + return "ubuntu@24.04" + + def config_type(self) -> type[StorageBackendConfig]: + """Return the configuration class for Hitachi backend.""" + return HitachiConfig diff --git a/sunbeam-python/sunbeam/storage/backends/purestorage/__init__.py b/sunbeam-python/sunbeam/storage/backends/purestorage/__init__.py new file mode 100644 index 000000000..45db76b8f --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/purestorage/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""PureStorage backend for Sunbeam storage.""" diff --git a/sunbeam-python/sunbeam/storage/backends/purestorage/backend.py b/sunbeam-python/sunbeam/storage/backends/purestorage/backend.py new file mode 100644 index 000000000..a77fea55d --- /dev/null +++ b/sunbeam-python/sunbeam/storage/backends/purestorage/backend.py @@ -0,0 +1,177 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Pure Storage FlashArray backend implementation using base step classes.""" + +import logging +from enum import StrEnum +from typing import Annotated, Literal + +from pydantic import Field +from rich.console import Console + +from sunbeam.core.manifest import StorageBackendConfig +from sunbeam.storage.base import StorageBackendBase +from sunbeam.storage.models import SecretDictField + +LOG = logging.getLogger(__name__) +console = Console() + + +class Personality(StrEnum): + """Enumeration of valid host personality types.""" + + AIX = "aix" + ESXI = "esxi" + HITACHI_VSP = "hitachi-vsp" + HPUX = "hpux" + ORACLE_VM_SERVER = "oracle-vm-server" + SOLARIS = "solaris" + VMS = "vms" + + +class PureStorageConfig(StorageBackendConfig): + """Configuration model for Pure Storage FlashArray backend. + + This model includes the essential configuration options for deploying + a Pure Storage backend. Additional configuration can be managed dynamically + through the charm configuration system. + """ + + # Mandatory connection parameters + san_ip: Annotated[ + str, Field(description="Pure Storage FlashArray management IP or hostname") + ] + pure_api_token: Annotated[ + str, + Field(description="REST API authorization token from FlashArray"), + SecretDictField(field="token"), + ] + + # Optional backend configuration + protocol: Annotated[ + Literal["iscsi", "fc", "nvme"] | None, + Field(description="Pure Storage protocol (iscsi, fc, nvme)"), + ] = None + # Protocol-specific options + pure_iscsi_cidr: Annotated[ + str | None, + Field( + description="CIDR of FlashArray iSCSI targets hosts can connect to", + ), + ] = None + pure_iscsi_cidr_list: Annotated[ + str | None, + Field(description="Comma-separated list of CIDR for iSCSI targets"), + ] = None + pure_nvme_cidr: Annotated[ + str | None, + Field(description="CIDR of FlashArray NVMe targets hosts can connect to"), + ] = None + + pure_nvme_cidr_list: Annotated[ + str | None, + Field(description="Comma-separated list of CIDR for NVMe targets"), + ] = None + pure_nvme_transport: Annotated[ + Literal["tcp"] | None, # note(gboutry): roce not supported yet + Field(description="NVMe transport layer"), + ] = None + + # Host and protocol tuning + pure_host_personality: Annotated[ + Personality | None, Field(description="Host personality for protocol tuning") + ] = None + + # Storage management + pure_automatic_max_oversubscription_ratio: Annotated[ + bool | None, + Field(description="Automatically determine oversubscription ratio"), + ] = None + pure_eradicate_on_delete: Annotated[ + bool | None, + Field( + description="Immediately eradicate volumes on delete " + "(WARNING: not recoverable)", + ), + ] = None + + # Replication settings + pure_replica_interval_default: Annotated[ + int | None, Field(description="Snapshot replication interval in seconds") + ] = None + pure_replica_retention_short_term_default: Annotated[ + int | None, + Field(description="Retain all snapshots on target for this time (seconds)"), + ] = None + pure_replica_retention_long_term_per_day_default: Annotated[ + int | None, Field(description="Retain how many snapshots for each day") + ] = None + pure_replica_retention_long_term_default: Annotated[ + int | None, + Field(description="Retain snapshots per day on target for this time (days)"), + ] = None + pure_replication_pg_name: Annotated[ + str | None, + Field( + description="Pure Protection Group name for async replication", + ), + ] = None + pure_replication_pod_name: Annotated[ + str | None, + Field(description="Pure Pod name for sync replication"), + ] = None + + # Advanced replication + pure_trisync_enabled: Annotated[ + bool | None, + Field(description="Enable 3-site replication (sync + async)"), + ] = None + pure_trisync_pg_name: Annotated[ + str | None, + Field(description="Protection Group name for trisync replication"), + ] = None + + # SSL and security + driver_ssl_cert: Annotated[ + str | None, Field(description="SSL certificate content in PEM format") + ] = None + + # Performance options + use_multipath_for_image_xfer: Annotated[ + bool | None, + Field( + description="Enable multipathing for image transfer operations", + ), + ] = None + + +class PureStorageBackend(StorageBackendBase): + """Pure Storage FlashArray backend implementation.""" + + backend_type = "purestorage" + display_name = "Pure Storage FlashArray" + + @property + def charm_name(self) -> str: + """Return the charm name for this backend.""" + return "cinder-volume-purestorage" + + @property + def charm_channel(self) -> str: + """Return the charm channel for this backend.""" + return "latest/edge" + + @property + def charm_revision(self) -> str | None: + """Return the charm revision for this backend.""" + return None + + @property + def charm_base(self) -> str: + """Return the charm base for this backend.""" + return "ubuntu@24.04" + + def config_type(self) -> type[StorageBackendConfig]: + """Return the configuration class for Pure Storage backend.""" + return PureStorageConfig diff --git a/sunbeam-python/sunbeam/storage/base.py b/sunbeam-python/sunbeam/storage/base.py new file mode 100644 index 000000000..757e7e7b8 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/base.py @@ -0,0 +1,621 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Storage backend base class with integrated Terraform functionality.""" + +import enum +import ipaddress +import logging +import re +import types +import typing +from pathlib import Path +from typing import Any + +import click +import pydantic +from packaging.version import Version +from rich.console import Console +from rich.table import Table + +from sunbeam import utils +from sunbeam.clusterd.client import Client +from sunbeam.core.common import BaseStep, RiskLevel, run_plan +from sunbeam.core.deployment import Deployment, Networks +from sunbeam.core.juju import JujuHelper +from sunbeam.core.manifest import Manifest, StorageBackendConfig +from sunbeam.core.terraform import TerraformHelper, TerraformInitStep +from sunbeam.storage.cli_base import StorageBackendCLIBase +from sunbeam.storage.models import ( + BackendAlreadyExistsException, + SecretDictField, +) +from sunbeam.storage.service import StorageBackendService +from sunbeam.storage.steps import ( + BaseStorageBackendDeployStep, + BaseStorageBackendDestroyStep, + ValidateStoragePrerequisitesStep, +) + +LOG = logging.getLogger(__name__) +console = Console() + +# Juju application name validation pattern +# Based on Juju's naming rules: must start with letter, contain only +# letters, numbers, hyphens. Cannot end with hyphen, cannot have +# consecutive hyphens, cannot have numbers after final hyphen +JUJU_APP_NAME_PATTERN = re.compile(r"^[a-z]([a-z0-9]*(-[a-z0-9]*)*)?$") + +# Regex pattern for validating FQDN (Fully Qualified Domain Name) +FQDN_PATTERN = ( + r"^[a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?" + r"(\.[a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?)*$" +) + + +def validate_juju_application_name(name: str) -> bool: + """Validate that a name is a valid Juju application name. + + Args: + name: The application name to validate + + Returns: + True if valid, False otherwise + """ + if not name: + return False + + # Check basic pattern + if not JUJU_APP_NAME_PATTERN.match(name): + return False + + # Additional checks for edge cases + if name.endswith("-"): + return False + + if "--" in name: + return False + + # Check that numbers don't appear after the final hyphen + if "-" in name: + parts = name.split("-") + last_part = parts[-1] + if any(char.isdigit() for char in last_part): + return False + + return True + + +BackendConfig = typing.TypeVar("BackendConfig", bound=StorageBackendConfig) + + +class StorageBackendBase(typing.Generic[BackendConfig]): + """Base class for storage backends with integrated Terraform functionality.""" + + backend_type: str = "base" + display_name: str = "Base Storage Backend" + version = Version("0.0.1") + user_manifest = None # Path to user manifest file + # By default, any storage backend is considered edge risk. + # It will be needed to override in subclasses if the backend is + # considered stable. + risk_availability: RiskLevel = RiskLevel.EDGE + + def __init__(self) -> None: + """Initialize storage backend.""" + self.tfplan = "storage-backend-plan" + self.tfplan_dir = "deploy-storage-backend" + self._manifest: Manifest | None = None + + # Common CLI registration pattern (Abstraction 3: CLI registration) + def register_add_cli(self, add: click.Group) -> None: # noqa: F811 + """Register 'sunbeam storage add ' command. + + Default implementation delegates to CLI class following the pattern. + Subclasses can override if they need custom behavior. + """ + cli_class = self._get_cli_class() + cli = cli_class(self) + cli.register_add_cli(add) + + def register_options_cli(self, options: click.Group) -> None: + """Register 'sunbeam storage options ' command. + + Default implementation delegates to CLI class following the pattern. + Subclasses can override if they need custom behavior. + """ + cli_class = self._get_cli_class() + cli = cli_class(self) + cli.register_options_cli(options) + + # Terraform-related properties and methods + @property + def manifest(self) -> Manifest: + """Return the manifest.""" + if self._manifest: + return self._manifest + + manifest = click.get_current_context().obj.get_manifest(self.user_manifest) + self._manifest = manifest + if self._manifest is None: + raise ValueError("Failed to load manifest") + return self._manifest + + @property + def tfvar_config_key(self) -> str: + """Config key for storing Terraform variables in clusterd.""" + return "TerraformVarsStorageBackends" + + def config_key(self, name: str) -> str: + """Config key for a specific backend instance.""" + return f"Storage-{name}" + + def create_deploy_step( + self, + deployment: Deployment, + client: Client, + tfhelper: TerraformHelper, + jhelper: JujuHelper, + manifest: Manifest, + preseed: dict, + backend_name: str, + model: str, + accept_defaults: bool = False, + ) -> BaseStep: + """Create a deployment step for this backend.""" + return BaseStorageBackendDeployStep( + deployment, + client, + tfhelper, + jhelper, + manifest, + preseed, + backend_name, + self, + model, + accept_defaults, + ) + + def create_destroy_step( + self, + deployment: Deployment, + client, + tfhelper: TerraformHelper, + jhelper: JujuHelper, + manifest: Manifest, + backend_name: str, + model: str, + ) -> BaseStep: + """Create a destruction step for this backend.""" + return BaseStorageBackendDestroyStep( + deployment, + client, + tfhelper, + jhelper, + manifest, + backend_name, + self, + model, + ) + + def register_terraform_plan(self, deployment: Deployment) -> None: + """Register storage backend Terraform plan with deployment system.""" + import shutil + + from sunbeam.core.terraform import TerraformHelper + + # Get the plan source path + backend_self_contained = ( + Path(__file__).parent.parent.parent.parent.parent.parent + / "etc/deploy-storage" # / "backends" / self.name / self.tfplan_dir + ) + + if backend_self_contained.exists(): + plan_source = backend_self_contained + else: + raise FileNotFoundError( + f"Terraform plan not found at {backend_self_contained}" + ) + + # Copy plan to deployment's plans directory + dst = deployment.plans_directory / self.tfplan_dir + shutil.copytree(plan_source, dst, dirs_exist_ok=True) + + # Create TerraformHelper + env = {} + env.update(deployment._get_juju_clusterd_env()) + env.update(deployment.get_proxy_settings()) + + tfhelper = TerraformHelper( + path=dst, + plan=self.tfplan, + tfvar_map={}, + backend="http", + env=env, + clusterd_address=deployment.get_clusterd_http_address(), + ) + + # Register the helper with the deployment's tfhelpers + deployment._tfhelpers[self.tfplan] = tfhelper + + def add_backend_instance( + self, + deployment: Deployment, + name: str, + config: dict, + console: Console, + accept_defaults: bool = False, + ) -> None: + """Add a storage backend using Terraform deployment.""" + # Validate backend name follows Juju application naming rules + if not validate_juju_application_name(name): + raise click.ClickException( + f"Invalid backend name '{name}'. " + "Backend names must be valid Juju application names: " + "start with a letter, contain only lowercase letters, numbers," + "and hyphens, cannot end with hyphen, cannot" + "have consecutive hyphens, and cannot have numbers" + "after the final hyphen." + ) + + service = StorageBackendService( + deployment, JujuHelper(deployment.juju_controller) + ) + if service.backend_exists(name, self.backend_type): + raise BackendAlreadyExistsException(f"Backend {name!r} already exists") + + # Register our Terraform plan with the deployment system + self.register_terraform_plan(deployment) + + # Get standard Sunbeam helpers + client = deployment.get_client() + tfhelper = deployment.get_tfhelper(self.tfplan) + jhelper = JujuHelper(deployment.juju_controller) + + plan = [ + ValidateStoragePrerequisitesStep(deployment, client, jhelper), + TerraformInitStep(tfhelper), + self.create_deploy_step( + deployment, + client, + tfhelper, + jhelper, + self.manifest, + config, + name, + deployment.openstack_machines_model, + accept_defaults, + ), + ] + + run_plan(plan, console) + + def _get_field_descriptions(self, config_class: type[BackendConfig]) -> dict: + """Extract field descriptions from a Pydantic v2 model class.""" + desc: dict[str, str] = {} + for field_name, field_info in config_class.model_fields.items(): + desc[field_name] = field_info.description or "No description available" + return desc + + def _field_is_secret(self, finfo) -> bool: + """Check if a field is marked as a secret.""" + for constraint in finfo.metadata: + if isinstance(constraint, SecretDictField): + return True + return False + + def _format_config_value(self, value, is_secret: bool) -> str: + """Format configuration value for display, masking sensitive data.""" + display_value = str(value) + if is_secret: + display_value = "*" * min(8, len(display_value)) if display_value else "" + if len(display_value) > 23: + display_value = display_value[:20] + "..." + return display_value + + def _format_type(self, annotation: type) -> str: + """Return a consistent, human-readable representation of a type annotation.""" + origin = typing.get_origin(annotation) + args = typing.get_args(annotation) + + # Handle Optional / Union[..., None] + if origin is typing.Union and type(None) in args: + non_none_args = [a for a in args if a is not type(None)] + inner = " | ".join(self._format_type(a) for a in non_none_args) + return inner + + # Handle other Unions + if origin in (typing.Union, types.UnionType): + if args and args[-1] is type(None): + args = args[:-1] + return " | ".join(self._format_type(a) for a in args) + + # Handle Literal types + if origin is typing.Literal: + values = ", ".join(repr(a) for a in args) + return f"{values}" + + # Handle Enum subclasses + if isinstance(annotation, type) and issubclass(annotation, enum.Enum): + # Display allowed values (stringified) + options = ", ".join(repr(e.value) for e in annotation) + return f"{options}" + + # Handle parametrised generics, e.g., list[str], dict[str, int] + if origin is not None: + origin_name = getattr(origin, "__name__", str(origin)) + inner = ", ".join(self._format_type(a) for a in args) + return f"{origin_name}[{inner}]" if args else origin_name + + # Handle bare types like int, bool, str, MyClass + if hasattr(annotation, "__name__"): + return annotation.__name__ + + # Handle special typing constructs (e.g. Any) + return str(annotation) + + def _extract_field_info(self, field_info: pydantic.fields.FieldInfo) -> tuple: + """Extract field type, default value, and description from field info.""" + if field_info.annotation: + field_type = self._format_type(field_info.annotation) + else: + field_type = "str" + + if field_info.is_required(): + field_type += " [red]Required[/red]" + + description = field_info.description or "No description" + + return field_type, description + + def display_config_options(self) -> None: + """Display available configuration options for this backend.""" + console.print( + f"[blue]Available configuration options for {self.display_name}:[/blue]" + ) + fields = self.config_type().model_fields + if not fields: + console.print( + " Configuration options are managed dynamically via Terraform." + ) + console.print( + " Use 'sunbeam storage config show' to see current configuration." + ) + return + + table = Table(show_header=True, header_style="bold blue") + table.add_column("Option", style="cyan") + table.add_column("Type", style="green") + table.add_column("Description", style="white") + + for field_name, finfo in fields.items(): + if field_name == "name": + continue + try: + ftype, descr = self._extract_field_info(finfo) + table.add_row(field_name, ftype, descr) + except Exception: + table.add_row(field_name, "str", "Configuration option") + + console.print(table) + + def display_config_table(self, backend_name: str, config: BackendConfig) -> None: + """Display current configuration in a formatted table for this backend.""" + table = Table( + title=f"Configuration for {self.display_name} backend '{backend_name}'", + show_header=True, + header_style="bold blue", + title_style="bold cyan", + border_style="blue", + ) + + table.add_column("Option", style="cyan", no_wrap=True, width=30) + table.add_column("Value", style="green", width=25) + table.add_column("Description", style="dim", width=50) + + field_descriptions = self._get_field_descriptions(self.config_type()) + for field, finfo in self.config_type().model_fields.items(): + value = getattr(config, field, None) + if not value: + continue + # Skip empty values (None, empty string, empty dict, empty list) + # But keep 0 and False as valid values + if ( + value is None + or value == "" + or (isinstance(value, (dict, list)) and len(value) == 0) + ): + continue + + display_value = self._format_config_value( + value, is_secret=self._field_is_secret(finfo) + ) + description = field_descriptions.get(field, "Configuration option") + if len(description) > 47: + description = description[:44] + "..." + table.add_row(utils.to_kebab(field), display_value, description) + + if not config: + console.print( + ( + f"[yellow]No configuration found for {self.backend_type} " + f"backend '{backend_name}'[/yellow]" + ) + ) + else: + console.print(table) + console.print( + ( + f"[green]Configuration displayed for {self.display_name} " + f"backend '{backend_name}'[/green]" + ) + ) + + def remove_backend( + self, deployment: Deployment, backend_name: str, console: Console + ) -> None: + """Remove a storage backend using Terraform.""" + # Register our Terraform plan with the deployment system + self.register_terraform_plan(deployment) + + # Get standard Sunbeam helpers + client = deployment.get_client() + tfhelper = deployment.get_tfhelper(self.tfplan) + jhelper = JujuHelper(deployment.juju_controller) + + # Create removal plan - each backend should implement its own destroy step + plan = [ + ValidateStoragePrerequisitesStep(deployment, client, jhelper), + TerraformInitStep(tfhelper), + self.create_destroy_step( + deployment, + client, + tfhelper, + jhelper, + self.manifest, + backend_name, + deployment.openstack_machines_model, + ), + ] + + run_plan(plan, console) + + def config_type(self) -> type[BackendConfig]: + """Return the configuration class for this backend.""" + raise NotImplementedError("Subclasses must implement config_type") + + # Backend-specific properties that subclasses should override + @property + def charm_name(self) -> str: + """Charm name for this backend.""" + raise NotImplementedError("Subclasses must define charm_name") + + @property + def charm_channel(self) -> str: + """Charm channel for this backend.""" + return "latest/stable" + + @property + def charm_revision(self) -> str | None: + """Charm revision for this backend.""" + return None + + @property + def charm_base(self) -> str: + """Charm base for this backend.""" + return "ubuntu@22.04" + + @property + def principal_application(self) -> str: + """Principal application for this backend. + + To override when supporting non-ha backends. + """ + return "cinder-volume" + + def get_endpoint_bindings(self, deployment: Deployment) -> list[dict[str, str]]: + """Endpoint bindings for this backend.""" + return [ + {"space": deployment.get_space(Networks.MANAGEMENT)}, + { + "endpoint": "cinder-volume", + "space": deployment.get_space(Networks.STORAGE), + }, + ] + + def build_terraform_vars( + self, + deployment: Deployment, + manifest: Manifest, + backend_name: str, + config: BackendConfig, + ) -> dict[str, Any]: + """Generate Terraform variables for Pure Storage backend deployment.""" + # Map our configuration fields to the correct charm configuration option names + config_dict = config.model_dump(exclude_none=True, by_alias=True) + + # Secret fields that will be translated to juju secrets + # K: config field name, V: field key in juju secret + secret_fields = {} + alias_generator = self.config_type().model_config.get("alias_generator") + if alias_generator is None: + raise RuntimeError( + "Alias generator not defined in config model StorageBackendConfig" + ) + # raise if alias generator is callable + if not hasattr(alias_generator, "generate_aliases"): + raise RuntimeError( + "Alias generator is not of type AliasGenerator in" + " config model StorageBackendConfig" + ) + for fname, finfo in self.config_type().model_fields.items(): + for constraint in finfo.metadata: + if isinstance(constraint, SecretDictField): + secret_fields[alias_generator.generate_aliases(fname)[2]] = ( # type: ignore + constraint.field + ) + + charm_channel = self.charm_channel + charm_revision = None + if backends_cfg := manifest.storage.root.get(self.backend_type): + if backend_cfg := backends_cfg.root.get(backend_name): + if charm_cfg := backend_cfg.software.charms.get(self.charm_name): + if channel := charm_cfg.channel: + charm_channel = channel + if revision := charm_cfg.revision: + charm_revision = revision + + # Build Terraform variables to match the plan's expected format + tfvars = { + "principal_application": self.principal_application, + "charm_name": self.charm_name, + "charm_base": self.charm_base, + "charm_channel": charm_channel, + "charm_revision": charm_revision, + "endpoint_bindings": self.get_endpoint_bindings(deployment), + "charm_config": config_dict, + "secrets": secret_fields, + } + + return tfvars + + # Common utility methods (Abstraction 2: IP/FQDN validation) + @staticmethod + def _validate_ip_or_fqdn(value: str) -> str: + """Validate IP address or FQDN. + + Args: + value: IP address or FQDN to validate + + Returns: + The validated value + + Raises: + click.BadParameter: If value is not a valid IP or FQDN + """ + try: + ipaddress.ip_address(value) + return value + except ValueError: + # If not a valid IP, check if it's a valid FQDN + if re.match(FQDN_PATTERN, value): + return value + raise click.BadParameter("Must be a valid IP address or FQDN") + + def _get_cli_class(self) -> type[StorageBackendCLIBase]: + """Get the CLI class for this backend. + + Subclasses should override this to return their CLI class. + Default implementation attempts to import based on naming convention. + """ + try: + # Try to import CLI class based on naming convention + module_path = f"sunbeam.storage.backends.{self.backend_type}.cli" + cli_module = __import__( + module_path, fromlist=[f"{self.backend_type.title()}CLI"] + ) + cli_class_name = f"{self.backend_type.title()}CLI" + return getattr(cli_module, cli_class_name) + except (ImportError, AttributeError): + LOG.debug(f"{self.backend_type} does not implement custom cli class") + return StorageBackendCLIBase diff --git a/sunbeam-python/sunbeam/storage/cli_base.py b/sunbeam-python/sunbeam/storage/cli_base.py new file mode 100644 index 000000000..af8c76b07 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/cli_base.py @@ -0,0 +1,259 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Base CLI functionality for storage backends. + +This module contains the base CLI class that provides common functionality +for all storage backend CLI implementations. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import click +import yaml +from rich.console import Console + +from sunbeam.core.deployment import Deployment + +if TYPE_CHECKING: + from sunbeam.storage.base import StorageBackendBase + + +console = Console() + + +class StorageBackendCLIBase: + """Base CLI functionality for storage backends. + + This class provides common CLI operations for storage backends including: + - Loading configuration files (YAML) + - Building Click parameters from Pydantic models + - Registering add/remove/config commands + - Handling interactive and non-interactive modes + """ + + backend: StorageBackendBase + + def __init__(self, backend: StorageBackendBase): + """Initialize CLI with a backend instance. + + Args: + backend: The storage backend instance (must have config_class attribute) + """ + self.backend = backend + + def _load_config_file(self, path: Path | None = None) -> dict[str, Any]: + """Load YAML config file into a dictionary. + + Args: + path: Path to the configuration file + + Returns: + Dictionary containing the configuration + """ + if not path: + return {} + text = path.read_text() + return dict(yaml.safe_load(text) or {}) + + def _click_type_for(self, field_info) -> click.types.ParamType: + """Map pydantic field to Click type. + + Args: + field_info: Pydantic field info object + + Returns: + Appropriate Click parameter type + """ + ann = getattr(field_info, "annotation", None) + typ = None + if ann is not None: + typ = str(ann) + elif hasattr(field_info, "type_"): + typ = str(field_info.type_) + if typ and ("int" in typ): + return click.INT + if typ and ("float" in typ): + return click.FLOAT + if typ and ("bool" in typ): + return click.BOOL + return click.STRING + + def _build_add_params(self) -> list: + """Build Click parameters for the add command from config model.""" + params: list = [] + # name option (not required; prompt in interactive mode) + params.append(click.Argument(["name"], type=str, required=True)) + # config file (optional) + params.append( + click.Option( + ["--config-file"], + type=click.Path(exists=True, dir_okay=False, path_type=Path), + required=False, + help="YAML config file", + ) + ) + params.append( + click.Option( + ["--accept-defaults", "-a"], + is_flag=True, + required=False, + help="In interactive mode, accept default values where available", + ) + ) + # Model-derived options + fields = self.backend.config_type().model_fields + for fname, finfo in fields.items(): + if fname == "name": + continue + opt_name = "--" + fname.replace("_", "-") + click_type = self._click_type_for(finfo) + # For interactive UX, keep CLI options optional; the model + # enforces requiredness. + is_required = False + # Help text + descr = finfo.description + params.append( + click.Option( + [opt_name], type=click_type, required=is_required, help=descr + ) + ) + return params + + def _build_config_from_kwargs(self, kwargs: dict[str, Any]) -> dict[str, Any]: + """Extract config values from Click kwargs. + + Click converts dashes to underscores in parameter names. + + Args: + kwargs: Keyword arguments from Click command + + Returns: + Dictionary with non-None configuration values + """ + cfg: dict[str, Any] = {} + for k, v in kwargs.items(): + if v is None: + continue + cfg[k] = v + return cfg + + def _build_set_params(self) -> list: + """Build Click parameters for config set command. + + All parameters are optional since we only update provided values. + + Returns: + List of Click Option objects + """ + params = [] + # Add config-file option first + params.append( + click.Option( + ["--config-file"], + type=click.Path( + exists=True, dir_okay=False, readable=True, path_type=Path + ), + required=False, + help="YAML config file with updates", + ) + ) + fields = self.backend.config_type().model_fields + for fname, finfo in fields.items(): + if fname == "name": + continue + opt = "--" + fname.replace("_", "-") + # For updates, make everything optional with default None + # so we can detect presence + click_type: click.ParamType = click.STRING + ann = finfo.annotation + if ann is not None and ("bool" in str(ann)): + click_type = click.BOOL + elif ann is not None and ("int" in str(ann)): + click_type = click.INT + elif ann is not None and ("float" in str(ann)): + click_type = click.FLOAT + params.append( + click.Option( + [opt], + type=click_type, + required=False, + default=None, + help=finfo.description, + ) + ) + return params + + def register_add_cli(self, add: click.Group) -> None: # noqa: C901 + """Register 'sunbeam storage add ' command. + + Includes typed options and a --config-file flag. + Supports both interactive and non-interactive modes. + + Args: + add: Click group to add the command to + """ + + def add_callback(**kwargs): + deployment: Deployment = click.get_current_context().obj + cfg_file = kwargs.pop("config_file", None) + accept_defaults = kwargs.pop("accept_defaults", False) + file_cfg = self._load_config_file(cfg_file) + cli_cfg = self._build_config_from_kwargs(kwargs) + # Determine if interactive: no config-file and no CLI options supplied + provided_cli_values = {k: v for k, v in cli_cfg.items() if v is not None} + # Name is guaranted to be given through the CLI. + backend_name = provided_cli_values.pop("name") + + merged = {**file_cfg, **provided_cli_values} + self.backend.add_backend_instance( + deployment, backend_name, merged, console, accept_defaults + ) + + # Build command dynamically with parameters + params = self._build_add_params() + help_text = ( + f"Add {self.backend.display_name} backend.\n\n" + "Behavior:\n\n" + "- If no options are provided, runs in interactive mode and prompts " + "for all required fields.\n\n" + "- If options and/or --config-file are provided, runs non-" + "interactively and validates against the model.\n\n" + "- In non-interactive mode.\n\n" + "Examples:\n\n" + f" sunbeam storage add {self.backend.backend_type} my-backend\n\n" + f" sunbeam storage add {self.backend.backend_type} my-backend " + f"--config-file {self.backend.backend_type}.yaml\n" + ) + cmd = click.Command( + name=self.backend.backend_type, + params=params, + callback=add_callback, + help=help_text, + ) + add.add_command(cmd) + + def register_options_cli(self, options: click.Group) -> None: + """Register 'sunbeam storage options ' command. + + Args: + options: Click group to add the command to + """ + + def options_callback(**kwargs): + self.backend.display_config_options() + + help_text = ( + f"Show configuration options for all {self.backend.display_name} backends." + "\n\n" + "Displays the current configuration values for each backend instance.\n" + ) + cmd = click.Command( + name=self.backend.backend_type, + callback=options_callback, + help=help_text, + ) + options.add_command(cmd) diff --git a/sunbeam-python/sunbeam/storage/manager.py b/sunbeam-python/sunbeam/storage/manager.py new file mode 100644 index 000000000..0d1c5783d --- /dev/null +++ b/sunbeam-python/sunbeam/storage/manager.py @@ -0,0 +1,298 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +import importlib +import logging +import pathlib +import typing +from typing import Dict + +import click +from rich.console import Console +from rich.table import Table +from snaphelpers import Snap + +from sunbeam.core.common import infer_risk +from sunbeam.core.deployment import Deployment +from sunbeam.core.juju import JujuHelper +from sunbeam.core.manifest import StorageInstanceManifest +from sunbeam.storage.base import StorageBackendBase +from sunbeam.storage.models import BackendNotFoundException, StorageBackendInfo +from sunbeam.storage.service import StorageBackendService + +LOG = logging.getLogger(__name__) +console = Console() + +# Global registry for storage backends +_STORAGE_BACKENDS: Dict[str, StorageBackendBase] = {} + + +@click.group("storage", context_settings={"help_option_names": ["-h", "--help"]}) +@click.pass_context +def storage(ctx): + """Manage Cinder storage backends. + + Provides commands to add, remove, configure and list storage backends. + Supports multiple backend types including Hitachi VSP and others. + """ + # Ensure we have a deployment object + if not hasattr(ctx, "obj") or not isinstance(ctx.obj, Deployment): + raise click.ClickException( + "Storage commands require a valid deployment context. " + "Please ensure sunbeam is properly initialized." + ) + + +class StorageBackendManager: + """Registry for managing storage backends.""" + + _backends: dict[str, StorageBackendBase] = _STORAGE_BACKENDS + _loaded: bool = False + + def __init__(self) -> None: + if not self._backends: + self._load_backends() + + def _load_backends(self) -> None: + """Load all storage backends from the storage/backends directory.""" + if self._loaded: + return + + LOG.debug("Loading storage backends") + import sunbeam.storage.backends + + sunbeam_storage_backends = pathlib.Path( + sunbeam.storage.backends.__file__ + ).parent + + for path in sunbeam_storage_backends.iterdir(): + # Skip non-directories and special files + if not path.is_dir() or path.name.startswith("_") or path.name == "etc": + continue + + backend_name = path.name + backend_module_path = path / "backend.py" + + # Check if the backend.py file exists in the backend directory + if not backend_module_path.exists(): + LOG.debug(f"Skipping {backend_name}: no backend.py file found") + continue + + try: + LOG.debug(f"Loading storage backend: {backend_name}") + # Import the backend module from the backend subdirectory + mod = importlib.import_module( + f"sunbeam.storage.backends.{backend_name}.backend" + ) + + # Look for backend classes + for attr_name in dir(mod): + attr = getattr(mod, attr_name) + if ( + isinstance(attr, type) + and issubclass(attr, StorageBackendBase) + and attr != StorageBackendBase + ): + backend_instance = attr() + self._backends[backend_instance.backend_type] = backend_instance + LOG.debug( + "Registered storage backend: " + + backend_instance.backend_type + ) + + except Exception as e: + LOG.debug("Failed to load storage backend", exc_info=True) + LOG.warning(f"Failed to load storage backend {backend_name}: {e}") + + self._loaded = True + + def get_backend(self, name: str) -> StorageBackendBase: + """Get a storage backend by name.""" + self._load_backends() + if name not in self._backends: + raise ValueError(f"Storage backend '{name}' not found") + return self._backends[name] + + def backends(self) -> typing.Mapping[str, StorageBackendBase]: + """Get all available storage backends.""" + return self._backends + + def get_all_storage_manifests( + self, + ) -> dict[str, dict[str, StorageInstanceManifest]]: + """Return a dict of all feature manifest defaults.""" + manifests: dict[str, dict[str, StorageInstanceManifest]] = {} + for name in self.backends(): + manifests[name] = {} + + return manifests + + def register(self, cli: click.Group, deployment: Deployment) -> None: + """Register storage backend commands with the storage group. + + This function is called from main.py to register all storage backend + commands dynamically based on available backends. + """ + cli.add_command(storage) + try: + self.register_cli_commands(storage, deployment) + LOG.debug("Storage backend commands registered successfully") + except Exception as e: + LOG.error(f"Failed to register storage backend commands: {e}") + raise e + + def register_cli_commands( + self, storage_group: click.Group, deployment: Deployment + ) -> None: + """Register all backend commands with the storage CLI group. + + This follows the provider pattern: create stable top-level groups + and let each backend self-register its subcommands under those groups. + The CLI UX remains the same, e.g.: + sunbeam storage add [...] + sunbeam storage remove + sunbeam storage list all + sunbeam storage config show + sunbeam storage config set key=value ... + sunbeam storage config reset key ... + sunbeam storage config options [name] + """ + self._load_backends() + + # Top-level subgroups + @click.group(name="add") + def add_group(): + """Add a storage backend.""" + pass + + @click.group(name="options") + def options_group(): + """Show storage backend configuration options.""" + pass + + @click.command(name="list") + @click.pass_context + def list_all(ctx): + """List all storage backends.""" + jhelper = JujuHelper(deployment.juju_controller) + service = StorageBackendService(deployment, jhelper) + backends = service.list_backends() + self._display_backends_table(backends) + + @click.command(name="remove") + @click.argument("backend_name", type=str) + @click.option("--force", is_flag=True, help="Skip confirmation prompt") + @click.pass_context + def remove_backend(ctx, backend_name: str, force: bool): + """Remove a storage backend.""" + service = StorageBackendService( + deployment, JujuHelper(deployment.juju_controller) + ) + try: + storage_backend = service.get_backend(backend_name) + except BackendNotFoundException: + console.print( + f"[red]Error: Storage backend {backend_name!r} not found.[/red]" + ) + raise click.Abort() + backend = self.backends().get(storage_backend.type) + if not backend: + console.print( + f"[red]Error: Storage backend type " + f"{storage_backend.type!r} not recognized.[/red]" + ) + raise click.Abort() + if not force: + click.confirm( + f"Remove {backend.display_name} backend {backend_name!r}?", + abort=True, + ) + try: + backend.remove_backend(deployment, backend_name, console) + console.print( + f"Successfully removed {backend.display_name} " + f"backend {backend_name!r}" + ) + except Exception as e: + console.print(f"[red]Error removing backend: {e}[/red]") + raise click.Abort() + + @click.command(name="show") + @click.argument("backend_name", type=str) + @click.pass_context + def show_backend(ctx, backend_name: str): + """Show configuration for a storage backend.""" + service = StorageBackendService( + deployment, JujuHelper(deployment.juju_controller) + ) + try: + storage_backend = service.get_backend(backend_name) + except BackendNotFoundException: + console.print( + f"[red]Error: Storage backend {backend_name!r} not found.[/red]" + ) + raise click.Abort() + backend = self.backends().get(storage_backend.type) + if not backend: + console.print( + f"[red]Error: Storage backend type " + f"{storage_backend.type!r} not recognized.[/red]" + ) + raise click.Abort() + config = backend.config_type().model_validate( + storage_backend.config, by_alias=True + ) + backend.display_config_table(backend_name, config) + + installation_risk = infer_risk(Snap()) + + # Delegate CLI registration to each backend + for backend in self._backends.values(): + if backend.risk_availability > installation_risk: + LOG.debug( + "Not registering backend %r, " + "it is available at a higher risk level", + backend.backend_type, + ) + continue + try: + backend.register_add_cli(add_group) + backend.register_options_cli(options_group) + except Exception as e: + backend_name = getattr(backend, "name", "unknown") + LOG.warning( + "Backend %s failed to register CLI: %s", + backend_name, + e, + ) + raise + + # Mount groups under storage + storage_group.add_command(list_all) + storage_group.add_command(add_group) + storage_group.add_command(show_backend) + storage_group.add_command(options_group) + storage_group.add_command(remove_backend) + + def _display_backends_table(self, backends: list[StorageBackendInfo]) -> None: + """Display backends in a formatted table.""" + if not backends: + console.print("[yellow]No storage backends found[/yellow]") + return + + table = Table(title="Storage Backends") + table.add_column("Name", style="cyan") + table.add_column("Type", style="magenta") + table.add_column("Status", style="green") + table.add_column("Charm", style="blue") + + for backend in backends: + status_style = "green" if backend.status == "active" else "red" + table.add_row( + backend.name, + backend.backend_type, + f"[{status_style}]{backend.status}[/{status_style}]", + backend.charm, + ) + + console.print(table) diff --git a/sunbeam-python/sunbeam/storage/models.py b/sunbeam-python/sunbeam/storage/models.py new file mode 100644 index 000000000..1dd8f8cc9 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/models.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Storage backend models and exceptions.""" + +from typing import Any, Dict + +import pydantic + +from sunbeam.core.common import SunbeamException + +# ============================================================================= +# Exceptions +# ============================================================================= + + +class StorageBackendException(SunbeamException): + """Base exception for storage backend operations.""" + + pass + + +class BackendNotFoundException(StorageBackendException): + """Raised when storage backend is not found.""" + + pass + + +class BackendAlreadyExistsException(StorageBackendException): + """Raised when storage backend already exists.""" + + pass + + +class BackendValidationException(StorageBackendException): + """Raised when storage backend configuration is invalid.""" + + pass + + +# ============================================================================= +# Data Models +# ============================================================================= + + +class StorageBackendInfo(pydantic.BaseModel): + """Information about a deployed storage backend.""" + + name: str + backend_type: str + status: str + charm: str + config: Dict[str, Any] = {} + + +class SecretDictField: + """Marker class to indicate a field needs to be managed as a juju secret. + + This class is used as a field annotation in Pydantic models to indicate that + the field contains sensitive information (e.g., passwords, API tokens). + + The field name is the name of the key in the Juju secret dictionary. + """ + + def __init__(self, field: str): + self.field = field + + def __repr__(self) -> str: + """Return a string representation of the SecretDictField.""" + return f"SecretDictField(field={self.field})" diff --git a/sunbeam-python/sunbeam/storage/service.py b/sunbeam-python/sunbeam/storage/service.py new file mode 100644 index 000000000..938606e96 --- /dev/null +++ b/sunbeam-python/sunbeam/storage/service.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Storage backend service layer.""" + +import logging + +from rich.console import Console + +from sunbeam.clusterd.models import StorageBackend +from sunbeam.clusterd.service import ( + StorageBackendNotFoundException, +) +from sunbeam.core.deployment import Deployment +from sunbeam.core.juju import JujuHelper +from sunbeam.storage.models import ( + BackendAlreadyExistsException, + BackendNotFoundException, + StorageBackendInfo, +) + +LOG = logging.getLogger(__name__) +console = Console() + + +class StorageBackendService: + """Service layer for storage backend operations.""" + + def __init__(self, deployment: Deployment, jhelper: JujuHelper): + self.deployment = deployment + self.jhelper = jhelper + self.model = jhelper.get_model_name_with_owner( + self.deployment.openstack_machines_model + ) + # Use a consistent config key for all storage backends + self._tfvar_config_key = "TerraformVarsStorageBackends" + + def list_backends(self) -> list[StorageBackendInfo]: + """List all Terraform-managed storage backends with dynamic status. + + Returns: + List of StorageBackendInfo objects for all Terraform-managed + storage backends with real-time status and charm information + """ + backends = [] + client = self.deployment.get_client() + + enabled_backends = client.cluster.get_storage_backends() + # Search each backend type's individual config key + for backend in enabled_backends.root: + try: + # Get actual application name from Terraform config + app_name = backend.name + + # Query actual status and charm from Juju + status = self._get_application_status(self.jhelper, app_name) + charm_name = self._get_application_charm(self.jhelper, app_name) + + backend_info = StorageBackendInfo( + name=backend.name, + backend_type=backend.type, + status=status, + charm=charm_name, + config=backend.config, + ) + backends.append(backend_info) + LOG.debug(f"Found {backend.type} backend: {backend.name}") + except Exception as e: + LOG.warning( + f"Error processing {backend.type} backend {backend.name}: {e}" + ) + continue + return backends + + def _get_application_status(self, jhelper: JujuHelper, app_name: str) -> str: + """Get application status from Juju. + + Args: + jhelper: JujuHelper instance for Juju operations + app_name: Name of the Juju application + + Returns: + Application status string or "unknown" if not found + """ + try: + # Get model status using JujuHelper.get_model_status() + model_status = jhelper.get_model_status( + self.deployment.openstack_machines_model + ) + + # Check if application exists in the model + if app_name in model_status.apps: + app_status = model_status.apps[app_name] + return app_status.app_status.current + + return "not-found" + except Exception as e: + LOG.debug(f"Failed to get status for application {app_name}: {e}") + return "unknown" + + def _get_application_charm(self, jhelper: JujuHelper, app_name: str) -> str: + """Get charm name from Juju. + + Args: + jhelper: JujuHelper instance for Juju operations + app_name: Name of the Juju application + + Returns: + Charm name or fallback name if not found + """ + try: + # Get model status using JujuHelper.get_model_status() + model_status = jhelper.get_model_status( + self.deployment.openstack_machines_model + ) + + # Check if application exists in the model + if app_name in model_status.apps: + app_status = model_status.apps[app_name] + charm_url = app_status.charm + return charm_url + + return "Not Found" + + except Exception as e: + LOG.debug(f"Failed to get charm for application {app_name}: {e}") + return "Unknown" + + def backend_exists(self, backend_name: str, backend_type: str) -> bool: + """Check if a backend exists in Terraform configuration.""" + client = self.deployment.get_client() + try: + backend = client.cluster.get_storage_backend(backend_name) + if backend.type != backend_type: + raise BackendAlreadyExistsException("Backend type mismatch.") + return True + except StorageBackendNotFoundException: + return False + + def get_backend(self, backend_name: str) -> StorageBackend: + """Get a specific storage backend by name.""" + client = self.deployment.get_client() + try: + return client.cluster.get_storage_backend(backend_name) + except StorageBackendNotFoundException as e: + LOG.debug(f"Storage backend not found: {backend_name}", exc_info=True) + raise BackendNotFoundException() from e diff --git a/sunbeam-python/sunbeam/storage/steps.py b/sunbeam-python/sunbeam/storage/steps.py new file mode 100644 index 000000000..3bdb21dfd --- /dev/null +++ b/sunbeam-python/sunbeam/storage/steps.py @@ -0,0 +1,594 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Base step classes for storage backend implementations. + +This module provides base step classes that facilitate the implementation +of storage backend steps. Backends can inherit from these base classes +to get common functionality while customizing specific behavior. +""" + +import logging +from typing import TYPE_CHECKING, Any, Callable + +import pydantic +import tenacity +from rich.console import Console +from rich.status import Status + +from sunbeam.clusterd.client import Client +from sunbeam.clusterd.service import ( + ConfigItemNotFoundException, + StorageBackendNotFoundException, +) +from sunbeam.core.common import ( + BaseStep, + Result, + ResultType, + friendly_terraform_lock_retry_callback, + read_config, + update_config, +) +from sunbeam.core.deployment import Deployment +from sunbeam.core.juju import ( + ControllerNotFoundException, + ControllerNotReachableException, + JujuException, + JujuHelper, +) +from sunbeam.core.manifest import Manifest +from sunbeam.core.questions import ( + ConfirmQuestion, + PasswordPromptQuestion, + PromptQuestion, + Question, + QuestionBank, + load_answers, + write_answers, +) +from sunbeam.core.terraform import ( + TerraformException, + TerraformHelper, + TerraformStateLockedException, +) +from sunbeam.storage.models import SecretDictField + +if TYPE_CHECKING: + from sunbeam.storage.base import StorageBackendBase + +LOG = logging.getLogger(__name__) +console = Console() + + +class ValidateStoragePrerequisitesStep(BaseStep): + """Validate that Sunbeam is bootstrapped and storage role is deployed.""" + + def __init__(self, deployment: Deployment, client: Client, jhelper: JujuHelper): + super().__init__( + "Validate storage prerequisites", + "Checking Sunbeam bootstrap and storage role deployment", + ) + self.deployment = deployment + self.client = client + self.jhelper = jhelper + self.OPENSTACK_MACHINE_MODEL = self.deployment.openstack_machines_model + + def _check_juju_authentication(self) -> Result: + """Check if the current user is authenticated with Juju.""" + try: + # Use the existing JujuHelper to check authentication + # If we can list models, we're authenticated + models = self.jhelper.models() + LOG.debug( + f"Juju authentication check successful, found {len(models)} models" + ) + return Result(ResultType.COMPLETED) + + except ControllerNotFoundException: + return Result( + ResultType.FAILED, + "Juju controller not found. Please ensure Sunbeam is bootstrapped:\n" + "'sunbeam cluster bootstrap'", + ) + except ControllerNotReachableException: + return Result( + ResultType.FAILED, + "Juju controller not reachable. Please check network connectivity\n" + "or re-authenticate with 'sunbeam utils juju-login'", + ) + except JujuException as e: + # Check if it's an authentication-related error + error_msg = str(e).lower() + if any( + keyword in error_msg + for keyword in [ + "not logged in", + "authentication", + "unauthorized", + "permission denied", + "please enter password", + ] + ): + return Result( + ResultType.FAILED, + "Not authenticated with Juju controller. Please run:\n" + "'sunbeam utils juju-login'\n" + "or authenticate manually with 'juju login'", + ) + else: + return Result(ResultType.FAILED, f"Juju operation failed: {e}") + except Exception as e: + return Result( + ResultType.FAILED, f"Failed to check Juju authentication: {e}" + ) + + def run(self, status: Status | None = None) -> Result: + """Validate storage backend prerequisites.""" + try: + # 0. Check Juju authentication first + auth_result = self._check_juju_authentication() + if auth_result.result_type != ResultType.COMPLETED: + return auth_result + + # 1. Check if Sunbeam is bootstrapped + is_bootstrapped = self.client.cluster.check_sunbeam_bootstrapped() + if not is_bootstrapped: + return Result( + ResultType.FAILED, + "Deployment not bootstrapped. Please run\n" + "'sunbeam cluster bootstrap' first.", + ) + + # 2. Check if OpenStack model exists + if not self.jhelper.model_exists(self.OPENSTACK_MACHINE_MODEL): + return Result( + ResultType.FAILED, + f"OpenStack model '{self.OPENSTACK_MACHINE_MODEL}' not found. " + "Please deploy OpenStack first with\n" + "'sunbeam configure --openstack'.", + ) + + # 3. Check if storage role is deployed (at least one storage node) + storage_nodes = self.client.cluster.list_nodes_by_role("storage") + if not storage_nodes: + return Result( + ResultType.FAILED, + "No storage role found. Please add storage nodes to the cluster " + "before deploying storage backends.", + ) + + # 4. Check if cinder-volume application exists in OpenStack model + try: + cinder_volume_app = self.jhelper.get_application( + "cinder-volume", self.OPENSTACK_MACHINE_MODEL + ) + if not cinder_volume_app: + return Result( + ResultType.FAILED, + "cinder-volume application not found in OpenStack model. " + "Please deploy OpenStack storage services first.", + ) + except Exception as e: + LOG.debug(f"Failed to check cinder-volume application: {e}") + return Result( + ResultType.FAILED, + "Unable to verify cinder-volume application. " + "Please ensure OpenStack storage services are deployed.", + ) + + return Result(ResultType.COMPLETED) + + except Exception as e: + LOG.error(f"Failed to validate storage prerequisites: {e}") + return Result(ResultType.FAILED, str(e)) + + +def basemodel_validator( + model: type[pydantic.BaseModel], +) -> Callable[[str], Callable[[Any], None]]: + """Return a factory producing value validators for Pydantic model fields.""" + validator = model.__pydantic_validator__ + fields = dict(model.model_fields.items()) + constructed = model.model_construct() + + def field_validator(field: str) -> Callable[[Any], None]: + if field not in fields: + raise ValueError(f"{model.__name__} has no field named {field!r}") + + def value_validator(value: Any) -> None: + try: + validator.validate_assignment(constructed, field, value) + except pydantic.ValidationError as exc: + messages: list[str] = [] + for error in exc.errors(): + location = ".".join(str(part) for part in error.get("loc", ())) + message = error.get("msg", str(error)) + if location: + messages.append(f"{location}: {message}") + else: + messages.append(message) + raise ValueError("; ".join(messages)) + + return value_validator + + return field_validator + + +def generate_questions_from_config( + config_type: type[pydantic.BaseModel], *, optional: bool = False +) -> dict[str, Question]: + questions = {} # type: ignore + field_validator = basemodel_validator(config_type) + for field, finfo in config_type.model_fields.items(): + if optional and finfo.is_required(): + continue + if not optional and not finfo.is_required(): + continue + question_type: type[Question] = PromptQuestion + for constraint in finfo.metadata: + if isinstance(constraint, SecretDictField): + question_type = PasswordPromptQuestion + prompt_suffix = " (optional)" if optional else "" + questions[field] = question_type( + f"Enter value for {field!r}{prompt_suffix}", + description=finfo.description, + validation_function=field_validator(field), + ) + return questions + + +class BaseStorageBackendDeployStep(BaseStep): + """Base class for storage backend deployment steps. + + Provides common deployment functionality that backends can inherit from + and customize as needed. + """ + + def __init__( + self, + deployment: Deployment, + client: Client, + tfhelper: TerraformHelper, + jhelper: JujuHelper, + manifest: Manifest, + preseed: dict, + backend_name: str, + backend_instance: "StorageBackendBase", + model: str, + accept_defaults: bool = False, + ): + super().__init__( + f"Deploy {backend_instance.display_name} backend {backend_name}", + f"Deploying {backend_instance.display_name} storage backend {backend_name}", + ) + self.deployment = deployment + self.client = client + self.tfhelper = tfhelper + self.jhelper = jhelper + self.manifest = manifest + self.backend_name = backend_name + self.backend_instance = backend_instance + self.model = model + self.preseed = preseed + self.accept_defaults = accept_defaults + self.variables: dict = {} + self.config_key = self.backend_instance.config_key(self.backend_name) + + def prompt( + self, + console: Console | None = None, + show_hint: bool = False, + ) -> None: + """Determines if the step can take input from the user. + + Prompts are used by Steps to gather the necessary input prior to + running the step. Steps should not expect that the prompt will be + available and should provide a reasonable default where possible. + """ + self.variables = load_answers(self.client, self.config_key) + + preseed = {} + if self.manifest and self.manifest.storage: + if backends := self.manifest.storage.root.get( + self.backend_instance.backend_type + ): + if crt := backends.root.get(self.backend_name): + # Since question generation depends on field name, + # do not dump by alias + preseed = crt.model_dump(by_alias=False)["config"] + + # Preseed from user is higher priority than manifest + preseed.update(self.preseed) + + manifest_configured = False + + if preseed: + manifest_configured = True + + required_questions_bank = QuestionBank( + questions=generate_questions_from_config( + self.backend_instance.config_type() + ), + console=console, + preseed=preseed, + previous_answers=self.variables, + accept_defaults=self.accept_defaults, + show_hint=show_hint, + ) + for name, question in required_questions_bank.questions.items(): + answer = question.ask() + while not answer: + answer = question.ask() + self.variables[name] = answer + + res = ConfirmQuestion( + "Set optional configurations?", + accept_defaults=self.accept_defaults, + default_value=manifest_configured, + ).ask() + + if not res: + write_answers(self.client, self.config_key, self.variables) + return + + optional_questions_bank = QuestionBank( + questions=generate_questions_from_config( + self.backend_instance.config_type(), optional=True + ), + console=console, + preseed=preseed, + previous_answers=self.variables, + accept_defaults=self.accept_defaults, + show_hint=show_hint, + ) + + for name, question in optional_questions_bank.questions.items(): + if ConfirmQuestion( + f"Configure option {name!r}?", + accept_defaults=self.accept_defaults, + default_value=name in preseed, + ).ask(): + self.variables[name] = question.ask() + else: + # Remove variable if previously set for + # subsequent runs + self.variables.pop(name, None) + + try: + # Validate configuration + self.backend_instance.config_type().model_validate( + self.variables, by_name=True + ) + except pydantic.ValidationError as e: + LOG.error(f"Invalid configuration: {e}") + raise e + + write_answers(self.client, self.config_key, self.variables) + + def has_prompts(self) -> bool: + """Returns true if the step has prompts that it can ask the user. + + :return: True if the step can ask the user for prompts, + False otherwise + """ + return True + + @tenacity.retry( + wait=tenacity.wait_fixed(60), + stop=tenacity.stop_after_delay(300), + retry=tenacity.retry_if_exception_type(TerraformStateLockedException), + retry_error_callback=friendly_terraform_lock_retry_callback, + before_sleep=lambda retry_state: console.print( + f"Terraform state locked, retrying in 60 seconds... " + f"(attempt {retry_state.attempt_number}/5)" + ), + ) + def run(self, status: Status | None = None) -> Result: + """Deploy the storage backend using Terraform.""" + # Ensure fresh Juju credentials and Terraform env before applying + try: + self.deployment.reload_tfhelpers() + except Exception as cred_err: + LOG.debug(f"Failed to reload credentials/env: {cred_err}") + + # Merge with existing backends so we don't overwrite them + backend_key = self.backend_name + try: + tfvars = read_config(self.client, self.backend_instance.tfvar_config_key) + except Exception: + tfvars = {} + + model = self.jhelper.get_model(self.model) + + backends = tfvars.setdefault("backends", {}) + + tfvars["model"] = model["model-uuid"] + + # Remove backend if in current config, to ensure we remove the keys + # no longer used + backends.pop(backend_key, None) + validated_config = self.backend_instance.config_type().model_validate( + self.variables, by_name=True + ) + backends[backend_key] = self.backend_instance.build_terraform_vars( + self.deployment, + self.manifest, + self.backend_name, + validated_config, + ) + try: + # Update Terraform variables and apply with merged map + self.tfhelper.update_tfvars_and_apply_tf( + self.client, + self.manifest, + tfvar_config=self.backend_instance.tfvar_config_key, + override_tfvars=tfvars, + ) + except TerraformStateLockedException as e: + # Bubble up to trigger retry + raise e + except Exception as e: + LOG.error( + f"Failed to deploy {self.backend_instance.display_name} " + f"backend {self.backend_name}: {e}" + ) + return Result(ResultType.FAILED, str(e)) + # Let's save backend if not present + self.client.cluster.add_storage_backend( + self.backend_name, + self.backend_instance.backend_type, + validated_config.model_dump(exclude_none=True, by_alias=True), + self.backend_instance.principal_application, + model["model-uuid"], + ) + + try: + self.jhelper.wait_application_ready( + self.backend_name, + model["model-uuid"], + accepted_status=self.get_accepted_application_status(), + timeout=self.get_application_timeout(), + ) + except TimeoutError as e: + LOG.warning(str(e)) + return Result(ResultType.FAILED, str(e)) + + console.print( + f"Successfully deployed {self.backend_instance.display_name} " + f"backend {self.backend_name!r}" + ) + return Result(ResultType.COMPLETED) + + def get_application_timeout(self) -> int: + """Return application timeout in seconds. Override for custom timeout.""" + return 1200 # 20 minutes, same as cinder-volume + + def get_accepted_application_status(self) -> list[str]: + """Return accepted application status.""" + return ["active"] + + +class BaseStorageBackendDestroyStep(BaseStep): + """Base class for storage backend destruction steps. + + Provides common destruction functionality that backends can inherit from + and customize as needed. Handles Terraform state cleanup and configuration + removal from clusterd. + """ + + def __init__( + self, + deployment: Deployment, + client: Client, + tfhelper: TerraformHelper, + jhelper: JujuHelper, + manifest: Manifest, + backend_name: str, + backend_instance: "StorageBackendBase", + model: str, + ): + super().__init__( + f"Destroy {backend_instance.display_name} backend {backend_name}", + f"Destroying {backend_instance.display_name} storage " + f"backend {backend_name}", + ) + self.deployment = deployment + self.client = client + self.tfhelper = tfhelper + self.jhelper = jhelper + self.manifest = manifest + self.backend_name = backend_name + self.backend_instance = backend_instance + self.model = model + + @tenacity.retry( + wait=tenacity.wait_fixed(60), + stop=tenacity.stop_after_delay(300), + retry=tenacity.retry_if_exception_type(TerraformStateLockedException), + retry_error_callback=friendly_terraform_lock_retry_callback, + before_sleep=lambda retry_state: console.print( + f"Terraform state locked, retrying in 60 seconds... " + f"(attempt {retry_state.attempt_number}/5)" + ), + ) + def run(self, status: Status | None = None) -> Result: + """Run the destroy step atomically. + + This step removes the backend from the Terraform configuration + and applies the changes to destroy the associated resources. + The operation is atomic: either it succeeds completely or fails + without modifying the configuration. + """ + # Ensure fresh Juju credentials and Terraform env before destroying/applying + try: + self.deployment.reload_tfhelpers() + except Exception as cred_err: + LOG.debug(f"Failed to reload credentials/env: {cred_err}") + + # First, read and validate the current configuration + try: + tfvars = read_config(self.client, self.backend_instance.tfvar_config_key) + except ConfigItemNotFoundException: + LOG.warning(f"No configuration found for backend {self.backend_name}") + tfvars = {} + + backends = tfvars.get("backends", {}) + + # Drop backend from current configuration + backends.pop(self.backend_name, None) + + # For removal: update config and apply atomically + LOG.info(f"Performing removal for backend {self.backend_name}") + LOG.info(f"Remaining backends after removal: {list(tfvars['backends'].keys())}") + + # First update the configuration + update_config( + self.client, + self.backend_instance.tfvar_config_key, + tfvars, + ) + LOG.info("Configuration updated, now running terraform apply...") + + try: + LOG.info( + f"Writing Terraform variables with backends: " + f"{list(tfvars.get('backends', {}).keys())}" + ) + self.tfhelper.update_tfvars_and_apply_tf( + self.client, + self.manifest, + tfvar_config=self.backend_instance.tfvar_config_key, + override_tfvars=tfvars, + ) + except TerraformStateLockedException as e: + # Bubble up to trigger retry + LOG.debug("Error: Terraform state locked") + raise e + except TerraformException: + # Restore the backend configuration if apply fails + LOG.debug("Terraform apply failed", exc_info=True) + return Result( + ResultType.FAILED, + f"Failed to destroy backend {self.backend_name!r}", + ) + + try: + self.client.cluster.delete_storage_backend(self.backend_name) + except StorageBackendNotFoundException: + LOG.debug(f"Backend {self.backend_name} not found in clusterd") + + try: + # Wipe previously saved answers + self.client.cluster.delete_config( + self.backend_instance.config_key(self.backend_name) + ) + except ConfigItemNotFoundException: + LOG.debug( + f"Configuration for backend {self.backend_name} not found in clusterd" + ) + + return Result(ResultType.COMPLETED) + + def get_application_timeout(self) -> int: + """Return application timeout in seconds.""" + return 1200 # 20 minutes, same as cinder-volume diff --git a/sunbeam-python/sunbeam/utils.py b/sunbeam-python/sunbeam/utils.py index 5d8d520df..50146fb97 100644 --- a/sunbeam-python/sunbeam/utils.py +++ b/sunbeam-python/sunbeam/utils.py @@ -18,8 +18,9 @@ import click import netifaces # type: ignore [import-untyped] +import pydantic.alias_generators -from sunbeam.core.common import SunbeamException +from sunbeam.errors import SunbeamException from sunbeam.lazy import LazyImport if typing.TYPE_CHECKING: @@ -408,3 +409,8 @@ def clean_env(): for key in os.environ: if key.startswith("OS_"): os.environ.pop(key) + + +def to_kebab(value: str) -> str: + """Convert a string to kebab-case.""" + return pydantic.alias_generators.to_snake(value).replace("_", "-") diff --git a/sunbeam-python/tests/unit/sunbeam/storage/__init__.py b/sunbeam-python/tests/unit/sunbeam/storage/__init__.py new file mode 100644 index 000000000..12519b28d --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/storage/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 diff --git a/sunbeam-python/tests/unit/sunbeam/storage/backends/__init__.py b/sunbeam-python/tests/unit/sunbeam/storage/backends/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/sunbeam-python/tests/unit/sunbeam/storage/backends/conftest.py b/sunbeam-python/tests/unit/sunbeam/storage/backends/conftest.py new file mode 100644 index 000000000..f931caf25 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/storage/backends/conftest.py @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Common fixtures and utilities for backend-specific tests.""" + +import pytest + +from sunbeam.storage.backends.dellsc.backend import DellSCBackend +from sunbeam.storage.backends.hitachi.backend import HitachiBackend +from sunbeam.storage.backends.purestorage.backend import PureStorageBackend + + +@pytest.fixture +def hitachi_backend(): + """Provide a Hitachi backend instance.""" + return HitachiBackend() + + +@pytest.fixture +def purestorage_backend(): + """Provide a Pure Storage backend instance.""" + return PureStorageBackend() + + +@pytest.fixture +def dellsc_backend(): + """Provide a Dell Storage Center backend instance.""" + return DellSCBackend() + + +@pytest.fixture(params=["hitachi", "purestorage", "dellsc"]) +def any_backend(request): + """Parametrized fixture that provides each backend type.""" + backends = { + "hitachi": HitachiBackend(), + "purestorage": PureStorageBackend(), + "dellsc": DellSCBackend(), + } + return backends[request.param] diff --git a/sunbeam-python/tests/unit/sunbeam/storage/backends/test_common.py b/sunbeam-python/tests/unit/sunbeam/storage/backends/test_common.py new file mode 100644 index 000000000..dc83c98d8 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/storage/backends/test_common.py @@ -0,0 +1,208 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Common base test class for all storage backends. + +This module provides a base test class that can be inherited by backend-specific +test classes to ensure all backends implement the required interface correctly. +""" + +import pytest +from pydantic import BaseModel + +from sunbeam.core.manifest import StorageBackendConfig +from sunbeam.storage.base import StorageBackendBase + + +class BaseBackendTests: + """Base test class for all storage backends. + + This class provides common tests that verify each backend implements + the required interface and behaves correctly. Backend-specific test + classes should inherit from this class and override the `backend` fixture + to provide their specific backend instance. + + Example: + class TestHitachiBackend(BaseBackendTests): + @pytest.fixture + def backend(self, hitachi_backend): + return hitachi_backend + """ + + @pytest.fixture + def backend(self): + """Override this fixture in subclasses to provide the backend instance. + + Raises: + NotImplementedError: If not overridden in subclass. + """ + raise NotImplementedError( + "Subclasses must override the backend fixture to provide a backend instance" + ) + + # Core attribute tests + + def test_backend_has_type(self, backend): + """Test that backend has a type identifier.""" + assert hasattr(backend, "backend_type") + assert isinstance(backend.backend_type, str) + assert len(backend.backend_type) > 0 + + def test_backend_has_display_name(self, backend): + """Test that backend has a display name.""" + assert hasattr(backend, "display_name") + assert isinstance(backend.display_name, str) + assert len(backend.display_name) > 0 + + def test_backend_is_storage_backend_base(self, backend): + """Test that backend inherits from StorageBackendBase.""" + assert isinstance(backend, StorageBackendBase) + + # Charm property tests + + def test_charm_name_is_set(self, backend): + """Test that charm_name property is set.""" + assert backend.charm_name + assert isinstance(backend.charm_name, str) + assert backend.charm_name.startswith("cinder-volume-") + + def test_charm_channel_is_set(self, backend): + """Test that charm_channel property is set.""" + assert backend.charm_channel + assert isinstance(backend.charm_channel, str) + + def test_charm_base_is_set(self, backend): + """Test that charm_base property is set.""" + assert backend.charm_base + assert isinstance(backend.charm_base, str) + + # Configuration tests + + def test_config_type_returns_class(self, backend): + """Test that config_type() returns a class.""" + config_class = backend.config_type() + assert isinstance(config_class, type) + + def test_config_type_is_storage_backend_config(self, backend): + """Test that config_type() returns a StorageBackendConfig subclass.""" + config_class = backend.config_type() + assert issubclass(config_class, StorageBackendConfig) + + def test_config_type_is_pydantic_model(self, backend): + """Test that config_type() returns a Pydantic model.""" + config_class = backend.config_type() + assert issubclass(config_class, BaseModel) + + # Required field tests + + def test_config_has_san_ip_field(self, backend): + """Test that config has san_ip field.""" + config_class = backend.config_type() + assert "san_ip" in config_class.model_fields + + def test_config_has_protocol_field(self, backend): + """Test that config has protocol field.""" + config_class = backend.config_type() + # Protocol field should exist + assert "protocol" in config_class.model_fields + + # Method existence tests + + def test_has_get_endpoint_bindings_method(self, backend): + """Test that backend has get_endpoint_bindings method.""" + assert hasattr(backend, "get_endpoint_bindings") + assert callable(backend.get_endpoint_bindings) + + def test_has_validate_ip_or_fqdn_static_method(self, backend): + """Test that backend class has _validate_ip_or_fqdn static method.""" + # This is a static method on the base class + assert hasattr(StorageBackendBase, "_validate_ip_or_fqdn") + assert callable(StorageBackendBase._validate_ip_or_fqdn) + + def test_has_register_terraform_plan_method(self, backend): + """Test that backend has register_terraform_plan method.""" + assert hasattr(backend, "register_terraform_plan") + assert callable(backend.register_terraform_plan) + + def test_has_add_backend_instance_method(self, backend): + """Test that backend has add_backend_instance method.""" + assert hasattr(backend, "add_backend_instance") + assert callable(backend.add_backend_instance) + + def test_has_remove_backend_method(self, backend): + """Test that backend has remove_backend method.""" + assert hasattr(backend, "remove_backend") + assert callable(backend.remove_backend) + + def test_has_build_terraform_vars_method(self, backend): + """Test that backend has build_terraform_vars method.""" + assert hasattr(backend, "build_terraform_vars") + assert callable(backend.build_terraform_vars) + + +class TestAllBackends(BaseBackendTests): + """Test all backends using parametrized fixture. + + This class runs the common tests against all backends to ensure + they all implement the required interface consistently. + """ + + @pytest.fixture + def backend(self, any_backend): + """Use the parametrized any_backend fixture.""" + return any_backend + + +# Backend uniqueness tests + + +def test_all_backends_have_unique_types( + hitachi_backend, purestorage_backend, dellsc_backend +): + """Test that all backends have unique type identifiers.""" + backends = [hitachi_backend, purestorage_backend, dellsc_backend] + types = [b.backend_type for b in backends] + + # Check no duplicates + assert len(types) == len(set(types)), f"Duplicate backend types found: {types}" + + +def test_all_backends_have_unique_charm_names( + hitachi_backend, purestorage_backend, dellsc_backend +): + """Test that all backends have unique charm names.""" + backends = [hitachi_backend, purestorage_backend, dellsc_backend] + charm_names = [b.charm_name for b in backends] + + # Check no duplicates + assert len(charm_names) == len(set(charm_names)), ( + f"Duplicate charm names found: {charm_names}" + ) + + +@pytest.mark.parametrize( + "backend_type,expected_type", + [ + ("hitachi", "hitachi"), + ("purestorage", "purestorage"), + ("dellsc", "dellsc"), + ], +) +def test_backend_types_match_expected(any_backend, backend_type, expected_type): + """Test that backend types match expected values.""" + if any_backend.backend_type == backend_type: + assert any_backend.backend_type == expected_type + + +@pytest.mark.parametrize( + "backend_type,expected_charm", + [ + ("hitachi", "cinder-volume-hitachi"), + ("purestorage", "cinder-volume-purestorage"), + ("dellsc", "cinder-volume-dellsc"), + ], +) +def test_backend_charm_names_match_expected(any_backend, backend_type, expected_charm): + """Test that backend charm names match expected values.""" + if any_backend.backend_type == backend_type: + assert any_backend.charm_name == expected_charm diff --git a/sunbeam-python/tests/unit/sunbeam/storage/backends/test_dellsc.py b/sunbeam-python/tests/unit/sunbeam/storage/backends/test_dellsc.py new file mode 100644 index 000000000..1fe74efb6 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/storage/backends/test_dellsc.py @@ -0,0 +1,322 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for Dell Storage Center backend.""" + +import pytest + +from tests.unit.sunbeam.storage.backends.test_common import BaseBackendTests + + +class TestDellSCBackend(BaseBackendTests): + """Tests for Dell Storage Center backend. + + Inherits all generic tests from BaseBackendTests and adds + backend-specific tests. + """ + + @pytest.fixture + def backend(self, dellsc_backend): + """Provide Dell SC backend instance.""" + return dellsc_backend + + # Backend-specific tests + + def test_backend_type_is_dellsc(self, backend): + """Test that backend type is 'dellsc'.""" + assert backend.backend_type == "dellsc" + + def test_display_name_mentions_dell(self, backend): + """Test that display name mentions Dell.""" + assert "dell" in backend.display_name.lower() + + def test_charm_name_is_dellsc_charm(self, backend): + """Test that charm name is cinder-volume-dellsc.""" + assert backend.charm_name == "cinder-volume-dellsc" + + def test_dellsc_config_has_required_fields(self, backend): + """Test that Dell SC config has all required fields.""" + config_class = backend.config_type() + fields = config_class.model_fields + + # Verify Dell SC-specific required fields + required_fields = [ + "san_ip", + "san_username", + "san_password", + ] + for field in required_fields: + assert field in fields, f"Required field {field} not found in config" + + def test_dellsc_protocol_is_optional_literal(self, backend): + """Test that protocol field accepts fc or iscsi.""" + config_class = backend.config_type() + protocol_field = config_class.model_fields.get("protocol") + assert protocol_field is not None + + # Test config without protocol (optional) + config_no_protocol = config_class.model_validate( + { + "san-ip": "192.168.1.1", + "san-username": "admin", + "san-password": "secret", + } + ) + assert config_no_protocol.protocol is None + + # Test valid config with fc + valid_config_fc = config_class.model_validate( + { + "san-ip": "192.168.1.1", + "san-username": "admin", + "san-password": "secret", + "protocol": "fc", + } + ) + assert valid_config_fc.protocol == "fc" + + # Test valid config with iscsi + valid_config_iscsi = config_class.model_validate( + { + "san-ip": "192.168.1.1", + "san-username": "admin", + "san-password": "secret", + "protocol": "iscsi", + } + ) + assert valid_config_iscsi.protocol == "iscsi" + + def test_dellsc_san_credentials_are_secret(self, backend): + """Test that SAN credentials are properly marked as secrets.""" + from sunbeam.storage.models import SecretDictField + + config_class = backend.config_type() + + # Check san_username is marked as secret + username_field = config_class.model_fields.get("san_username") + assert username_field is not None + has_secret_marker = any( + isinstance(m, SecretDictField) for m in username_field.metadata + ) + assert has_secret_marker, "san_username should be marked as secret" + + # Check san_password is marked as secret + password_field = config_class.model_fields.get("san_password") + assert password_field is not None + has_secret_marker = any( + isinstance(m, SecretDictField) for m in password_field.metadata + ) + assert has_secret_marker, "san_password should be marked as secret" + + def test_dellsc_secondary_credentials_are_secret(self, backend): + """Test that secondary SAN credentials are properly marked as secrets.""" + from sunbeam.storage.models import SecretDictField + + config_class = backend.config_type() + + # Check secondary_san_username is marked as secret + sec_user_field = config_class.model_fields.get("secondary_san_username") + assert sec_user_field is not None + has_secret_marker = any( + isinstance(m, SecretDictField) for m in sec_user_field.metadata + ) + assert has_secret_marker, "secondary_san_username should be marked as secret" + + # Check secondary_san_password is marked as secret + sec_pass_field = config_class.model_fields.get("secondary_san_password") + assert sec_pass_field is not None + has_secret_marker = any( + isinstance(m, SecretDictField) for m in sec_pass_field.metadata + ) + assert has_secret_marker, "secondary_san_password should be marked as secret" + + def test_dellsc_config_optional_fields_work(self, backend): + """Test that optional fields can be omitted.""" + config_class = backend.config_type() + + # Create config with only required fields + config = config_class.model_validate( + { + "san-ip": "192.168.1.1", + "san-username": "admin", + "san-password": "secret", + } + ) + + # Verify optional fields default to None + assert config.dell_sc_ssn is None + assert config.protocol is None + assert config.volume_backend_name is None + assert config.backend_availability_zone is None + assert config.dell_sc_api_port is None + + def test_dellsc_dell_specific_fields_exist(self, backend): + """Test that Dell SC-specific fields exist.""" + config_class = backend.config_type() + fields = config_class.model_fields + + dell_specific_fields = [ + "dell_sc_ssn", + "dell_sc_api_port", + "dell_sc_server_folder", + "dell_sc_volume_folder", + "dell_server_os", + "dell_sc_verify_cert", + ] + for field in dell_specific_fields: + assert field in fields, f"Dell SC field {field} not found" + + def test_dellsc_dual_dsm_fields_exist(self, backend): + """Test that dual DSM configuration fields exist.""" + config_class = backend.config_type() + fields = config_class.model_fields + + dual_dsm_fields = [ + "secondary_san_ip", + "secondary_san_username", + "secondary_san_password", + "secondary_sc_api_port", + ] + for field in dual_dsm_fields: + assert field in fields, f"Dual DSM field {field} not found" + + def test_dellsc_network_filtering_fields_exist(self, backend): + """Test that network filtering fields exist.""" + config_class = backend.config_type() + fields = config_class.model_fields + + network_fields = [ + "excluded_domain_ips", + "included_domain_ips", + ] + for field in network_fields: + assert field in fields, f"Network filtering field {field} not found" + + def test_dellsc_ssh_fields_exist(self, backend): + """Test that SSH configuration fields exist.""" + config_class = backend.config_type() + fields = config_class.model_fields + + ssh_fields = [ + "ssh_conn_timeout", + "ssh_max_pool_conn", + "ssh_min_pool_conn", + ] + for field in ssh_fields: + assert field in fields, f"SSH field {field} not found" + + def test_dellsc_api_timeout_fields_exist(self, backend): + """Test that API timeout fields exist.""" + config_class = backend.config_type() + fields = config_class.model_fields + + timeout_fields = [ + "dell_api_async_rest_timeout", + "dell_api_sync_rest_timeout", + ] + for field in timeout_fields: + assert field in fields, f"API timeout field {field} not found" + + +class TestDellSCConfigValidation: + """Test Dell SC config validation behavior.""" + + def test_protocol_accepts_only_valid_values(self, dellsc_backend): + """Test that protocol field rejects invalid values.""" + from pydantic import ValidationError + + config_class = dellsc_backend.config_type() + + # Should reject invalid protocol + with pytest.raises(ValidationError) as exc_info: + config_class.model_validate( + { + "san-ip": "192.168.1.1", + "san-username": "admin", + "san-password": "secret", + "protocol": "INVALID", + } + ) + + assert "protocol" in str(exc_info.value).lower() + + def test_boolean_fields_accept_boolean_values(self, dellsc_backend): + """Test that boolean fields accept boolean values.""" + config_class = dellsc_backend.config_type() + + config = config_class.model_validate( + { + "san-ip": "192.168.1.1", + "san-username": "admin", + "san-password": "secret", + "san-thin-provision": True, + "dell-sc-verify-cert": False, + } + ) + assert config.san_thin_provision is True + assert config.dell_sc_verify_cert is False + + def test_integer_fields_accept_integer_values(self, dellsc_backend): + """Test that integer fields accept integer values.""" + config_class = dellsc_backend.config_type() + + config = config_class.model_validate( + { + "san-ip": "192.168.1.1", + "san-username": "admin", + "san-password": "secret", + "dell-sc-ssn": 12345, + "dell-sc-api-port": 3033, + "secondary-sc-api-port": 3033, + "dell-api-async-rest-timeout": 30, + "dell-api-sync-rest-timeout": 60, + } + ) + assert config.dell_sc_ssn == 12345 + assert config.dell_sc_api_port == 3033 + assert config.secondary_sc_api_port == 3033 + assert config.dell_api_async_rest_timeout == 30 + assert config.dell_api_sync_rest_timeout == 60 + + def test_ssh_pool_connection_values(self, dellsc_backend): + """Test that SSH pool connection values are accepted.""" + config_class = dellsc_backend.config_type() + + config = config_class.model_validate( + { + "san-ip": "192.168.1.1", + "san-username": "admin", + "san-password": "secret", + "ssh-conn-timeout": 30, + "ssh-max-pool-conn": 5, + "ssh-min-pool-conn": 1, + } + ) + assert config.ssh_conn_timeout == 30 + assert config.ssh_max_pool_conn == 5 + assert config.ssh_min_pool_conn == 1 + + def test_dual_dsm_configuration(self, dellsc_backend): + """Test that dual DSM configuration works together.""" + config_class = dellsc_backend.config_type() + + config = config_class.model_validate( + { + "san-ip": "192.168.1.1", + "san-username": "admin", + "san-password": "secret", + "secondary-san-ip": "192.168.1.2", + "secondary-san-username": "admin2", + "secondary-san-password": "secret2", + "secondary-sc-api-port": 3034, + } + ) + assert config.secondary_san_ip == "192.168.1.2" + assert config.secondary_san_username == "admin2" + assert config.secondary_san_password == "secret2" + assert config.secondary_sc_api_port == 3034 + + +if __name__ == "__main__": + # This allows running the file directly with pytest + pytest.main([__file__, "-v"]) diff --git a/sunbeam-python/tests/unit/sunbeam/storage/backends/test_hitachi.py b/sunbeam-python/tests/unit/sunbeam/storage/backends/test_hitachi.py new file mode 100644 index 000000000..dbf72e674 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/storage/backends/test_hitachi.py @@ -0,0 +1,244 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for Hitachi VSP storage backend.""" + +import pytest + +from tests.unit.sunbeam.storage.backends.test_common import BaseBackendTests + + +class TestHitachiBackend(BaseBackendTests): + """Tests for Hitachi VSP storage backend. + + Inherits all generic tests from BaseBackendTests and adds + backend-specific tests. + """ + + @pytest.fixture + def backend(self, hitachi_backend): + """Provide Hitachi backend instance.""" + return hitachi_backend + + # Backend-specific tests + + def test_backend_type_is_hitachi(self, backend): + """Test that backend type is 'hitachi'.""" + assert backend.backend_type == "hitachi" + + def test_display_name_mentions_hitachi(self, backend): + """Test that display name mentions Hitachi.""" + assert "hitachi" in backend.display_name.lower() + + def test_charm_name_is_hitachi_charm(self, backend): + """Test that charm name is cinder-volume-hitachi.""" + assert backend.charm_name == "cinder-volume-hitachi" + + def test_hitachi_config_has_required_fields(self, backend): + """Test that Hitachi config has all required fields.""" + config_class = backend.config_type() + fields = config_class.model_fields + + # Verify Hitachi-specific required fields + required_fields = [ + "hitachi_storage_id", + "hitachi_pools", + "san_ip", + "san_username", + "san_password", + "protocol", + ] + for field in required_fields: + assert field in fields, f"Required field {field} not found in config" + + def test_hitachi_protocol_is_literal(self, backend): + """Test that protocol field only accepts FC or iSCSI.""" + config_class = backend.config_type() + protocol_field = config_class.model_fields.get("protocol") + assert protocol_field is not None + + # Test valid config with FC + valid_config_fc = config_class.model_validate( + { + "hitachi-storage-id": "12345", + "hitachi-pools": "pool1", + "san-ip": "192.168.1.1", + "san-username": "admin", + "san-password": "secret", + "protocol": "FC", + } + ) + assert valid_config_fc.protocol == "FC" + + # Test valid config with iSCSI + valid_config_iscsi = config_class.model_validate( + { + "hitachi-storage-id": "12345", + "hitachi-pools": "pool1", + "san-ip": "192.168.1.1", + "san-username": "admin", + "san-password": "secret", + "protocol": "iSCSI", + } + ) + assert valid_config_iscsi.protocol == "iSCSI" + + def test_hitachi_san_credentials_are_secret(self, backend): + """Test that SAN credentials are properly marked as secrets.""" + from sunbeam.storage.models import SecretDictField + + config_class = backend.config_type() + + # Check san_username is marked as secret + username_field = config_class.model_fields.get("san_username") + assert username_field is not None + has_secret_marker = any( + isinstance(m, SecretDictField) for m in username_field.metadata + ) + assert has_secret_marker, "san_username should be marked as secret" + + # Check san_password is marked as secret + password_field = config_class.model_fields.get("san_password") + assert password_field is not None + has_secret_marker = any( + isinstance(m, SecretDictField) for m in password_field.metadata + ) + assert has_secret_marker, "san_password should be marked as secret" + + def test_hitachi_chap_credentials_are_secret(self, backend): + """Test that CHAP credentials are properly marked as secrets.""" + from sunbeam.storage.models import SecretDictField + + config_class = backend.config_type() + + # Check chap_username is marked as secret + chap_user_field = config_class.model_fields.get("chap_username") + assert chap_user_field is not None + has_secret_marker = any( + isinstance(m, SecretDictField) for m in chap_user_field.metadata + ) + assert has_secret_marker, "chap_username should be marked as secret" + + # Check chap_password is marked as secret + chap_pass_field = config_class.model_fields.get("chap_password") + assert chap_pass_field is not None + has_secret_marker = any( + isinstance(m, SecretDictField) for m in chap_pass_field.metadata + ) + assert has_secret_marker, "chap_password should be marked as secret" + + def test_hitachi_config_optional_fields_work(self, backend): + """Test that optional fields can be omitted.""" + config_class = backend.config_type() + + # Create config with only required fields + config = config_class.model_validate( + { + "hitachi-storage-id": "12345", + "hitachi-pools": "pool1", + "san-ip": "192.168.1.1", + "san-username": "admin", + "san-password": "secret", + "protocol": "FC", + } + ) + + # Verify optional fields default to None + assert config.volume_backend_name is None + assert config.backend_availability_zone is None + assert config.hitachi_target_ports is None + assert config.hitachi_copy_speed is None + + def test_hitachi_mirror_rest_credentials_are_secret(self, backend): + """Test that mirror REST credentials are properly marked as secrets.""" + from sunbeam.storage.models import SecretDictField + + config_class = backend.config_type() + + # Check mirror REST username is marked as secret + mirror_user_field = config_class.model_fields.get( + "hitachi_mirror_rest_username" + ) + assert mirror_user_field is not None + has_secret_marker = any( + isinstance(m, SecretDictField) for m in mirror_user_field.metadata + ) + assert has_secret_marker, ( + "hitachi_mirror_rest_username should be marked as secret" + ) + + # Check mirror REST password is marked as secret + mirror_pass_field = config_class.model_fields.get( + "hitachi_mirror_rest_password" + ) + assert mirror_pass_field is not None + has_secret_marker = any( + isinstance(m, SecretDictField) for m in mirror_pass_field.metadata + ) + assert has_secret_marker, ( + "hitachi_mirror_rest_password should be marked as secret" + ) + + +class TestHitachiConfigValidation: + """Test Hitachi config validation behavior.""" + + def test_protocol_accepts_only_valid_values(self, hitachi_backend): + """Test that protocol field rejects invalid values.""" + from pydantic import ValidationError + + config_class = hitachi_backend.config_type() + + # Should reject invalid protocol + with pytest.raises(ValidationError) as exc_info: + config_class.model_validate( + { + "hitachi-storage-id": "12345", + "hitachi-pools": "pool1", + "san-ip": "192.168.1.1", + "san-username": "admin", + "san-password": "secret", + "protocol": "INVALID", + } + ) + + assert "protocol" in str(exc_info.value).lower() + + def test_copy_speed_validates_range(self, hitachi_backend): + """Test that copy_speed validates range (1-15) if configured.""" + config_class = hitachi_backend.config_type() + + # Valid copy speed + config = config_class.model_validate( + { + "hitachi-storage-id": "12345", + "hitachi-pools": "pool1", + "san-ip": "192.168.1.1", + "san-username": "admin", + "san-password": "secret", + "protocol": "FC", + "hitachi-copy-speed": 10, + } + ) + assert config.hitachi_copy_speed == 10 + + def test_boolean_fields_accept_boolean_values(self, hitachi_backend): + """Test that boolean fields accept boolean values.""" + config_class = hitachi_backend.config_type() + + config = config_class.model_validate( + { + "hitachi-storage-id": "12345", + "hitachi-pools": "pool1", + "san-ip": "192.168.1.1", + "san-username": "admin", + "san-password": "secret", + "protocol": "FC", + "use-chap-auth": True, + "hitachi-discard-zero-page": False, + "hitachi-group-create": True, + } + ) + assert config.use_chap_auth is True + assert config.hitachi_discard_zero_page is False + assert config.hitachi_group_create is True diff --git a/sunbeam-python/tests/unit/sunbeam/storage/backends/test_purestorage.py b/sunbeam-python/tests/unit/sunbeam/storage/backends/test_purestorage.py new file mode 100644 index 000000000..5b92a0828 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/storage/backends/test_purestorage.py @@ -0,0 +1,269 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for Pure Storage FlashArray backend.""" + +import pytest + +from tests.unit.sunbeam.storage.backends.test_common import BaseBackendTests + + +class TestPureStorageBackend(BaseBackendTests): + """Tests for Pure Storage FlashArray backend. + + Inherits all generic tests from BaseBackendTests and adds + backend-specific tests. + """ + + @pytest.fixture + def backend(self, purestorage_backend): + """Provide Pure Storage backend instance.""" + return purestorage_backend + + # Backend-specific tests + + def test_backend_type_is_purestorage(self, backend): + """Test that backend type is 'purestorage'.""" + assert backend.backend_type == "purestorage" + + def test_display_name_mentions_pure(self, backend): + """Test that display name mentions Pure Storage.""" + assert "pure" in backend.display_name.lower() + + def test_charm_name_is_purestorage_charm(self, backend): + """Test that charm name is cinder-volume-purestorage.""" + assert backend.charm_name == "cinder-volume-purestorage" + + def test_purestorage_config_has_required_fields(self, backend): + """Test that Pure Storage config has all required fields.""" + config_class = backend.config_type() + fields = config_class.model_fields + + # Verify Pure Storage-specific required fields + required_fields = [ + "san_ip", + "pure_api_token", + ] + for field in required_fields: + assert field in fields, f"Required field {field} not found in config" + + def test_purestorage_protocol_is_optional_literal(self, backend): + """Test that protocol field accepts iscsi, fc, or nvme.""" + config_class = backend.config_type() + protocol_field = config_class.model_fields.get("protocol") + assert protocol_field is not None + + # Test config without protocol (optional) + config_no_protocol = config_class.model_validate( + { + "san-ip": "192.168.1.1", + "pure-api-token": "secret-token", + } + ) + assert config_no_protocol.protocol is None + + # Test valid config with iscsi + valid_config_iscsi = config_class.model_validate( + { + "san-ip": "192.168.1.1", + "pure-api-token": "secret-token", + "protocol": "iscsi", + } + ) + assert valid_config_iscsi.protocol == "iscsi" + + # Test valid config with fc + valid_config_fc = config_class.model_validate( + { + "san-ip": "192.168.1.1", + "pure-api-token": "secret-token", + "protocol": "fc", + } + ) + assert valid_config_fc.protocol == "fc" + + # Test valid config with nvme + valid_config_nvme = config_class.model_validate( + { + "san-ip": "192.168.1.1", + "pure-api-token": "secret-token", + "protocol": "nvme", + } + ) + assert valid_config_nvme.protocol == "nvme" + + def test_purestorage_api_token_is_secret(self, backend): + """Test that API token is properly marked as secret.""" + from sunbeam.storage.models import SecretDictField + + config_class = backend.config_type() + + # Check pure_api_token is marked as secret + token_field = config_class.model_fields.get("pure_api_token") + assert token_field is not None + has_secret_marker = any( + isinstance(m, SecretDictField) for m in token_field.metadata + ) + assert has_secret_marker, "pure_api_token should be marked as secret" + + def test_purestorage_config_optional_fields_work(self, backend): + """Test that optional fields can be omitted.""" + config_class = backend.config_type() + + # Create config with only required fields + config = config_class.model_validate( + { + "san-ip": "192.168.1.1", + "pure-api-token": "secret-token", + } + ) + + # Verify optional fields default to None + assert config.protocol is None + assert config.pure_iscsi_cidr is None + assert config.pure_nvme_cidr is None + assert config.pure_host_personality is None + assert config.pure_eradicate_on_delete is None + + def test_purestorage_personality_enum(self, backend): + """Test that host personality accepts valid enum values.""" + config_class = backend.config_type() + + # Test with valid personality + config = config_class.model_validate( + { + "san-ip": "192.168.1.1", + "pure-api-token": "secret-token", + "pure-host-personality": "esxi", + } + ) + assert config.pure_host_personality == "esxi" + + def test_purestorage_replication_fields_exist(self, backend): + """Test that replication-related fields exist.""" + config_class = backend.config_type() + fields = config_class.model_fields + + replication_fields = [ + "pure_replica_interval_default", + "pure_replica_retention_short_term_default", + "pure_replication_pg_name", + "pure_replication_pod_name", + "pure_trisync_enabled", + ] + for field in replication_fields: + assert field in fields, f"Replication field {field} not found" + + def test_purestorage_iscsi_fields_exist(self, backend): + """Test that iSCSI-related fields exist.""" + config_class = backend.config_type() + fields = config_class.model_fields + + iscsi_fields = [ + "pure_iscsi_cidr", + "pure_iscsi_cidr_list", + ] + for field in iscsi_fields: + assert field in fields, f"iSCSI field {field} not found" + + def test_purestorage_nvme_fields_exist(self, backend): + """Test that NVMe-related fields exist.""" + config_class = backend.config_type() + fields = config_class.model_fields + + nvme_fields = [ + "pure_nvme_cidr", + "pure_nvme_cidr_list", + "pure_nvme_transport", + ] + for field in nvme_fields: + assert field in fields, f"NVMe field {field} not found" + + +class TestPureStorageConfigValidation: + """Test Pure Storage config validation behavior.""" + + def test_protocol_accepts_only_valid_values(self, purestorage_backend): + """Test that protocol field rejects invalid values.""" + from pydantic import ValidationError + + config_class = purestorage_backend.config_type() + + # Should reject invalid protocol + with pytest.raises(ValidationError) as exc_info: + config_class.model_validate( + { + "san-ip": "192.168.1.1", + "pure-api-token": "secret-token", + "protocol": "INVALID", + } + ) + + assert "protocol" in str(exc_info.value).lower() + + def test_nvme_transport_accepts_only_tcp(self, purestorage_backend): + """Test that NVMe transport only accepts tcp.""" + from pydantic import ValidationError + + config_class = purestorage_backend.config_type() + + # Valid transport + config = config_class.model_validate( + { + "san-ip": "192.168.1.1", + "pure-api-token": "secret-token", + "pure-nvme-transport": "tcp", + } + ) + assert config.pure_nvme_transport == "tcp" + + # Should reject invalid transport + with pytest.raises(ValidationError) as exc_info: + config_class.model_validate( + { + "san-ip": "192.168.1.1", + "pure-api-token": "secret-token", + "pure-nvme-transport": "roce", # Not supported yet + } + ) + + assert ( + "pure_nvme_transport" in str(exc_info.value).lower() + or "nvme" in str(exc_info.value).lower() + ) + + def test_boolean_fields_accept_boolean_values(self, purestorage_backend): + """Test that boolean fields accept boolean values.""" + config_class = purestorage_backend.config_type() + + config = config_class.model_validate( + { + "san-ip": "192.168.1.1", + "pure-api-token": "secret-token", + "pure-automatic-max-oversubscription-ratio": True, + "pure-eradicate-on-delete": False, + "pure-trisync-enabled": True, + } + ) + assert config.pure_automatic_max_oversubscription_ratio is True + assert config.pure_eradicate_on_delete is False + assert config.pure_trisync_enabled is True + + def test_integer_fields_accept_integer_values(self, purestorage_backend): + """Test that integer fields accept integer values.""" + config_class = purestorage_backend.config_type() + + config = config_class.model_validate( + { + "san-ip": "192.168.1.1", + "pure-api-token": "secret-token", + "pure-replica-interval-default": 3600, + "pure-replica-retention-short-term-default": 86400, + "pure-replica-retention-long-term-per-day-default": 3, + "pure-replica-retention-long-term-default": 7, + } + ) + assert config.pure_replica_interval_default == 3600 + assert config.pure_replica_retention_short_term_default == 86400 + assert config.pure_replica_retention_long_term_per_day_default == 3 + assert config.pure_replica_retention_long_term_default == 7 diff --git a/sunbeam-python/tests/unit/sunbeam/storage/conftest.py b/sunbeam-python/tests/unit/sunbeam/storage/conftest.py new file mode 100644 index 000000000..9ecad71a1 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/storage/conftest.py @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Common fixtures for storage backend tests.""" + +from pathlib import Path +from typing import Annotated +from unittest.mock import MagicMock + +import pytest +from pydantic import Field + +from sunbeam.core.manifest import StorageBackendConfig +from sunbeam.storage.base import StorageBackendBase +from sunbeam.storage.models import SecretDictField + + +class MockStorageConfig(StorageBackendConfig): + """Mock configuration for testing.""" + + required_field: Annotated[str, Field(description="A required field")] + optional_field: Annotated[str | None, Field(description="An optional field")] = None + secret_field: Annotated[ + str, + Field(description="A secret field"), + SecretDictField(field="secret-key"), + ] + int_field: Annotated[int | None, Field(description="An integer field")] = None + + +class MockStorageBackend(StorageBackendBase[MockStorageConfig]): + """Mock storage backend for testing.""" + + backend_type = "mock" + display_name = "Mock Storage Backend" + + @property + def charm_name(self) -> str: + return "mock-charm" + + @property + def charm_channel(self) -> str: + return "latest/stable" + + def config_type(self) -> type[MockStorageConfig]: + return MockStorageConfig + + +@pytest.fixture +def mock_backend(): + """Create a mock backend instance for testing.""" + return MockStorageBackend() + + +@pytest.fixture +def mock_deployment(tmp_path: Path): + """Create a mock deployment object.""" + deployment = MagicMock() + deployment.name = "test_deployment" + deployment.plans_directory = tmp_path / "plans" + deployment.plans_directory.mkdir(parents=True) + deployment.openstack_machines_model = "openstack" + deployment.juju_controller = "test-controller" + + # Mock get_space method + deployment.get_space.return_value = "test-space" + + # Mock get_client + mock_client = MagicMock() + deployment.get_client.return_value = mock_client + + # Mock get_tfhelper + mock_tfhelper = MagicMock() + deployment.get_tfhelper.return_value = mock_tfhelper + + # Mock proxy settings + deployment.get_proxy_settings.return_value = {} + + # Mock _get_juju_clusterd_env + deployment._get_juju_clusterd_env.return_value = {} + + # Mock get_clusterd_http_address + deployment.get_clusterd_http_address.return_value = "http://localhost:7000" + + # Mock _tfhelpers + deployment._tfhelpers = {} + + return deployment + + +@pytest.fixture +def mock_jhelper(): + """Create a mock JujuHelper.""" + jhelper = MagicMock() + jhelper.get_model_name_with_owner.return_value = "admin/openstack" + + # Mock model status + mock_status = MagicMock() + mock_status.apps = {} + jhelper.get_model_status.return_value = mock_status + + # Mock get_model + jhelper.get_model.return_value = {"model-uuid": "test-uuid"} + + return jhelper + + +@pytest.fixture +def mock_manifest(): + """Create a mock manifest.""" + manifest = MagicMock() + manifest.storage.root = {} + return manifest + + +@pytest.fixture +def mock_console(): + """Create a mock console.""" + return MagicMock() + + +@pytest.fixture +def terraform_plan_dir(tmp_path: Path): + """Create a temporary terraform plan directory.""" + plan_dir = tmp_path / "etc" / "deploy-storage" + plan_dir.mkdir(parents=True) + + # Create some dummy terraform files + (plan_dir / "main.tf").write_text("# Terraform config") + (plan_dir / "variables.tf").write_text("# Variables") + + return plan_dir + + +@pytest.fixture +def mock_click_context(mock_deployment, mock_manifest): + """Create a mock Click context.""" + ctx = MagicMock() + ctx.obj = mock_deployment + mock_deployment.get_manifest.return_value = mock_manifest + return ctx diff --git a/sunbeam-python/tests/unit/sunbeam/storage/test_base.py b/sunbeam-python/tests/unit/sunbeam/storage/test_base.py new file mode 100644 index 000000000..7a278ce27 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/storage/test_base.py @@ -0,0 +1,507 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for StorageBackendBase class. + +These tests are designed to be generic and can be reused by child classes +by overriding the backend fixture. +""" + +from unittest.mock import Mock, patch + +import click +import pytest +from packaging.version import Version + +from sunbeam.core.manifest import StorageBackendConfig +from sunbeam.storage.base import ( + FQDN_PATTERN, + JUJU_APP_NAME_PATTERN, + validate_juju_application_name, +) +from sunbeam.storage.models import ( + BackendAlreadyExistsException, +) + + +class TestJujuApplicationNameValidation: + """Test Juju application name validation logic.""" + + def test_valid_names(self): + """Test valid Juju application names.""" + valid_names = [ + "myapp", + "my-app", + "my-app-backend", + "a", + "a1", + "app123", + "my-storage", + ] + for name in valid_names: + assert validate_juju_application_name(name), f"{name} should be valid" + + def test_invalid_names(self): + """Test invalid Juju application names.""" + invalid_names = [ + "", # Empty + "MyApp", # Uppercase + "my_app", # Underscore + "123app", # Starts with number + "-myapp", # Starts with hyphen + "myapp-", # Ends with hyphen + "my--app", # Consecutive hyphens + "my-app-1", # Number after final hyphen + ] + for name in invalid_names: + assert not validate_juju_application_name(name), f"{name} should be invalid" + + def test_pattern_matching(self): + """Test the regex pattern directly.""" + assert JUJU_APP_NAME_PATTERN.match("myapp") + assert not JUJU_APP_NAME_PATTERN.match("MyApp") + assert not JUJU_APP_NAME_PATTERN.match("123app") + + +class TestFQDNPattern: + """Test FQDN pattern validation.""" + + def test_valid_fqdns(self): + """Test valid FQDNs.""" + import re + + valid_fqdns = [ + "example.com", + "sub.example.com", + "my-server.example.com", + "server1.example.com", + "a.b.c.d.example.com", + ] + pattern = re.compile(FQDN_PATTERN) + for fqdn in valid_fqdns: + assert pattern.match(fqdn), f"{fqdn} should be valid" + + def test_invalid_fqdns(self): + """Test invalid FQDNs.""" + import re + + invalid_fqdns = [ + "", + "-example.com", + "example-.com", + "exa mple.com", + "example..com", + ] + pattern = re.compile(FQDN_PATTERN) + for fqdn in invalid_fqdns: + assert not pattern.match(fqdn), f"{fqdn} should be invalid" + + +class BaseStorageBackendTests: + """Base test class for storage backends. + + Subclasses can inherit from this to get comprehensive testing + for their backend implementations. + """ + + @pytest.fixture + def backend(self, mock_backend): + """Override this fixture to test a specific backend.""" + return mock_backend + + def test_backend_type_is_set(self, backend): + """Test that backend_type is set.""" + assert backend.backend_type + assert isinstance(backend.backend_type, str) + assert backend.backend_type != "base" + + def test_display_name_is_set(self, backend): + """Test that display_name is set.""" + assert backend.display_name + assert isinstance(backend.display_name, str) + + def test_version_is_set(self, backend): + """Test that version is set.""" + assert backend.version + assert isinstance(backend.version, Version) + + def test_charm_name_is_set(self, backend): + """Test that charm_name is set.""" + assert backend.charm_name + assert isinstance(backend.charm_name, str) + + def test_charm_channel_is_set(self, backend): + """Test that charm_channel is set.""" + assert backend.charm_channel + assert isinstance(backend.charm_channel, str) + + def test_charm_base_is_set(self, backend): + """Test that charm_base is set.""" + assert backend.charm_base + assert isinstance(backend.charm_base, str) + + def test_principal_application_is_set(self, backend): + """Test that principal_application is set.""" + assert backend.principal_application + assert isinstance(backend.principal_application, str) + + def test_tfplan_properties(self, backend): + """Test Terraform plan properties.""" + assert backend.tfplan + assert backend.tfplan_dir + assert isinstance(backend.tfplan, str) + assert isinstance(backend.tfplan_dir, str) + + def test_tfvar_config_key(self, backend): + """Test tfvar config key.""" + assert backend.tfvar_config_key == "TerraformVarsStorageBackends" + + def test_config_key(self, backend): + """Test config key generation.""" + name = "test-backend" + key = backend.config_key(name) + assert key == f"Storage-{name}" + + def test_config_type_returns_pydantic_model(self, backend): + """Test that config_type returns a Pydantic model class.""" + config_class = backend.config_type() + assert issubclass(config_class, StorageBackendConfig) + + def test_get_endpoint_bindings(self, backend, mock_deployment): + """Test endpoint bindings generation.""" + bindings = backend.get_endpoint_bindings(mock_deployment) + assert isinstance(bindings, list) + assert len(bindings) >= 1 + for binding in bindings: + assert isinstance(binding, dict) + + def test_validate_ip_or_fqdn_with_valid_ip(self, backend): + """Test IP validation with valid IPs.""" + valid_ips = ["192.168.1.1", "10.0.0.1", "2001:db8::1"] + for ip in valid_ips: + assert backend._validate_ip_or_fqdn(ip) == ip + + def test_validate_ip_or_fqdn_with_valid_fqdn(self, backend): + """Test FQDN validation with valid FQDNs.""" + valid_fqdns = ["example.com", "server.example.com", "my-server.local"] + for fqdn in valid_fqdns: + assert backend._validate_ip_or_fqdn(fqdn) == fqdn + + def test_validate_ip_or_fqdn_with_invalid_value(self, backend): + """Test IP/FQDN validation with invalid values.""" + invalid_values = ["not an ip", "example..com", "", "-invalid.com"] + for value in invalid_values: + with pytest.raises(click.BadParameter): + backend._validate_ip_or_fqdn(value) + + +class TestStorageBackendBase(BaseStorageBackendTests): + """Tests for the base StorageBackendBase class using mock backend.""" + + def test_register_terraform_plan(self, backend, mock_deployment, tmp_path): + """Test Terraform plan registration raises error when plan not found.""" + # Mock the deployment's plan directory + mock_deployment.plans_directory = tmp_path / "plans" + mock_deployment.plans_directory.mkdir(parents=True, exist_ok=True) + + # Without a valid plan source, should raise FileNotFoundError + with pytest.raises(FileNotFoundError): + backend.register_terraform_plan(mock_deployment) + + def test_add_backend_instance_success( + self, backend, mock_deployment, mock_console, tmp_path + ): + """Test adding a backend instance successfully.""" + # Setup + backend_name = "test-backend" + config = {"required_field": "value", "secret_field": "secret"} + + # Mock the manifest property + mock_manifest = Mock() + mock_manifest.storage.root = {} + + # Mock the service and JujuHelper + with patch("sunbeam.storage.base.StorageBackendService") as mock_service_class: + with patch("sunbeam.storage.base.JujuHelper") as mock_jhelper_class: + # Patch the manifest property without accessing it + with patch.object( + type(backend), + "manifest", + new_callable=lambda: property(lambda self: mock_manifest), + ): + mock_service = Mock() + mock_service.backend_exists.return_value = False + mock_service_class.return_value = mock_service + + mock_jhelper = Mock() + mock_jhelper_class.return_value = mock_jhelper + + # Mock register_terraform_plan + with patch.object(backend, "register_terraform_plan"): + # Mock run_plan + with patch("sunbeam.storage.base.run_plan"): + backend.add_backend_instance( + mock_deployment, backend_name, config, mock_console + ) + + def test_add_backend_instance_invalid_name( + self, backend, mock_deployment, mock_console + ): + """Test adding a backend with invalid name.""" + invalid_names = ["MyApp", "app_name", "123app", "app-"] + + for invalid_name in invalid_names: + with pytest.raises(click.ClickException) as exc_info: + backend.add_backend_instance( + mock_deployment, invalid_name, {}, mock_console + ) + assert "Invalid backend name" in str(exc_info.value) + + def test_add_backend_instance_already_exists( + self, backend, mock_deployment, mock_console + ): + """Test adding a backend that already exists.""" + backend_name = "existing-backend" + config = {} + + with patch("sunbeam.storage.base.StorageBackendService") as mock_service_class: + with patch("sunbeam.storage.base.JujuHelper") as mock_jhelper_class: + mock_service = Mock() + mock_service.backend_exists.return_value = True + mock_service_class.return_value = mock_service + + mock_jhelper = Mock() + mock_jhelper_class.return_value = mock_jhelper + + with pytest.raises(BackendAlreadyExistsException): + backend.add_backend_instance( + mock_deployment, backend_name, config, mock_console + ) + + def test_remove_backend(self, backend, mock_deployment, mock_console, tmp_path): + """Test removing a backend.""" + backend_name = "test-backend" + + # Mock the manifest property + mock_manifest = Mock() + mock_manifest.storage.root = {} + + # Mock the plan directory + mock_deployment.plans_directory = tmp_path / "plans" + mock_deployment.plans_directory.mkdir(parents=True, exist_ok=True) + + with patch.object( + type(backend), + "manifest", + new_callable=lambda: property(lambda self: mock_manifest), + ): + with patch("sunbeam.storage.base.JujuHelper") as mock_jhelper_class: + mock_jhelper = Mock() + mock_jhelper_class.return_value = mock_jhelper + + with patch.object(backend, "register_terraform_plan"): + with patch("sunbeam.storage.base.run_plan"): + backend.remove_backend( + mock_deployment, backend_name, mock_console + ) + + def test_build_terraform_vars(self, backend, mock_deployment, mock_manifest): + """Test Terraform variables generation.""" + backend_name = "test-backend" + config = backend.config_type().model_validate( + { + "required-field": "test", + "secret-field": "secret123", + } + ) + + tfvars = backend.build_terraform_vars( + mock_deployment, mock_manifest, backend_name, config + ) + + assert "principal_application" in tfvars + assert tfvars["principal_application"] == backend.principal_application + assert "charm_name" in tfvars + assert tfvars["charm_name"] == backend.charm_name + assert "charm_channel" in tfvars + assert "charm_base" in tfvars + assert "endpoint_bindings" in tfvars + assert "charm_config" in tfvars + assert "secrets" in tfvars + + def test_display_config_options(self, backend, mock_console): + """Test display of configuration options.""" + with patch("sunbeam.storage.base.console", mock_console): + backend.display_config_options() + # Verify console.print was called + assert mock_console.print.called + + def test_display_config_table(self, backend, mock_console): + """Test display of configuration table.""" + backend_name = "test-backend" + config = backend.config_type().model_validate( + { + "required-field": "test_value", + "secret-field": "secret123", + "optional-field": "optional_value", + } + ) + + with patch("sunbeam.storage.base.console", mock_console): + backend.display_config_table(backend_name, config) + # Verify console.print was called + assert mock_console.print.called + + def test_display_config_table_with_empty_config(self, backend, mock_console): + """Test display of empty configuration.""" + backend_name = "test-backend" + config = None + + with patch("sunbeam.storage.base.console", mock_console): + backend.display_config_table(backend_name, config) + # Should still print something + assert mock_console.print.called + + def test_format_config_value_non_secret(self, backend): + """Test formatting of non-secret configuration values.""" + value = "test_value" + formatted = backend._format_config_value(value, is_secret=False) + assert formatted == "test_value" + + def test_format_config_value_secret(self, backend): + """Test formatting of secret configuration values.""" + value = "secret123" + formatted = backend._format_config_value(value, is_secret=True) + assert formatted == "********" + + def test_format_config_value_long(self, backend): + """Test formatting of long configuration values.""" + value = "a" * 30 + formatted = backend._format_config_value(value, is_secret=False) + assert formatted.endswith("...") + assert len(formatted) == 23 + + def test_field_is_secret(self, backend): + """Test detection of secret fields.""" + from sunbeam.storage.models import SecretDictField + + config_class = backend.config_type() + for field_name, field_info in config_class.model_fields.items(): + is_secret = backend._field_is_secret(field_info) + if field_name == "secret_field": + assert is_secret + else: + # Check if it's actually marked as secret + has_secret_metadata = any( + isinstance(m, SecretDictField) for m in field_info.metadata + ) + assert is_secret == has_secret_metadata + + def test_get_field_descriptions(self, backend): + """Test extraction of field descriptions.""" + config_class = backend.config_type() + descriptions = backend._get_field_descriptions(config_class) + + assert isinstance(descriptions, dict) + for field_name in config_class.model_fields.keys(): + assert field_name in descriptions + assert isinstance(descriptions[field_name], str) + + def test_extract_field_info(self, backend): + """Test extraction of field information.""" + config_class = backend.config_type() + for field_name, field_info in config_class.model_fields.items(): + field_type, description = backend._extract_field_info(field_info) + assert isinstance(field_type, str) + assert isinstance(description, str) + + def test_create_deploy_step(self, backend, mock_deployment, mock_jhelper): + """Test creation of deploy step.""" + from sunbeam.storage.steps import BaseStorageBackendDeployStep + + mock_client = Mock() + mock_tfhelper = Mock() + mock_manifest = Mock() + preseed = {} + backend_name = "test-backend" + model = "openstack" + + step = backend.create_deploy_step( + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + preseed, + backend_name, + model, + ) + + assert isinstance(step, BaseStorageBackendDeployStep) + + def test_create_destroy_step(self, backend, mock_deployment, mock_jhelper): + """Test creation of destroy step.""" + from sunbeam.storage.steps import BaseStorageBackendDestroyStep + + mock_client = Mock() + mock_tfhelper = Mock() + mock_manifest = Mock() + backend_name = "test-backend" + model = "openstack" + + step = backend.create_destroy_step( + mock_deployment, + mock_client, + mock_tfhelper, + mock_jhelper, + mock_manifest, + backend_name, + model, + ) + + assert isinstance(step, BaseStorageBackendDestroyStep) + + def test_register_add_cli(self, backend): + """Test CLI registration.""" + mock_add_group = Mock(spec=click.Group) + + with patch.object(backend, "_get_cli_class") as mock_get_cli_class: + mock_cli_class = Mock() + mock_cli_instance = Mock() + mock_cli_class.return_value = mock_cli_instance + mock_get_cli_class.return_value = mock_cli_class + + backend.register_add_cli(mock_add_group) + + mock_cli_class.assert_called_once_with(backend) + mock_cli_instance.register_add_cli.assert_called_once_with(mock_add_group) + + def test_get_cli_class_default(self, backend): + """Test getting CLI class with default implementation.""" + from sunbeam.storage.cli_base import StorageBackendCLIBase + + # For mock backend, it should return the base CLI class + cli_class = backend._get_cli_class() + assert cli_class == StorageBackendCLIBase + + def test_manifest_property(self, backend, mock_click_context): + """Test manifest property.""" + with patch("click.get_current_context", return_value=mock_click_context): + manifest = backend.manifest + assert manifest is not None + + def test_manifest_property_caching(self, backend, mock_click_context): + """Test that manifest is cached after first access.""" + with patch("click.get_current_context", return_value=mock_click_context): + manifest1 = backend.manifest + manifest2 = backend.manifest + # Should be the same object (cached) + assert manifest1 is manifest2 + + def test_manifest_property_failure(self, backend, mock_click_context): + """Test manifest property when loading fails.""" + mock_click_context.obj.get_manifest.return_value = None + + with patch("click.get_current_context", return_value=mock_click_context): + with pytest.raises(ValueError, match="Failed to load manifest"): + _ = backend.manifest diff --git a/sunbeam-python/tests/unit/sunbeam/storage/test_manager.py b/sunbeam-python/tests/unit/sunbeam/storage/test_manager.py new file mode 100644 index 000000000..3888f651c --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/storage/test_manager.py @@ -0,0 +1,377 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for StorageBackendManager class.""" + +from unittest.mock import Mock, patch + +import click +import pytest + +from sunbeam.storage.manager import StorageBackendManager +from sunbeam.storage.models import StorageBackendInfo + + +@pytest.fixture +def manager(): + """Create a fresh manager instance.""" + # Reset the class-level state before each test + StorageBackendManager._backends = {} + StorageBackendManager._loaded = False + return StorageBackendManager() + + +@pytest.fixture +def mock_backend(): + """Create a mock backend.""" + from sunbeam.core.common import RiskLevel + + backend = Mock() + backend.backend_type = "test-backend" + backend.display_name = "Test Backend" + backend.register_add_cli = Mock() + backend.register_options_cli = Mock() + backend.risk_availability = RiskLevel.STABLE + return backend + + +@pytest.fixture +def mock_deployment(): + """Create a mock deployment.""" + deployment = Mock() + deployment.openstack_machines_model = "openstack" + deployment.juju_controller = "test-controller" + return deployment + + +class TestStorageBackendManager: + """Tests for StorageBackendManager.""" + + def test_init_loads_backends(self, manager): + """Test that initialization triggers backend loading.""" + with patch.object(manager, "_load_backends") as mock_load: + # Create a new manager + new_manager = StorageBackendManager() + # _load_backends should be called if backends are empty + if not new_manager._backends: + mock_load.assert_called_once() + + def test_load_backends_sets_loaded_flag(self, manager): + """Test that _load_backends sets the loaded flag.""" + # Reset the loaded flag + manager._loaded = False + assert not manager._loaded + with patch("importlib.import_module"): + with patch("pathlib.Path.iterdir", return_value=[]): + manager._load_backends() + assert manager._loaded + + def test_load_backends_only_once(self, manager): + """Test that backends are only loaded once.""" + with patch("importlib.import_module"): + with patch("pathlib.Path.iterdir", return_value=[]): + manager._load_backends() + manager._load_backends() + # Should only load once due to _loaded flag + # But init also calls it, so check the total is reasonable + + def test_load_backends_skips_non_directories(self, manager): + """Test that non-directory items are skipped.""" + mock_file = Mock() + mock_file.is_dir.return_value = False + mock_file.name = "test.py" + + with patch("pathlib.Path.iterdir", return_value=[mock_file]): + with patch("importlib.import_module"): + manager._load_backends() + # Should not attempt to import non-directories + + def test_load_backends_skips_special_directories(self, manager): + """Test that special directories are skipped.""" + special_dirs = ["__pycache__", "_internal", "etc"] + mock_paths = [] + + for name in special_dirs: + mock_path = Mock() + mock_path.is_dir.return_value = True + mock_path.name = name + mock_paths.append(mock_path) + + with patch("pathlib.Path.iterdir", return_value=mock_paths): + with patch("importlib.import_module") as mock_import: + manager._load_backends() + # Should not attempt to import special directories + mock_import.assert_not_called() + + def test_load_backends_skips_missing_backend_file(self, manager): + """Test that directories without backend.py are skipped.""" + mock_dir = Mock() + mock_dir.is_dir.return_value = True + mock_dir.name = "test-backend" + mock_dir.__truediv__ = Mock(return_value=Mock(exists=Mock(return_value=False))) + + with patch("pathlib.Path.iterdir", return_value=[mock_dir]): + with patch("importlib.import_module") as mock_import: + manager._load_backends() + # Should not attempt to import if backend.py is missing + mock_import.assert_not_called() + + def test_load_backends_registers_valid_backend(self, manager): + """Test that valid backends are registered.""" + from sunbeam.storage.base import StorageBackendBase + + # Create a mock backend class + class TestBackend(StorageBackendBase): + backend_type = "test" + display_name = "Test Backend" + + @property + def charm_name(self): + return "test-charm" + + def config_type(self): + from sunbeam.core.manifest import StorageBackendConfig + + return StorageBackendConfig + + # Create mock module with backend class + mock_module = Mock() + mock_module.TestBackend = TestBackend + + mock_dir = Mock() + mock_dir.is_dir.return_value = True + mock_dir.name = "test" + backend_py = Mock() + backend_py.exists.return_value = True + mock_dir.__truediv__ = Mock(return_value=backend_py) + + with patch("pathlib.Path.iterdir", return_value=[mock_dir]): + with patch("importlib.import_module", return_value=mock_module): + with patch("pathlib.Path.exists", return_value=True): + # Clear existing backends and reset loaded flag + manager._backends = {} + manager._loaded = False + manager._load_backends() + # Backend should be registered + assert "test" in manager._backends + + def test_load_backends_handles_import_error(self, manager): + """Test that import errors are handled gracefully.""" + mock_dir = Mock() + mock_dir.is_dir.return_value = True + mock_dir.name = "broken-backend" + backend_py = Mock() + backend_py.exists.return_value = True + mock_dir.__truediv__ = Mock(return_value=backend_py) + + with patch("pathlib.Path.iterdir", return_value=[mock_dir]): + with patch( + "importlib.import_module", side_effect=ImportError("Test error") + ): + # Should not raise, just log warning + manager._load_backends() + + def test_get_backend_success(self, manager, mock_backend): + """Test getting a backend by name.""" + manager._backends["test-backend"] = mock_backend + manager._loaded = True + + backend = manager.get_backend("test-backend") + assert backend == mock_backend + + def test_get_backend_not_found(self, manager): + """Test getting a non-existent backend.""" + manager._loaded = True + + with pytest.raises(ValueError, match="Storage backend .* not found"): + manager.get_backend("nonexistent") + + def test_backends_property(self, manager, mock_backend): + """Test backends property returns all backends.""" + manager._backends["test-backend"] = mock_backend + manager._loaded = True + + backends = manager.backends() + assert "test-backend" in backends + assert backends["test-backend"] == mock_backend + + def test_get_all_storage_manifests(self, manager, mock_backend): + """Test getting all storage manifests.""" + manager._backends["test-backend"] = mock_backend + manager._loaded = True + + manifests = manager.get_all_storage_manifests() + assert isinstance(manifests, dict) + assert "test-backend" in manifests + assert manifests["test-backend"] == {} + + def test_register(self, manager, mock_deployment): + """Test registering storage commands.""" + mock_cli_group = Mock(spec=click.Group) + + with patch.object(manager, "register_cli_commands"): + manager.register(mock_cli_group, mock_deployment) + mock_cli_group.add_command.assert_called_once() + + def test_register_handles_errors(self, manager, mock_deployment): + """Test that register handles errors appropriately.""" + mock_cli_group = Mock(spec=click.Group) + + with patch.object( + manager, "register_cli_commands", side_effect=ValueError("Test error") + ): + with pytest.raises(ValueError): + manager.register(mock_cli_group, mock_deployment) + + def test_register_cli_commands(self, manager, mock_backend, mock_deployment): + """Test CLI command registration.""" + from sunbeam.core.common import RiskLevel + + manager._backends["test-backend"] = mock_backend + manager._loaded = True + + mock_storage_group = Mock(spec=click.Group) + + # Mock infer_risk to return a default risk level + with patch("sunbeam.storage.manager.infer_risk", return_value=RiskLevel.STABLE): + manager.register_cli_commands(mock_storage_group, mock_deployment) + + # Verify that backend's register_add_cli was called + mock_backend.register_add_cli.assert_called_once() + + # Verify that commands were added to the group + assert mock_storage_group.add_command.called + + def test_register_cli_commands_handles_backend_errors( + self, manager, mock_backend, mock_deployment + ): + """Test that CLI registration handles backend errors.""" + from sunbeam.core.common import RiskLevel + + manager._backends["test-backend"] = mock_backend + manager._loaded = True + mock_backend.register_add_cli.side_effect = ValueError("Backend error") + + mock_storage_group = Mock(spec=click.Group) + + # Mock infer_risk to return a default risk level + with patch("sunbeam.storage.manager.infer_risk", return_value=RiskLevel.STABLE): + with pytest.raises(ValueError): + manager.register_cli_commands(mock_storage_group, mock_deployment) + + def test_display_backends_table_empty(self, manager): + """Test displaying empty backend list.""" + from rich.console import Console + + mock_console = Mock(spec=Console) + + with patch("sunbeam.storage.manager.console", mock_console): + manager._display_backends_table([]) + # Should print a message about no backends + mock_console.print.assert_called_once() + + def test_display_backends_table_with_backends(self, manager): + """Test displaying backend list.""" + from rich.console import Console + + backends = [ + StorageBackendInfo( + name="backend1", + backend_type="type1", + status="active", + charm="charm1", + config={}, + ), + StorageBackendInfo( + name="backend2", + backend_type="type2", + status="error", + charm="charm2", + config={}, + ), + ] + + mock_console = Mock(spec=Console) + + with patch("sunbeam.storage.manager.console", mock_console): + manager._display_backends_table(backends) + # Should print the table + assert mock_console.print.called + + +class TestStorageCLICommands: + """Test the CLI command functions created by the manager.""" + + def test_list_all_command(self, manager, mock_deployment): + """Test the list command.""" + from sunbeam.core.common import RiskLevel + from sunbeam.storage.service import StorageBackendService + + mock_service = Mock(spec=StorageBackendService) + mock_service.list_backends.return_value = [] + + with patch( + "sunbeam.storage.manager.StorageBackendService", return_value=mock_service + ): + with patch.object(manager, "_display_backends_table"): + with patch( + "sunbeam.storage.manager.infer_risk", return_value=RiskLevel.STABLE + ): + manager._loaded = True + mock_storage_group = Mock(spec=click.Group) + manager.register_cli_commands(mock_storage_group, mock_deployment) + + # Get the list command that was registered + calls = mock_storage_group.add_command.call_args_list + list_command = None + for call in calls: + if call[0][0].name == "list": + list_command = call[0][0] + break + + assert list_command is not None + + def test_remove_backend_command_success(self, manager, mock_deployment): + """Test the remove command with successful removal.""" + from sunbeam.core.common import RiskLevel + + manager._backends["test-backend"] = Mock() + manager._loaded = True + + mock_storage_group = Mock(spec=click.Group) + + # Mock infer_risk to return a default risk level + with patch("sunbeam.storage.manager.infer_risk", return_value=RiskLevel.STABLE): + manager.register_cli_commands(mock_storage_group, mock_deployment) + + # Verify remove command was registered + calls = mock_storage_group.add_command.call_args_list + remove_command = None + for call in calls: + if hasattr(call[0][0], "name") and call[0][0].name == "remove": + remove_command = call[0][0] + break + + assert remove_command is not None + + def test_show_backend_command(self, manager, mock_deployment): + """Test the show command.""" + from sunbeam.core.common import RiskLevel + + manager._loaded = True + + mock_storage_group = Mock(spec=click.Group) + + # Mock infer_risk to return a default risk level + with patch("sunbeam.storage.manager.infer_risk", return_value=RiskLevel.STABLE): + manager.register_cli_commands(mock_storage_group, mock_deployment) + + # Verify show command was registered + calls = mock_storage_group.add_command.call_args_list + show_command = None + for call in calls: + if hasattr(call[0][0], "name") and call[0][0].name == "show": + show_command = call[0][0] + break + + assert show_command is not None diff --git a/sunbeam-python/tests/unit/sunbeam/storage/test_service.py b/sunbeam-python/tests/unit/sunbeam/storage/test_service.py new file mode 100644 index 000000000..b87d4f98c --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/storage/test_service.py @@ -0,0 +1,264 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for StorageBackendService class.""" + +from unittest.mock import Mock + +import pytest + +from sunbeam.clusterd.models import StorageBackend +from sunbeam.clusterd.service import StorageBackendNotFoundException +from sunbeam.storage.models import ( + BackendAlreadyExistsException, + BackendNotFoundException, + StorageBackendInfo, +) +from sunbeam.storage.service import StorageBackendService + + +@pytest.fixture +def mock_deployment(): + """Create a mock deployment.""" + deployment = Mock() + deployment.openstack_machines_model = "openstack" + deployment.juju_controller = "test-controller" + deployment.get_client.return_value = Mock() + return deployment + + +@pytest.fixture +def mock_jhelper(): + """Create a mock JujuHelper.""" + jhelper = Mock() + jhelper.get_model_name_with_owner.return_value = "admin/openstack" + + # Mock model status + mock_status = Mock() + mock_status.apps = {} + jhelper.get_model_status.return_value = mock_status + + return jhelper + + +@pytest.fixture +def service(mock_deployment, mock_jhelper): + """Create a StorageBackendService instance.""" + return StorageBackendService(mock_deployment, mock_jhelper) + + +class TestStorageBackendService: + """Tests for StorageBackendService.""" + + def test_init(self, service, mock_deployment, mock_jhelper): + """Test service initialization.""" + assert service.deployment == mock_deployment + assert service.jhelper == mock_jhelper + assert service.model == "admin/openstack" + assert service._tfvar_config_key == "TerraformVarsStorageBackends" + + def test_list_backends_empty(self, service, mock_deployment): + """Test listing backends when none exist.""" + mock_client = mock_deployment.get_client.return_value + mock_client.cluster.get_storage_backends.return_value.root = [] + + backends = service.list_backends() + assert backends == [] + + def test_list_backends_with_backends(self, service, mock_deployment, mock_jhelper): + """Test listing backends with some backends present.""" + # Create mock backend data + mock_backend = Mock(spec=StorageBackend) + mock_backend.name = "test-backend" + mock_backend.type = "test-type" + mock_backend.config = {"key": "value"} + + mock_client = mock_deployment.get_client.return_value + mock_client.cluster.get_storage_backends.return_value.root = [mock_backend] + + # Mock Juju status + mock_app_status = Mock() + mock_app_status.app_status.current = "active" + mock_app_status.charm = "test-charm" + + mock_status = Mock() + mock_status.apps = {"test-backend": mock_app_status} + mock_jhelper.get_model_status.return_value = mock_status + + backends = service.list_backends() + + assert len(backends) == 1 + assert isinstance(backends[0], StorageBackendInfo) + assert backends[0].name == "test-backend" + assert backends[0].backend_type == "test-type" + assert backends[0].status == "active" + assert backends[0].charm == "test-charm" + assert backends[0].config == {"key": "value"} + + def test_list_backends_handles_errors(self, service, mock_deployment): + """Test that list_backends handles errors gracefully.""" + mock_backend = Mock(spec=StorageBackend) + mock_backend.name = "broken-backend" + mock_backend.type = "test-type" + # Make config access raise an error + mock_backend.config = property(lambda self: (_ for _ in ()).throw(ValueError())) + + mock_client = mock_deployment.get_client.return_value + mock_client.cluster.get_storage_backends.return_value.root = [mock_backend] + + # Should not raise, just skip the broken backend + backends = service.list_backends() + assert len(backends) == 0 + + def test_get_application_status_active(self, service, mock_jhelper): + """Test getting application status for active app.""" + mock_app_status = Mock() + mock_app_status.app_status.current = "active" + + mock_status = Mock() + mock_status.apps = {"test-app": mock_app_status} + mock_jhelper.get_model_status.return_value = mock_status + + status = service._get_application_status(mock_jhelper, "test-app") + assert status == "active" + + def test_get_application_status_not_found(self, service, mock_jhelper): + """Test getting application status for non-existent app.""" + mock_status = Mock() + mock_status.apps = {} + mock_jhelper.get_model_status.return_value = mock_status + + status = service._get_application_status(mock_jhelper, "nonexistent") + assert status == "not-found" + + def test_get_application_status_error(self, service, mock_jhelper): + """Test getting application status when Juju errors.""" + mock_jhelper.get_model_status.side_effect = Exception("Juju error") + + status = service._get_application_status(mock_jhelper, "test-app") + assert status == "unknown" + + def test_get_application_charm_success(self, service, mock_jhelper): + """Test getting application charm successfully.""" + mock_app_status = Mock() + mock_app_status.charm = "ch:amd64/focal/test-charm-123" + + mock_status = Mock() + mock_status.apps = {"test-app": mock_app_status} + mock_jhelper.get_model_status.return_value = mock_status + + charm = service._get_application_charm(mock_jhelper, "test-app") + assert charm == "ch:amd64/focal/test-charm-123" + + def test_get_application_charm_not_found(self, service, mock_jhelper): + """Test getting charm for non-existent app.""" + mock_status = Mock() + mock_status.apps = {} + mock_jhelper.get_model_status.return_value = mock_status + + charm = service._get_application_charm(mock_jhelper, "nonexistent") + assert charm == "Not Found" + + def test_get_application_charm_error(self, service, mock_jhelper): + """Test getting charm when Juju errors.""" + mock_jhelper.get_model_status.side_effect = Exception("Juju error") + + charm = service._get_application_charm(mock_jhelper, "test-app") + assert charm == "Unknown" + + def test_backend_exists_true(self, service, mock_deployment): + """Test checking if backend exists - true case.""" + mock_backend = Mock(spec=StorageBackend) + mock_backend.type = "test-type" + + mock_client = mock_deployment.get_client.return_value + mock_client.cluster.get_storage_backend.return_value = mock_backend + + exists = service.backend_exists("test-backend", "test-type") + assert exists is True + + def test_backend_exists_false(self, service, mock_deployment): + """Test checking if backend exists - false case.""" + mock_client = mock_deployment.get_client.return_value + mock_client.cluster.get_storage_backend.side_effect = ( + StorageBackendNotFoundException() + ) + + exists = service.backend_exists("nonexistent", "test-type") + assert exists is False + + def test_backend_exists_type_mismatch(self, service, mock_deployment): + """Test checking backend exists with type mismatch.""" + mock_backend = Mock(spec=StorageBackend) + mock_backend.type = "different-type" + + mock_client = mock_deployment.get_client.return_value + mock_client.cluster.get_storage_backend.return_value = mock_backend + + with pytest.raises(BackendAlreadyExistsException): + service.backend_exists("test-backend", "expected-type") + + def test_get_backend_success(self, service, mock_deployment): + """Test getting a backend successfully.""" + mock_backend = Mock(spec=StorageBackend) + mock_backend.name = "test-backend" + mock_backend.type = "test-type" + + mock_client = mock_deployment.get_client.return_value + mock_client.cluster.get_storage_backend.return_value = mock_backend + + backend = service.get_backend("test-backend") + assert backend == mock_backend + + def test_get_backend_not_found(self, service, mock_deployment): + """Test getting a non-existent backend.""" + mock_client = mock_deployment.get_client.return_value + mock_client.cluster.get_storage_backend.side_effect = ( + StorageBackendNotFoundException() + ) + + with pytest.raises(BackendNotFoundException): + service.get_backend("nonexistent") + + def test_multiple_backends_different_types( + self, service, mock_deployment, mock_jhelper + ): + """Test listing multiple backends of different types.""" + mock_backend1 = Mock(spec=StorageBackend) + mock_backend1.name = "backend1" + mock_backend1.type = "type1" + mock_backend1.config = {} + + mock_backend2 = Mock(spec=StorageBackend) + mock_backend2.name = "backend2" + mock_backend2.type = "type2" + mock_backend2.config = {} + + mock_client = mock_deployment.get_client.return_value + mock_client.cluster.get_storage_backends.return_value.root = [ + mock_backend1, + mock_backend2, + ] + + # Mock Juju status for both apps + mock_app_status1 = Mock() + mock_app_status1.app_status.current = "active" + mock_app_status1.charm = "charm1" + + mock_app_status2 = Mock() + mock_app_status2.app_status.current = "waiting" + mock_app_status2.charm = "charm2" + + mock_status = Mock() + mock_status.apps = {"backend1": mock_app_status1, "backend2": mock_app_status2} + mock_jhelper.get_model_status.return_value = mock_status + + backends = service.list_backends() + + assert len(backends) == 2 + assert backends[0].name == "backend1" + assert backends[0].backend_type == "type1" + assert backends[0].status == "active" + assert backends[1].name == "backend2" + assert backends[1].backend_type == "type2" + assert backends[1].status == "waiting" diff --git a/sunbeam-python/tests/unit/sunbeam/storage/test_steps.py b/sunbeam-python/tests/unit/sunbeam/storage/test_steps.py new file mode 100644 index 000000000..80015583a --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/storage/test_steps.py @@ -0,0 +1,96 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +from typing import Annotated + +import pydantic +import pytest + +from sunbeam.core.questions import PasswordPromptQuestion, PromptQuestion +from sunbeam.storage.models import SecretDictField +from sunbeam.storage.steps import basemodel_validator, generate_questions_from_config + + +class SampleConfig(pydantic.BaseModel): + required_field: Annotated[ + int, + pydantic.Field(ge=1, description="A positive integer"), + ] + secret_field: Annotated[ + str, + pydantic.Field(description="A secret value"), + SecretDictField(field="secret"), + ] + optional_field: Annotated[ + int | None, + pydantic.Field(ge=0, description="Optional value"), + ] = None + + @pydantic.field_validator("secret_field") + @classmethod + def no_digits(cls, value: str) -> str: + if any(ch.isdigit() for ch in value): + raise ValueError("must not contain digits") + return value + + @pydantic.model_validator(mode="after") + def disallow_thirteen(self): + if getattr(self, "required_field", None) == 13: + raise ValueError("thirteen is not allowed") + return self + + +class TestBasemodelValidator: + def test_valid_and_invalid_values(self): + field_validator = basemodel_validator(SampleConfig) + + # Valid value should pass without raising + field_validator("required_field")(10) + + # Root validator error should be surfaced as ValueError + with pytest.raises(ValueError, match="thirteen is not allowed"): + field_validator("required_field")(13) + + # Field-level validation should be applied + with pytest.raises(ValueError, match="must not contain digits"): + field_validator("secret_field")("password1") + + # Type enforcement should be handled by pydantic + with pytest.raises(ValueError): + field_validator("required_field")("not-an-int") + + def test_unknown_field_raises_value_error(self): + field_validator = basemodel_validator(SampleConfig) + with pytest.raises(ValueError, match="has no field named"): + field_validator("missing") + + +class TestGenerateQuestionsFromConfig: + def test_required_questions_include_validation(self): + questions = generate_questions_from_config(SampleConfig) + + assert set(questions.keys()) == {"required_field", "secret_field"} + assert all( + isinstance(question, (PromptQuestion, PasswordPromptQuestion)) + for question in questions.values() + ) + + secret_question = questions["secret_field"] + assert isinstance(secret_question, PasswordPromptQuestion) + with pytest.raises(ValueError, match="must not contain digits"): + secret_question.validation_function("password1") # type: ignore[arg-type] + + required_question = questions["required_field"] + assert required_question.validation_function is not None + with pytest.raises(ValueError): + required_question.validation_function("bad") # type: ignore[arg-type] + + def test_optional_questions_include_validation(self): + questions = generate_questions_from_config(SampleConfig, optional=True) + + assert set(questions.keys()) == {"optional_field"} + optional_question = questions["optional_field"] + assert optional_question.validation_function is not None + optional_question.validation_function(5) # type: ignore[arg-type] + with pytest.raises(ValueError): + optional_question.validation_function(-1) # type: ignore[arg-type]