diff --git a/.gitignore b/.gitignore index 907b4f7ca..04bac2d09 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ *.dll *.so *.dylib +*.tgz + build/** # Test binary, built with `go test -c` diff --git a/Makefile b/Makefile index 3d90f6161..a27df5b39 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ GOBIN=$(shell go env GOBIN) endif .PHONY: all -all: crds deepcopy lint test +all: crds deepcopy lint-fix format lint test .PHONY: help help: ## Display this help. diff --git a/api/v1alpha1/reservation_types.go b/api/v1alpha1/reservation_types.go index 5e6a30b01..4a7fe5cf2 100644 --- a/api/v1alpha1/reservation_types.go +++ b/api/v1alpha1/reservation_types.go @@ -29,7 +29,7 @@ const ( // LabelReservationType identifies the type of reservation. // This label is present on all reservations to enable type-based filtering. - LabelReservationType = "reservations.cortex.sap.com/type" + LabelReservationType = "reservations.cortex.cloud/type" // Reservation type label values ReservationTypeLabelCommittedResource = "committed-resource" @@ -152,6 +152,18 @@ type FailoverReservationStatus struct { // Key: VM/instance UUID, Value: Host name where the VM is currently running. // +kubebuilder:validation:Optional Allocations map[string]string `json:"allocations,omitempty"` + + // LastChanged tracks when the reservation was last modified. + // This is used to track pending changes that need acknowledgment. + // +kubebuilder:validation:Optional + LastChanged *metav1.Time `json:"lastChanged,omitempty"` + + // AcknowledgedAt is the timestamp when the last change was acknowledged. + // When nil, the reservation is in a pending state awaiting acknowledgment. + // This does not affect the Ready condition - reservations are still considered + // ready even when not yet acknowledged. + // +kubebuilder:validation:Optional + AcknowledgedAt *metav1.Time `json:"acknowledgedAt,omitempty"` } // ReservationStatus defines the observed state of Reservation. @@ -189,7 +201,7 @@ type ReservationStatus struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:scope=Cluster -// +kubebuilder:printcolumn:name="Type",type="string",JSONPath=".metadata.labels['reservations\\.cortex\\.sap\\.com/type']" +// +kubebuilder:printcolumn:name="Type",type="string",JSONPath=".metadata.labels['reservations\\.cortex\\.cloud/type']" // +kubebuilder:printcolumn:name="Host",type="string",JSONPath=".status.host" // +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type=='Ready')].status" diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 96043cc1f..d187be943 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -540,6 +540,14 @@ func (in *FailoverReservationStatus) DeepCopyInto(out *FailoverReservationStatus (*out)[key] = val } } + if in.LastChanged != nil { + in, out := &in.LastChanged, &out.LastChanged + *out = (*in).DeepCopy() + } + if in.AcknowledgedAt != nil { + in, out := &in.AcknowledgedAt, &out.AcknowledgedAt + *out = (*in).DeepCopy() + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FailoverReservationStatus. diff --git a/cmd/main.go b/cmd/main.go index 46a244de1..2b7bbb31f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -42,13 +42,16 @@ import ( "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis" "github.com/cobaltcore-dev/cortex/internal/scheduling/cinder" "github.com/cobaltcore-dev/cortex/internal/scheduling/explanation" + "github.com/cobaltcore-dev/cortex/internal/scheduling/external" schedulinglib "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/cobaltcore-dev/cortex/internal/scheduling/machines" "github.com/cobaltcore-dev/cortex/internal/scheduling/manila" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" "github.com/cobaltcore-dev/cortex/internal/scheduling/pods" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments" reservationscontroller "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/controller" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/failover" "github.com/cobaltcore-dev/cortex/pkg/conf" "github.com/cobaltcore-dev/cortex/pkg/monitoring" "github.com/cobaltcore-dev/cortex/pkg/multicluster" @@ -142,6 +145,12 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + // Log the main configuration + setupLog.Info("loaded main configuration", + "enabledControllers", mainConfig.EnabledControllers, + "enabledTasks", mainConfig.EnabledTasks, + "leaderElectionID", mainConfig.LeaderElectionID) + // if the enable-http2 flag is false (the default), http/2 should be disabled // due to its vulnerabilities. More specifically, disabling http/2 will // prevent from being vulnerable to the HTTP/2 Stream Cancellation and @@ -350,6 +359,7 @@ func main() { } } if slices.Contains(mainConfig.EnabledControllers, "nova-deschedulings-executor") { + setupLog.Info("enabling controller", "controller", "nova-deschedulings-executor") executorConfig := conf.GetConfigOrDie[nova.DeschedulingsExecutorConfig]() novaClient := nova.NewNovaClient() novaClientConfig := conf.GetConfigOrDie[nova.NovaClientConfig]() @@ -379,6 +389,7 @@ func main() { } } if slices.Contains(mainConfig.EnabledControllers, "manila-decisions-pipeline-controller") { + setupLog.Info("enabling controller", "controller", "manila-decisions-pipeline-controller") controller := &manila.FilterWeigherPipelineController{ Monitor: filterWeigherPipelineMonitor, } @@ -398,6 +409,7 @@ func main() { } } if slices.Contains(mainConfig.EnabledControllers, "cinder-decisions-pipeline-controller") { + setupLog.Info("enabling controller", "controller", "cinder-decisions-pipeline-controller") controller := &cinder.FilterWeigherPipelineController{ Monitor: filterWeigherPipelineMonitor, } @@ -417,6 +429,7 @@ func main() { } } if slices.Contains(mainConfig.EnabledControllers, "ironcore-decisions-pipeline-controller") { + setupLog.Info("enabling controller", "controller", "ironcore-decisions-pipeline-controller") controller := &machines.FilterWeigherPipelineController{ Monitor: filterWeigherPipelineMonitor, } @@ -435,6 +448,7 @@ func main() { } } if slices.Contains(mainConfig.EnabledControllers, "pods-decisions-pipeline-controller") { + setupLog.Info("enabling controller", "controller", "pods-decisions-pipeline-controller") controller := &pods.FilterWeigherPipelineController{ Monitor: filterWeigherPipelineMonitor, } @@ -453,6 +467,7 @@ func main() { } } if slices.Contains(mainConfig.EnabledControllers, "explanation-controller") { + setupLog.Info("enabling controller", "controller", "explanation-controller") // Setup a controller which will reconcile the history and explanation for // decision resources. explanationControllerConfig := conf.GetConfigOrDie[explanation.ControllerConfig]() @@ -466,6 +481,7 @@ func main() { } } if slices.Contains(mainConfig.EnabledControllers, "reservations-controller") { + setupLog.Info("enabling controller", "controller", "reservations-controller") monitor := reservationscontroller.NewControllerMonitor(multiclusterClient) metrics.Registry.MustRegister(&monitor) reservationsControllerConfig := conf.GetConfigOrDie[reservationscontroller.Config]() @@ -480,6 +496,7 @@ func main() { } } if slices.Contains(mainConfig.EnabledControllers, "datasource-controllers") { + setupLog.Info("enabling controller", "controller", "datasource-controllers") monitor := datasources.NewMonitor() metrics.Registry.MustRegister(&monitor) if err := (&openstack.OpenStackDatasourceReconciler{ @@ -502,6 +519,7 @@ func main() { } } if slices.Contains(mainConfig.EnabledControllers, "knowledge-controllers") { + setupLog.Info("enabling controller", "controller", "knowledge-controllers") monitor := extractor.NewMonitor() metrics.Registry.MustRegister(&monitor) if err := (&extractor.KnowledgeReconciler{ @@ -523,6 +541,7 @@ func main() { } } if slices.Contains(mainConfig.EnabledControllers, "kpis-controller") { + setupLog.Info("enabling controller", "controller", "kpis-controller") kpisControllerConfig := conf.GetConfigOrDie[kpis.ControllerConfig]() if err := (&kpis.Controller{ Client: multiclusterClient, @@ -532,6 +551,93 @@ func main() { os.Exit(1) } } + if slices.Contains(mainConfig.EnabledControllers, "failover-reservations-controller") { + setupLog.Info("enabling controller", "controller", "failover-reservations-controller") + failoverConfig := conf.GetConfigOrDie[failover.FailoverConfig]() + + // Apply defaults for unset values + defaults := failover.DefaultConfig() + if failoverConfig.DatasourceName == "" { + failoverConfig.DatasourceName = defaults.DatasourceName + } + if failoverConfig.SchedulerURL == "" { + failoverConfig.SchedulerURL = defaults.SchedulerURL + } + if failoverConfig.ReconcileInterval == 0 { + failoverConfig.ReconcileInterval = defaults.ReconcileInterval + } + if failoverConfig.Creator == "" { + failoverConfig.Creator = defaults.Creator + } + if failoverConfig.FlavorFailoverRequirements == nil { + failoverConfig.FlavorFailoverRequirements = defaults.FlavorFailoverRequirements + } + if failoverConfig.RevalidationInterval == 0 { + failoverConfig.RevalidationInterval = defaults.RevalidationInterval + } + + // DatasourceName is still required - check after applying defaults + if failoverConfig.DatasourceName == "" { + setupLog.Error(nil, "failover-reservations-controller requires datasourceName to be configured") + os.Exit(1) + } + + // The scheduler client calls the nova external scheduler API to get placement decisions + schedulerClient := reservations.NewSchedulerClient(failoverConfig.SchedulerURL) + + // Defer the initialization of PostgresReader until the manager starts + // because the cache is not ready during setup + if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { + // Create PostgresReader from the configured Datasource CRD + // This runs after the cache is started + postgresReader, err := external.NewPostgresReader(ctx, multiclusterClient, failoverConfig.DatasourceName) + if err != nil { + setupLog.Error(err, "unable to create postgres reader for failover controller", + "datasourceName", failoverConfig.DatasourceName) + return err + } + + // Create NovaReader and DBVMSource + novaReader := external.NewNovaReader(postgresReader) + vmSource := failover.NewDBVMSource(novaReader) + + // Create the unified failover controller + // It handles both: + // 1. Watch-based per-reservation reconciliation (acknowledgment, validation) + // 2. Periodic bulk VM processing (creating/assigning reservations) + failoverController := failover.NewFailoverReservationController( + multiclusterClient, + vmSource, + failoverConfig, + schedulerClient, + ) + + // Set up the watch-based reconciler for per-reservation reconciliation + if err := failoverController.SetupWithManager(mgr, multiclusterClient); err != nil { + setupLog.Error(err, "unable to set up failover reservation controller") + return err + } + + setupLog.Info("failover-reservations-controller starting", + "datasourceName", failoverConfig.DatasourceName, + "schedulerURL", failoverConfig.SchedulerURL, + "reconcileInterval", failoverConfig.ReconcileInterval, + "revalidationInterval", failoverConfig.RevalidationInterval) + + // Start the controller's periodic reconciliation loop + return failoverController.Start(ctx) + })); err != nil { + setupLog.Error(err, "unable to add failover controller to manager") + os.Exit(1) + } + setupLog.Info("failover-reservations-controller registered", + "datasourceName", failoverConfig.DatasourceName, + "schedulerURL", failoverConfig.SchedulerURL, + "reconcileInterval", failoverConfig.ReconcileInterval, + "revalidationInterval", failoverConfig.RevalidationInterval, + "trustHypervisorLocation", failoverConfig.TrustHypervisorLocation, + "maxVMsToProcess", failoverConfig.MaxVMsToProcess) + } // +kubebuilder:scaffold:builder diff --git a/docs/reservations/failover-reservations.md b/docs/reservations/failover-reservations.md new file mode 100644 index 000000000..1fa36d79d --- /dev/null +++ b/docs/reservations/failover-reservations.md @@ -0,0 +1,184 @@ +# Failover Reservation System + +The failover reservation system ensures VMs have pre-reserved capacity on alternate hypervisors for evacuation. It's a Kubernetes controller that manages `Reservation` CRDs. + +## File Structure + +```text +internal/scheduling/reservations/failover/ +├── config.go # Configuration struct (intervals, flavor requirements) +├── controller.go # Handles lifecycle of Reservation CRD of type failover +├── vm_source.go # VM data source (reads from Nova DB via postgres) +├── reservation_eligibility.go # Checks if a VM can use a failover reservation from a HA perspective (independent of normal scheduling constraints) +├── reservation_scheduling.go # Scheduling (new and reusing) of failover reservations via our scheduling pipeline +└── helpers.go # Utility functions for reservation manipulation +``` + +## Reconciliation Flow + +The controller has two reconciliation modes: + +### Periodic Reconciliation + +*See: `controller.go`* + +```mermaid +flowchart TD + P1[List Hypervisors from K8s] + P2["List VMs from Postgres
(vm_source.go)"] + P3["Remove Invalid VMs from reservations
(e.g., vm:host mapping wrong or vm deleted)"] + P4["Remove Non-eligible VMs from reservations
(via eligibility rules, reservation_eligibility.go)"] + P5[Delete Empty Reservations] + P6["Create/Assign Reservations
(reservation_scheduling.go)"] + + P1 --> P2 --> P3 --> P4 --> P5 --> P6 +``` + + +### Watch-based Reconciliation + +*See: `controller.go`* + +```mermaid +flowchart TD + W1[Reservation Changed] + W2["Validate All Allocated VMs
(via scheduler evacuation check)"] + W3{Valid?} + W4[Update AcknowledgedAt] + W5[Delete Reservation] + + W1 --> W2 --> W3 + W3 -->|Yes| W4 + W3 -->|No| W5 +``` + +## Create/Assign Reservations Detail + +*See: `reservation_scheduling.go`* + +```mermaid +flowchart TD + Start[VM needs failover reservation] + + subgraph Reuse["Try Reuse Existing"] + R1["Query scheduler pipeline for valid hosts (valid_hypervisors)"] + R2["Find eligible reservations (eligibleReservations)"] + R4{Reservation in eligibleReservations on a host in valid_hypervisors?} + R5["Add VM to reservation
(helpers.go)"] + end + + subgraph Create["Create New"] + C1[Query scheduler pipeline for valid hosts with capacity] + C2["Check eligibility for each host"] + C3{Eligible host found?} + C4["Build new Reservation CRD on that host"] + C5[Persist to K8s] + end + + Start --> R1 --> R2 --> R4 + R4 -->|Yes| R5 + R4 -->|No| C1 --> C2 --> C3 + C3 -->|Yes| C4 --> C5 + C3 -->|No| Fail[Log failure, continue] +``` + +## Create new Reservation for a VM flow + +```mermaid +sequenceDiagram + participant C as Controller + participant S as Scheduler API + participant E as Eligibility + + C->>S: ScheduleReservation(VM, eligible hosts, pipeline) + S->>S: Run filter plugins + S-->>C: Valid hosts list + + loop For each valid host + C->>E: IsVMEligibleForReservation(VM, host) + E->>E: Check eligibility constraints + E-->>C: eligible/not eligible + end + + C->>C: Select first eligible host + C->>C: Build/update Reservation CRD +``` + +## Key Components + +### 1. Controller (`controller.go`) + +The main orchestrator with dual reconciliation: +- **Periodic bulk processing**: Runs every `ReconcileInterval` (default 5s), processes all VMs +- **Watch-based validation**: Triggered by Reservation CRD changes, validates individual reservations + +### 2. VM Source (`vm_source.go`) + +Interface `VMSource` with `DBVMSource` implementation: +- Reads VMs from Nova postgres database (servers + flavors join) +- Can trust either postgres (`OSEXTSRVATTRHost`) or Hypervisor CRD for VM location +- Returns `VM` structs with UUID, flavor, resources, extra specs, AZ + +### 3. Eligibility Constraints (`reservation_eligibility.go`) + +Five constraints ensure safe failover without conflicts: + +| # | Constraint | Purpose | +|---|------------|---------| +| 1 | VM cannot reserve slot on its own hypervisor | Failover must be to a different host | +| 2 | VM's N slots must be on N distinct hypervisors | Spread failover capacity | +| 3 | No two VMs using same reservation can be on same hypervisor | Avoid double-booking on failure | +| 4 | VMs sharing slots can't run on each other's hypervisors or slot hosts | Prevent cascading failures | +| 5 | No two VMs using a VM's slots can be on the same hypervisor | Ensure evacuation capacity | + +### 4. Scheduling (`reservation_scheduling.go`) + +Integrates with Nova external scheduler API using three pipelines. + +## Scheduler Pipelines + +We use three different scheduler pipelines for failover reservations, each serving a specific purpose: + +### `kvm-valid-host-reuse-failover-reservation` +**Used when:** Trying to reuse an existing reservation for a VM. + +**Why:** When reusing a reservation, capacity is already reserved on the target host. We only need to verify that the VM is compatible with the host (traits, capabilities, AZ, etc.) without checking if there's enough free capacity. + +### `kvm-valid-host-new-failover-reservation` +**Used when:** Creating a new failover reservation. + +**Why:** When creating a new reservation, we need to find a host that: +1. Is compatible with the VM (traits, capabilities, AZ, etc.) +2. Has enough free capacity to accommodate the VM if it needs to evacuate + +This is the most restrictive pipeline since we're actually reserving new capacity. + +### `kvm-acknowledge-failover-reservation` +**Used when:** Validating that an existing reservation is still valid (watch-based reconciliation). + +**Why:** Periodically we need to verify that a VM could still evacuate to its reserved host. This sends an evacuation-style scheduling request with only the reservation's host as the eligible target. If the scheduler rejects it, the reservation is no longer valid and should be deleted so the periodic controller can create a new one on a valid host. + +## Data Model + +### VM Struct +```go +type VM struct { + UUID string + FlavorName string + ProjectID string + CurrentHypervisor string + AvailabilityZone string + Resources map[string]resource.Quantity // "memory", "vcpus" + FlavorExtraSpecs map[string]string +} +``` + +### Reservation CRD Status +```go +type FailoverReservationStatus struct { + Allocations map[string]string // vmUUID -> hypervisor where VM runs + LastChanged *metav1.Time + AcknowledgedAt *metav1.Time // Set after validation passes +} +``` + diff --git a/helm/bundles/cortex-nova/templates/datasources_kvm.yaml b/helm/bundles/cortex-nova/templates/datasources_kvm.yaml index 3614bda26..c2d78a3ac 100644 --- a/helm/bundles/cortex-nova/templates/datasources_kvm.yaml +++ b/helm/bundles/cortex-nova/templates/datasources_kvm.yaml @@ -28,4 +28,4 @@ spec: timeRange: "1200s" # 20 minutes interval: "300s" # 5 minutes resolution: "60s" # 1 minute -{{- end }} \ No newline at end of file +{{- end }} diff --git a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml index 68ec01352..7d986eb01 100644 --- a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml +++ b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml @@ -82,7 +82,6 @@ spec: description: | This weigher implements the "soft affinity" and "soft anti-affinity" policy for instance groups in nova. - It assigns a weight to each host based on how many instances of the same instance group are already running on that host. The more instances of the same group on a host, the lower (for soft-anti-affinity) or higher @@ -350,6 +349,128 @@ spec: --- apiVersion: cortex.cloud/v1alpha1 kind: Pipeline +metadata: + name: kvm-new-failover-reservation +spec: + schedulingDomain: nova + description: | + This pipeline is used by the failover reservation controller to find a host + for creating a new failover reservation. It validates host compatibility AND + checks capacity. + + Note: Domain filtering (filter_external_customer) is not applied for failover + reservations because domains are currently not considered in failover scheduling. + + This is the pipeline used for KVM hypervisors (qemu and cloud-hypervisor). + type: filter-weigher + createDecisions: true + # Fetch all placement candidates, ignoring nova's preselection. + ignorePreselection: true + filters: + - name: filter_host_instructions + description: | + This step will consider the `ignore_hosts` and `force_hosts` instructions + from the nova scheduler request spec to filter out or exclusively allow + certain hosts. + - name: filter_has_enough_capacity + description: | + This step will filter out hosts that do not have enough available capacity + to host the requested flavor. If enabled, this step will subtract the + current reservations residing on this host from the available capacity. + params: + # If reserved space should be locked even for matching requests. + # For the reservations pipeline, we don't want to unlock + # reserved space, to avoid reservations for the same project + # and flavor to overlap. + - {key: lockReserved, boolValue: true} + - name: filter_has_requested_traits + description: | + This step filters hosts that do not have the requested traits given by the + nova flavor extra spec: "trait:": "forbidden" means the host must + not have the specified trait. "trait:": "required" means the host + must have the specified trait. + - name: filter_has_accelerators + description: | + This step will filter out hosts without the trait `COMPUTE_ACCELERATORS` if + the nova flavor extra specs request accelerators via "accel:device_profile". + - name: filter_correct_az + description: | + This step will filter out hosts whose aggregate information indicates they + are not placed in the requested availability zone. + - name: filter_status_conditions + description: | + This step will filter out hosts for which the hypervisor status conditions + do not meet the expected values, for example, that the hypervisor is ready + and not disabled. + # Note: filter_external_customer is intentionally omitted. + # Domains are currently not considered in failover reservations. + - name: filter_allowed_projects + description: | + This step filters hosts based on allowed projects defined in the + hypervisor resource. Note that hosts allowing all projects are still + accessible and will not be filtered out. In this way some hypervisors + are made accessible to some projects only. + - name: filter_capabilities + description: | + This step will filter out hosts that do not meet the compute capabilities + requested by the nova flavor extra specs, like `{"arch": "x86_64", + "maxphysaddr:bits": 46, ...}`. + + Note: currently, advanced boolean/numeric operators for the capabilities + like `>`, `!`, ... are not supported because they are not used by any of our + flavors in production. + - name: filter_instance_group_affinity + description: | + This step selects hosts in the instance group specified in the nova + scheduler request spec. + - name: filter_instance_group_anti_affinity + description: | + This step selects hosts not in the instance group specified in the nova + scheduler request spec, but only until the max_server_per_host limit is + reached (default = 1). + - name: filter_live_migratable + description: | + This step ensures that the target host of a live migration can accept + the migrating VM, by checking cpu architecture, cpu features, emulated + devices, and cpu modes. + - name: filter_requested_destination + params: {{ .Values.kvm.filterRequestedDestinationParams | toYaml | nindent 8 }} + description: | + This step filters hosts based on the `requested_destination` instruction + from the nova scheduler request spec. It supports filtering by host and + by aggregates. + weighers: + - name: kvm_prefer_smaller_hosts + params: + - {key: resourceWeights, floatMapValue: {"memory": 1.0}} + description: | + This step pulls virtual machines onto smaller hosts (by capacity). This + ensures that larger hosts are not overly fragmented with small VMs, + and can still accommodate larger VMs when they need to be scheduled. + - name: kvm_instance_group_soft_affinity + description: | + This weigher implements the "soft affinity" and "soft anti-affinity" policy + for instance groups in nova. + + It assigns a weight to each host based on how many instances of the same + instance group are already running on that host. The more instances of the + same group on a host, the lower (for soft-anti-affinity) or higher + (for soft-affinity) the weight, which makes it less likely or more likely, + respectively, for the scheduler to choose that host for new instances of + the same group. + - name: kvm_binpack + multiplier: -1.0 # inverted = balancing + params: + - {key: resourceWeights, floatMapValue: {"memory": 1.0}} + description: | + This step implements a balancing weigher for workloads on kvm hypervisors, + which is the opposite of binpacking. Instead of pulling the requested vm + into the smallest gaps possible, it spreads the load to ensure + workloads are balanced across hosts. In this pipeline, the balancing will + focus on general purpose virtual machines. +--- +apiVersion: cortex.cloud/v1alpha1 +kind: Pipeline metadata: name: kvm-descheduler spec: @@ -367,4 +488,138 @@ spec: the observed time span. params: - {key: maxStealPctOverObservedTimeSpan, floatValue: 20.0} -{{- end }} \ No newline at end of file +--- +apiVersion: cortex.cloud/v1alpha1 +kind: Pipeline +metadata: + name: kvm-valid-host-reuse-failover-reservation +spec: + schedulingDomain: nova + description: | + This pipeline is used by the failover reservation controller to check if a VM + can reuse an existing failover reservation. It validates that the reservation + host is compatible with the VM's requirements (traits, capabilities, etc.) + without checking capacity, since the reservation already has capacity reserved. + + Use case: When a VM needs failover protection and there's an existing reservation + on a host, this pipeline validates the host is still suitable for the VM. + type: filter-weigher + createDecisions: false + filters: + - name: filter_host_instructions + description: | + This step will consider the `ignore_hosts` and `force_hosts` instructions + from the nova scheduler request spec to filter out or exclusively allow + certain hosts. + - name: filter_has_requested_traits + description: | + This step filters hosts that do not have the requested traits given by the + nova flavor extra spec: "trait:": "forbidden" means the host must + not have the specified trait. "trait:": "required" means the host + must have the specified trait. + - name: filter_has_accelerators + description: | + This step will filter out hosts without the trait `COMPUTE_ACCELERATORS` if + the nova flavor extra specs request accelerators via "accel:device_profile". + - name: filter_correct_az + description: | + This step will filter out hosts whose aggregate information indicates they + are not placed in the requested availability zone. This ensures VMs can + only reuse reservations in their own AZ. + - name: filter_status_conditions + description: | + This step will filter out hosts for which the hypervisor status conditions + do not meet the expected values, for example, that the hypervisor is ready + and not disabled. + - name: filter_external_customer + description: | + This step prefix-matches the domain name for external customer domains and + filters out hosts that are not intended for external customers. It considers + the `CUSTOM_EXTERNAL_CUSTOMER_SUPPORTED` trait on hosts as well as the + `domain_name` scheduler hint from the nova request spec. + params: + - {key: domainNamePrefixes, stringListValue: ["iaas-"]} + - name: filter_allowed_projects + description: | + This step filters hosts based on allowed projects defined in the + hypervisor resource. Note that hosts allowing all projects are still + accessible and will not be filtered out. In this way some hypervisors + are made accessible to some projects only. + - name: filter_capabilities + description: | + This step will filter out hosts that do not meet the compute capabilities + requested by the nova flavor extra specs, like `{"arch": "x86_64", + "maxphysaddr:bits": 46, ...}`. + weighers: [] +--- +apiVersion: cortex.cloud/v1alpha1 +kind: Pipeline +metadata: + name: kvm-acknowledge-failover-reservation +spec: + schedulingDomain: nova + description: | + This pipeline is used by the failover reservation controller to validate that + a failover reservation is still valid for all its allocated VMs. It sends an + evacuation-style scheduling request for each VM with only the reservation's + host as the eligible target. + + Use case: After a reservation is created or modified, this pipeline validates + that the reservation host can still accommodate all allocated VMs. If validation + fails for any VM, the reservation is deleted (nack). + type: filter-weigher + createDecisions: false + filters: + - name: filter_host_instructions + description: | + This step will consider the `ignore_hosts` and `force_hosts` instructions + from the nova scheduler request spec to filter out or exclusively allow + certain hosts. + - name: filter_has_enough_capacity + description: | + This step will filter out hosts that do not have enough available capacity + to host the requested flavor. Reservations are considered to ensure we + don't double-book capacity. + params: + - {key: lockReserved, boolValue: true} + - name: filter_has_requested_traits + description: | + This step filters hosts that do not have the requested traits given by the + nova flavor extra spec: "trait:": "forbidden" means the host must + not have the specified trait. "trait:": "required" means the host + must have the specified trait. + - name: filter_has_accelerators + description: | + This step will filter out hosts without the trait `COMPUTE_ACCELERATORS` if + the nova flavor extra specs request accelerators via "accel:device_profile". + - name: filter_correct_az + description: | + This step will filter out hosts whose aggregate information indicates they + are not placed in the requested availability zone. This ensures reservation + validation respects AZ boundaries. + - name: filter_status_conditions + description: | + This step will filter out hosts for which the hypervisor status conditions + do not meet the expected values, for example, that the hypervisor is ready + and not disabled. + - name: filter_external_customer + description: | + This step prefix-matches the domain name for external customer domains and + filters out hosts that are not intended for external customers. It considers + the `CUSTOM_EXTERNAL_CUSTOMER_SUPPORTED` trait on hosts as well as the + `domain_name` scheduler hint from the nova request spec. + params: + - {key: domainNamePrefixes, stringListValue: ["iaas-"]} + - name: filter_allowed_projects + description: | + This step filters hosts based on allowed projects defined in the + hypervisor resource. Note that hosts allowing all projects are still + accessible and will not be filtered out. In this way some hypervisors + are made accessible to some projects only. + - name: filter_capabilities + description: | + This step will filter out hosts that do not meet the compute capabilities + requested by the nova flavor extra specs, like `{"arch": "x86_64", + "maxphysaddr:bits": 46, ...}`. + weighers: [] +{{- end }} diff --git a/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml b/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml index d9256e5db..ab622c1c0 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml @@ -15,7 +15,7 @@ spec: scope: Cluster versions: - additionalPrinterColumns: - - jsonPath: .metadata.labels['reservations\.cortex\.sap\.com/type'] + - jsonPath: .metadata.labels['reservations\.cortex\.cloud/type'] name: Type type: string - jsonPath: .status.host @@ -242,6 +242,14 @@ spec: FailoverReservation contains status fields specific to failover reservations. Only used when Type is FailoverReservation. properties: + acknowledgedAt: + description: |- + AcknowledgedAt is the timestamp when the last change was acknowledged. + When nil, the reservation is in a pending state awaiting acknowledgment. + This does not affect the Ready condition - reservations are still considered + ready even when not yet acknowledged. + format: date-time + type: string allocations: additionalProperties: type: string @@ -249,6 +257,12 @@ spec: Allocations maps VM/instance UUIDs to the host they are currently allocated on. Key: VM/instance UUID, Value: Host name where the VM is currently running. type: object + lastChanged: + description: |- + LastChanged tracks when the reservation was last modified. + This is used to track pending changes that need acknowledgment. + format: date-time + type: string type: object host: description: |- diff --git a/internal/scheduling/external/nova.go b/internal/scheduling/external/nova.go new file mode 100644 index 000000000..b59a37d5b --- /dev/null +++ b/internal/scheduling/external/nova.go @@ -0,0 +1,109 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package external + +import ( + "context" + "fmt" + + nova "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/nova" +) + +// NovaReaderInterface defines the methods needed to read Nova data. +// This interface allows mocking the NovaReader in tests. +type NovaReaderInterface interface { + GetAllServers(ctx context.Context) ([]nova.Server, error) + GetAllFlavors(ctx context.Context) ([]nova.Flavor, error) + GetServerByID(ctx context.Context, serverID string) (*nova.Server, error) + GetFlavorByName(ctx context.Context, flavorName string) (*nova.Flavor, error) +} + +// NovaReader provides read access to Nova data stored in the database. +// It uses a PostgresReader to connect to the database. +type NovaReader struct { + *PostgresReader +} + +// NewNovaReader creates a new NovaReader from a PostgresReader. +func NewNovaReader(reader *PostgresReader) *NovaReader { + return &NovaReader{PostgresReader: reader} +} + +// GetAllServers returns all Nova servers from the database. +func (r *NovaReader) GetAllServers(ctx context.Context) ([]nova.Server, error) { + var servers []nova.Server + query := "SELECT * FROM " + nova.Server{}.TableName() + if err := r.Select(ctx, &servers, query); err != nil { + return nil, fmt.Errorf("failed to query servers: %w", err) + } + return servers, nil +} + +// GetAllFlavors returns all Nova flavors from the database. +func (r *NovaReader) GetAllFlavors(ctx context.Context) ([]nova.Flavor, error) { + var flavors []nova.Flavor + query := "SELECT * FROM " + nova.Flavor{}.TableName() + if err := r.Select(ctx, &flavors, query); err != nil { + return nil, fmt.Errorf("failed to query flavors: %w", err) + } + return flavors, nil +} + +// GetAllHypervisors returns all Nova hypervisors from the database. +func (r *NovaReader) GetAllHypervisors(ctx context.Context) ([]nova.Hypervisor, error) { + var hypervisors []nova.Hypervisor + query := "SELECT * FROM " + nova.Hypervisor{}.TableName() + if err := r.Select(ctx, &hypervisors, query); err != nil { + return nil, fmt.Errorf("failed to query hypervisors: %w", err) + } + return hypervisors, nil +} + +// GetAllMigrations returns all Nova migrations from the database. +func (r *NovaReader) GetAllMigrations(ctx context.Context) ([]nova.Migration, error) { + var migrations []nova.Migration + query := "SELECT * FROM " + nova.Migration{}.TableName() + if err := r.Select(ctx, &migrations, query); err != nil { + return nil, fmt.Errorf("failed to query migrations: %w", err) + } + return migrations, nil +} + +// GetAllAggregates returns all Nova aggregates from the database. +func (r *NovaReader) GetAllAggregates(ctx context.Context) ([]nova.Aggregate, error) { + var aggregates []nova.Aggregate + query := "SELECT * FROM " + nova.Aggregate{}.TableName() + if err := r.Select(ctx, &aggregates, query); err != nil { + return nil, fmt.Errorf("failed to query aggregates: %w", err) + } + return aggregates, nil +} + +// GetServerByID returns a Nova server by its ID. +// Returns nil, nil if the server is not found. +func (r *NovaReader) GetServerByID(ctx context.Context, serverID string) (*nova.Server, error) { + var servers []nova.Server + query := "SELECT * FROM " + nova.Server{}.TableName() + " WHERE id = $1" + if err := r.Select(ctx, &servers, query, serverID); err != nil { + return nil, fmt.Errorf("failed to query server by ID: %w", err) + } + if len(servers) == 0 { + return nil, nil + } + return &servers[0], nil +} + +// GetFlavorByName returns a Nova flavor by its name. +// Returns nil, nil if the flavor is not found. +func (r *NovaReader) GetFlavorByName(ctx context.Context, flavorName string) (*nova.Flavor, error) { + var flavors []nova.Flavor + query := "SELECT * FROM " + nova.Flavor{}.TableName() + " WHERE name = $1" + if err := r.Select(ctx, &flavors, query, flavorName); err != nil { + return nil, fmt.Errorf("failed to query flavor by name: %w", err) + } + if len(flavors) == 0 { + return nil, nil + } + return &flavors[0], nil +} diff --git a/internal/scheduling/external/postgres.go b/internal/scheduling/external/postgres.go new file mode 100644 index 000000000..ac7468749 --- /dev/null +++ b/internal/scheduling/external/postgres.go @@ -0,0 +1,75 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +// Package external provides access to external data sources for scheduling. +package external + +import ( + "context" + "fmt" + "sync" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/knowledge/db" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// PostgresReader provides read access to a postgres database. +// It reads the database connection info from a Datasource CRD. +type PostgresReader struct { + // Kubernetes client to read the Datasource CRD and secrets. + Client client.Client + // Reference to the database secret containing connection info. + DatabaseSecretRef corev1.SecretReference + // Mutex to protect lazy initialization of the database connection. + mu sync.Mutex + // Cached database connection (lazily initialized). + db *db.DB +} + +// NewPostgresReader creates a new PostgresReader from a Datasource CRD name. +// It looks up the Datasource CRD to get the database secret reference. +func NewPostgresReader(ctx context.Context, c client.Client, datasourceName string) (*PostgresReader, error) { + // Look up the Datasource CRD to get the database secret reference + datasource := &v1alpha1.Datasource{} + if err := c.Get(ctx, client.ObjectKey{Name: datasourceName}, datasource); err != nil { + return nil, fmt.Errorf("failed to get datasource %s: %w", datasourceName, err) + } + + return &PostgresReader{ + Client: c, + DatabaseSecretRef: datasource.Spec.DatabaseSecretRef, + }, nil +} + +// DB returns the database connection, initializing it if necessary. +// This method is safe for concurrent use. +func (r *PostgresReader) DB(ctx context.Context) (*db.DB, error) { + r.mu.Lock() + defer r.mu.Unlock() + + if r.db != nil { + return r.db, nil + } + + // Connect to the database using the secret reference + database, err := db.Connector{Client: r.Client}.FromSecretRef(ctx, r.DatabaseSecretRef) + if err != nil { + return nil, fmt.Errorf("failed to connect to database: %w", err) + } + + r.db = database + return r.db, nil +} + +// Select executes a SELECT query and returns the results. +func (r *PostgresReader) Select(ctx context.Context, dest any, query string, args ...any) error { + database, err := r.DB(ctx) + if err != nil { + return err + } + + _, err = database.Select(dest, query, args...) + return err +} diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go index 5e1f1dc3c..24b7f7815 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go @@ -190,33 +190,42 @@ func (s *FilterHasEnoughCapacity) Run(traceLog *slog.Logger, request api.Externa // Block the calculated resources on each host for host := range hostsToBlock { + // Skip hosts that don't have a corresponding Hypervisor resource. + if _, hostExists := freeResourcesByHost[host]; !hostExists { + traceLog.Debug("skipping reservation for unknown host", + "reservation", reservation.Name, + "host", host) + continue + } if cpu, ok := resourcesToBlock["cpu"]; ok { - freeCPU := freeResourcesByHost[host]["cpu"] - freeCPU.Sub(cpu) - if freeCPU.Value() < 0 { - traceLog.Warn("negative free CPU after blocking reservation", - "host", host, - "reservation", reservation.Name, - "reservationType", reservation.Spec.Type, - "freeCPU", freeCPU.String(), - "blocked", cpu.String()) - freeCPU = resource.MustParse("0") + if freeCPU, exists := freeResourcesByHost[host]["cpu"]; exists { + freeCPU.Sub(cpu) + if freeCPU.Value() < 0 { + traceLog.Warn("negative free CPU after blocking reservation", + "host", host, + "reservation", reservation.Name, + "reservationType", reservation.Spec.Type, + "freeCPU", freeCPU.String(), + "blocked", cpu.String()) + freeCPU = resource.MustParse("0") + } + freeResourcesByHost[host]["cpu"] = freeCPU } - freeResourcesByHost[host]["cpu"] = freeCPU } if memory, ok := resourcesToBlock["memory"]; ok { - freeMemory := freeResourcesByHost[host]["memory"] - freeMemory.Sub(memory) - if freeMemory.Value() < 0 { - traceLog.Warn("negative free memory after blocking reservation", - "host", host, - "reservation", reservation.Name, - "reservationType", reservation.Spec.Type, - "freeMemory", freeMemory.String(), - "blocked", memory.String()) - freeMemory = resource.MustParse("0") + if freeMemory, exists := freeResourcesByHost[host]["memory"]; exists { + freeMemory.Sub(memory) + if freeMemory.Value() < 0 { + traceLog.Warn("negative free memory after blocking reservation", + "host", host, + "reservation", reservation.Name, + "reservationType", reservation.Spec.Type, + "freeMemory", freeMemory.String(), + "blocked", memory.String()) + freeMemory = resource.MustParse("0") + } + freeResourcesByHost[host]["memory"] = freeMemory } - freeResourcesByHost[host]["memory"] = freeMemory } } } diff --git a/internal/scheduling/reservations/context.go b/internal/scheduling/reservations/context.go new file mode 100644 index 000000000..2b7099855 --- /dev/null +++ b/internal/scheduling/reservations/context.go @@ -0,0 +1,50 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package reservations + +import "context" + +// ContextKey is the type for context keys used in request tracking. +type ContextKey string + +const ( + // GlobalRequestIDKey is the context key for the global request ID. + GlobalRequestIDKey ContextKey = "globalRequestID" + // RequestIDKey is the context key for the request ID. + RequestIDKey ContextKey = "requestID" +) + +// WithGlobalRequestID returns a new context with the global request ID set. +// GlobalRequestID identifies the overall reconciliation context. +func WithGlobalRequestID(ctx context.Context, id string) context.Context { + return context.WithValue(ctx, GlobalRequestIDKey, id) +} + +// WithRequestID returns a new context with the request ID set. +// RequestID identifies the specific item being processed (typically VM UUID). +func WithRequestID(ctx context.Context, id string) context.Context { + return context.WithValue(ctx, RequestIDKey, id) +} + +// GlobalRequestIDFromContext retrieves the global request ID from the context. +// Returns empty string if not set. +func GlobalRequestIDFromContext(ctx context.Context) string { + if v := ctx.Value(GlobalRequestIDKey); v != nil { + if s, ok := v.(string); ok { + return s + } + } + return "" +} + +// RequestIDFromContext retrieves the request ID from the context. +// Returns empty string if not set. +func RequestIDFromContext(ctx context.Context) string { + if v := ctx.Value(RequestIDKey); v != nil { + if s, ok := v.(string); ok { + return s + } + } + return "" +} diff --git a/internal/scheduling/reservations/controller/controller.go b/internal/scheduling/reservations/controller/controller.go index 17177770c..635d3132e 100644 --- a/internal/scheduling/reservations/controller/controller.go +++ b/internal/scheduling/reservations/controller/controller.go @@ -19,8 +19,10 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" schedulerdelegationapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" @@ -29,6 +31,7 @@ import ( "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" "github.com/cobaltcore-dev/cortex/pkg/multicluster" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" corev1 "k8s.io/api/core/v1" ) @@ -73,6 +76,8 @@ type ReservationReconciler struct { // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. +// Note: Failover reservations are filtered out at the watch level by the predicate +// in SetupWithManager, so this function only handles non-failover reservations. func (r *ReservationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := logf.FromContext(ctx) // Fetch the reservation object. @@ -182,7 +187,7 @@ func (r *ReservationReconciler) Reconcile(ctx context.Context, req ctrl.Request) // Convert resource.Quantity to integers for the API var memoryMB uint64 - if memory, ok := res.Spec.Resources["memory"]; ok { + if memory, ok := res.Spec.Resources[hv1.ResourceMemory]; ok { memoryValue := memory.ScaledValue(resource.Mega) if memoryValue < 0 { return ctrl.Result{}, fmt.Errorf("invalid memory value: %d", memoryValue) @@ -191,7 +196,7 @@ func (r *ReservationReconciler) Reconcile(ctx context.Context, req ctrl.Request) } var cpu uint64 - if cpuQuantity, ok := res.Spec.Resources["cpu"]; ok { + if cpuQuantity, ok := res.Spec.Resources[hv1.ResourceCPU]; ok { cpuValue := cpuQuantity.ScaledValue(resource.Milli) if cpuValue < 0 { return ctrl.Result{}, fmt.Errorf("invalid cpu value: %d", cpuValue) @@ -466,6 +471,41 @@ func (r *ReservationReconciler) listServersByProjectID(ctx context.Context, proj return serverMap, nil } +// notFailoverReservationPredicate filters out failover reservations at the watch level. +// This prevents the controller from being notified about failover reservations, +// which are managed by the separate failover controller. +// Failover reservations are identified by the label v1alpha1.LabelReservationType. +var notFailoverReservationPredicate = predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + res, ok := e.Object.(*v1alpha1.Reservation) + if !ok { + return false + } + return res.Labels[v1alpha1.LabelReservationType] != v1alpha1.ReservationTypeLabelFailover + }, + UpdateFunc: func(e event.UpdateEvent) bool { + res, ok := e.ObjectNew.(*v1alpha1.Reservation) + if !ok { + return false + } + return res.Labels[v1alpha1.LabelReservationType] != v1alpha1.ReservationTypeLabelFailover + }, + DeleteFunc: func(e event.DeleteEvent) bool { + res, ok := e.Object.(*v1alpha1.Reservation) + if !ok { + return false + } + return res.Labels[v1alpha1.LabelReservationType] != v1alpha1.ReservationTypeLabelFailover + }, + GenericFunc: func(e event.GenericEvent) bool { + res, ok := e.Object.(*v1alpha1.Reservation) + if !ok { + return false + } + return res.Labels[v1alpha1.LabelReservationType] != v1alpha1.ReservationTypeLabelFailover + }, +} + // SetupWithManager sets up the controller with the Manager. func (r *ReservationReconciler) SetupWithManager(mgr ctrl.Manager, mcl *multicluster.Client) error { if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { @@ -478,6 +518,7 @@ func (r *ReservationReconciler) SetupWithManager(mgr ctrl.Manager, mcl *multiclu } return multicluster.BuildController(mcl, mgr). For(&v1alpha1.Reservation{}). + WithEventFilter(notFailoverReservationPredicate). Named("reservation"). WithOptions(controller.Options{ // We want to process reservations one at a time to avoid overbooking. diff --git a/internal/scheduling/reservations/failover/config.go b/internal/scheduling/reservations/failover/config.go new file mode 100644 index 000000000..f5e01ba09 --- /dev/null +++ b/internal/scheduling/reservations/failover/config.go @@ -0,0 +1,67 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failover + +import "time" + +// FailoverConfig defines the configuration for failover reservation management. +type FailoverConfig struct { + // FlavorFailoverRequirements maps flavor name patterns to required failover count. + // Example: {"hana_*": 2, "m1.xlarge": 1} + // A VM with a matching flavor will need this many failover reservations. + FlavorFailoverRequirements map[string]int `json:"flavorFailoverRequirements"` + + // ReconcileInterval is how often to check for missing failover reservations. + ReconcileInterval time.Duration `json:"reconcileInterval"` + + // Creator tag for failover reservations (for identification and cleanup). + Creator string `json:"creator"` + + // DatasourceName is the name of the Datasource CRD that provides database connection info. + // This is used to read VM data from the Nova database. + DatasourceName string `json:"datasourceName"` + + // SchedulerURL is the URL of the nova external scheduler API. + // Example: "http://localhost:8080/scheduler/nova/external" + SchedulerURL string `json:"schedulerURL"` + + // MaxVMsToProcess limits the number of VMs to process per reconciliation cycle. + // Set to 0 or negative to process all VMs (default behavior). + // Useful for debugging and testing with large VM counts. + MaxVMsToProcess int `json:"maxVMsToProcess"` + + // ShortReconcileInterval is used when MaxVMsToProcess limits processing. + // This allows faster catch-up when there are more VMs to process. + // Set to 0 to use ReconcileInterval (default behavior). + ShortReconcileInterval time.Duration `json:"shortReconcileInterval"` + + // TrustHypervisorLocation when true, uses the hypervisor CRD as the source of truth + // for VM location instead of postgres (OSEXTSRVATTRHost). This is useful when there + // are data sync issues between nova and the hypervisor operator. + // When enabled: + // - VM location comes from hypervisor CRD (which hypervisor lists the VM in its instances) + // - VM size/flavor still comes from postgres (needed for scheduling) + // Default: false (use postgres OSEXTSRVATTRHost for location) + TrustHypervisorLocation bool `json:"trustHypervisorLocation"` + + // RevalidationInterval is how often to re-validate acknowledged failover reservations. + // After a reservation is acknowledged, it will be re-validated after this interval + // to ensure the reservation host is still valid for all allocated VMs. + // Default: 30 minutes + RevalidationInterval time.Duration `json:"revalidationInterval"` +} + +// DefaultConfig returns a default configuration. +func DefaultConfig() FailoverConfig { + return FailoverConfig{ + FlavorFailoverRequirements: map[string]int{"*": 2}, // by default all VMs get 2 failover reservations + ReconcileInterval: 30 * time.Second, + ShortReconcileInterval: 100 * time.Millisecond, + Creator: "cortex-failover-controller", + DatasourceName: "nova-servers", // we have the server and flavor data source (both store in same postgres and same secret but still) + SchedulerURL: "http://localhost:8080/scheduler/nova/external", + TrustHypervisorLocation: false, + RevalidationInterval: 30 * time.Minute, + } +} diff --git a/internal/scheduling/reservations/failover/context.go b/internal/scheduling/reservations/failover/context.go new file mode 100644 index 000000000..e18d5eac5 --- /dev/null +++ b/internal/scheduling/reservations/failover/context.go @@ -0,0 +1,21 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failover + +import ( + "context" + + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/go-logr/logr" +) + +// LoggerFromContext returns a logger with greq and req values from the context. +// This creates a child logger with the request tracking values pre-attached, +// so you don't need to repeat them in every log call. +func LoggerFromContext(ctx context.Context) logr.Logger { + return log.WithValues( + "greq", reservations.GlobalRequestIDFromContext(ctx), + "req", reservations.RequestIDFromContext(ctx), + ) +} diff --git a/internal/scheduling/reservations/failover/controller.go b/internal/scheduling/reservations/failover/controller.go new file mode 100644 index 000000000..949ec894a --- /dev/null +++ b/internal/scheduling/reservations/failover/controller.go @@ -0,0 +1,823 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failover + +import ( + "context" + "fmt" + "path/filepath" + "sort" + "time" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/pkg/multicluster" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "github.com/google/uuid" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/events" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +var log = ctrl.Log.WithName("failover-reservation-controller").WithValues("module", "failover-reservations") + +// FailoverReservationController manages failover reservations for VMs. +// It provides two reconciliation modes: +// 1. Periodic bulk reconciliation (ReconcilePeriodic) - processes all VMs to ensure proper failover coverage +// 2. Watch-based per-reservation reconciliation (Reconcile) - handles acknowledgment and validation of individual reservations +type FailoverReservationController struct { + client.Client + VMSource VMSource + Config FailoverConfig + SchedulerClient *reservations.SchedulerClient + Recorder events.EventRecorder // Event recorder for emitting Kubernetes events + reconcileCount int64 // Track reconciliation count for rotating VM selection +} + +func NewFailoverReservationController(c client.Client, vmSource VMSource, config FailoverConfig, schedulerClient *reservations.SchedulerClient) *FailoverReservationController { + return &FailoverReservationController{ + Client: c, + VMSource: vmSource, + Config: config, + SchedulerClient: schedulerClient, + } +} + +type vmFailoverNeed struct { + VM VM + Count int // Number of failover reservations needed +} + +// ============================================================================ +// Watch-based Reconciliation (per-reservation) +// ============================================================================ + +// Reconcile handles watch-based reconciliation for a single failover reservation. +// It validates the reservation and acknowledges it if valid, or deletes it if invalid. +// After processing, it requeues for periodic re-validation. +func (c *FailoverReservationController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + // Generate a random UUID for request tracking + globalReqID := uuid.New().String() + ctx = reservations.WithGlobalRequestID(ctx, globalReqID) + logger := LoggerFromContext(ctx).WithValues("reservation", req.Name, "namespace", req.Namespace) + logger.Info("reconciling failover reservation", "reservation", req.Name) + + // Fetch the reservation + var res v1alpha1.Reservation + if err := c.Get(ctx, req.NamespacedName, &res); err != nil { + if apierrors.IsNotFound(err) { + logger.V(1).Info("reservation not found, likely deleted") + return ctrl.Result{}, nil + } + logger.Error(err, "failed to get reservation") + return ctrl.Result{}, err + } + + // Skip non-failover reservations (should be filtered by predicate, but double-check) + if res.Spec.Type != v1alpha1.ReservationTypeFailover { + logger.V(1).Info("skipping non-failover reservation") + return ctrl.Result{}, nil + } + + // Skip if no failover status (reservation not yet initialized by periodic controller) + if res.Status.FailoverReservation == nil { + logger.V(1).Info("skipping reservation without failover status") + return ctrl.Result{RequeueAfter: c.Config.RevalidationInterval}, nil + } + + // Validate and acknowledge the reservation + return c.reconcileValidateAndAcknowledge(ctx, &res) +} + +// reconcileValidateAndAcknowledge validates a reservation and acknowledges it if valid. +// If invalid, the reservation is marked as not ready. On transient errors, the reservation is requeued. +// On success, AcknowledgedAt is always updated. +func (c *FailoverReservationController) reconcileValidateAndAcknowledge(ctx context.Context, res *v1alpha1.Reservation) (ctrl.Result, error) { + logger := LoggerFromContext(ctx).WithValues("reservation", res.Name) + logger.V(1).Info("validating failover reservation") + + // Validate resource keys first (must be "cpu" and "memory" only) + if err := ValidateFailoverReservationResources(res); err != nil { + logger.Info("reservation has invalid resources, marking as not ready", "error", err) + + updatedRes := res.DeepCopy() + meta.SetStatusCondition(&updatedRes.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ReservationConditionReady, + Status: metav1.ConditionFalse, + Reason: "InvalidResources", + Message: err.Error(), + LastTransitionTime: metav1.Now(), + }) + + if patchErr := c.patchReservationStatus(ctx, updatedRes); patchErr != nil { + logger.Error(patchErr, "failed to update reservation status for invalid resources") + return ctrl.Result{}, patchErr + } + + return ctrl.Result{RequeueAfter: c.Config.RevalidationInterval}, nil + } + + // Validate the reservation + valid, validationErr := c.validateReservation(ctx, res) + + if validationErr != nil { + logger.Error(validationErr, "transient error during reservation validation, will retry", "host", res.Status.Host) + return ctrl.Result{RequeueAfter: c.Config.RevalidationInterval}, nil + } + + if !valid { + logger.Info("reservation validation failed, deleting", "host", res.Status.Host) + if err := c.Delete(ctx, res); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + logger.Error(err, "failed to delete invalid reservation") + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + + // Emit event for successful validation (doesn't trigger watch reconciliation) + c.Recorder.Eventf(res, nil, corev1.EventTypeNormal, "ValidationPassed", "Validated", + "Reservation validated successfully for host %s with %d VMs", + res.Status.Host, len(getFailoverAllocations(res))) + + // Only update AcknowledgedAt if there are unacknowledged changes + lastChanged := res.Status.FailoverReservation.LastChanged + acknowledgedAt := res.Status.FailoverReservation.AcknowledgedAt + if lastChanged != nil && (acknowledgedAt == nil || acknowledgedAt.Before(lastChanged)) { + updatedRes := res.DeepCopy() + now := metav1.Now() + updatedRes.Status.FailoverReservation.AcknowledgedAt = &now + + if err := c.patchReservationStatus(ctx, updatedRes); err != nil { + logger.Error(err, "failed to update reservation acknowledgment") + return ctrl.Result{}, err + } + + logger.V(1).Info("reservation changes acknowledged", "host", res.Status.Host, "lastChanged", lastChanged, "acknowledgedAt", now) + } else { + logger.V(1).Info("reservation validation passed (no new changes to acknowledge)", "host", res.Status.Host) + } + + return ctrl.Result{RequeueAfter: c.Config.RevalidationInterval}, nil +} + +// validateReservation validates that a reservation is still valid for all its allocated VMs. +// Returns: +// - (true, nil) if all VMs pass validation +// - (false, nil) if any VM definitively fails validation (reservation should be deleted) +// - (false, error) if a transient error occurred (reservation should be requeued, not deleted) +func (c *FailoverReservationController) validateReservation(ctx context.Context, res *v1alpha1.Reservation) (bool, error) { + logger := LoggerFromContext(ctx).WithValues("reservationName", res.Name) + allocations := getFailoverAllocations(res) + if len(allocations) == 0 { + return true, nil + } + + reservationHost := res.Status.Host + if reservationHost == "" { + logger.Info("reservation has no host, marking as invalid") + return false, nil + } + + logger.V(1).Info("validating reservation", "host", reservationHost, "vmCount", len(allocations)) + + for vmUUID, vmCurrentHost := range allocations { + vmCtx := reservations.WithRequestID(ctx, vmUUID) + vmLogger := LoggerFromContext(vmCtx).WithValues("vmUUID", vmUUID, "reservationName", res.Name) + + vm, err := c.VMSource.GetVM(vmCtx, vmUUID) + if err != nil { + vmLogger.Error(err, "transient error getting VM for validation") + return false, fmt.Errorf("failed to get VM %s: %w", vmUUID, err) + } + if vm == nil { + vmLogger.V(1).Info("VM not found during validation, skipping") + continue + } + + valid, err := c.validateVMViaSchedulerEvacuation(vmCtx, *vm, reservationHost) + if err != nil { + vmLogger.Error(err, "transient error validating VM for reservation host", "reservationHost", reservationHost) + return false, fmt.Errorf("failed to validate VM %s: %w", vmUUID, err) + } + + if !valid { + vmLogger.Info("VM failed validation for reservation host", "vmCurrentHost", vmCurrentHost, "reservationHost", reservationHost) + return false, nil + } + + vmLogger.V(1).Info("VM passed validation for reservation host", "reservationHost", reservationHost) + } + + return true, nil +} + +// ============================================================================ +// Periodic Bulk Reconciliation +// ============================================================================ + +// reconcileSummary holds statistics from the reconciliation cycle. +type reconcileSummary struct { + vmsProcessed int + reservationsNeeded int + totalReused int + totalCreated int + totalFailed int + reservationsUpdated int + reservationsDeleted int +} + +// ReconcilePeriodic handles the periodic bulk reconciliation of all VMs and reservations. +// This ensures VMs have proper failover coverage by creating, reusing, and cleaning up reservations. +// TODO consider moving Step 3-5 (particularly) to the watch-based reconciliation +func (c *FailoverReservationController) ReconcilePeriodic(ctx context.Context) (ctrl.Result, error) { + startTime := time.Now() + c.reconcileCount++ + globalReqID := uuid.New().String() + ctx = reservations.WithGlobalRequestID(ctx, globalReqID) + logger := LoggerFromContext(ctx) + + var summary reconcileSummary + + // 1. Get hypervisors from the cluster + var hypervisorList hv1.HypervisorList + if err := c.List(ctx, &hypervisorList); err != nil { + logger.Error(err, "failed to list hypervisors") + return ctrl.Result{}, err + } + + allHypervisors := make([]string, 0, len(hypervisorList.Items)) + for _, hv := range hypervisorList.Items { + allHypervisors = append(allHypervisors, hv.Name) + } + + // 2. Get all VMs that might need failover reservations + vms, err := c.VMSource.ListVMsOnHypervisors(ctx, &hypervisorList, c.Config.TrustHypervisorLocation) + if err != nil { + logger.Error(err, "failed to list VMs") + return ctrl.Result{}, err + } + logger.V(1).Info("found VMs from source", "count", len(vms)) + + // List only failover reservations using label selector + var reservationList v1alpha1.ReservationList + if err := c.List(ctx, &reservationList, client.MatchingLabels{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelFailover, + }); err != nil { + logger.Error(err, "failed to list failover reservations") + return ctrl.Result{}, err + } + failoverReservations := reservationList.Items + logger.V(1).Info("found failover reservations", "count", len(failoverReservations)) + + // 3. Remove VMs from reservations if they are no longer valid + failoverReservations, reservationsToUpdate := reconcileRemoveInvalidVMFromReservations(ctx, vms, failoverReservations) + + for _, res := range reservationsToUpdate { + if err := c.patchReservationStatus(ctx, res); err != nil { + logger.Error(err, "failed to update reservation after removing invalid VMs", "reservationName", res.Name) + } + } + summary.reservationsUpdated += len(reservationsToUpdate) + + // 4. Remove VMs from reservations if they no longer meet eligibility criteria + failoverReservations, nonEligibleReservationsToUpdate := reconcileRemoveNoneligibleVMFromReservations(ctx, vms, failoverReservations) + + for _, res := range nonEligibleReservationsToUpdate { + if err := c.patchReservationStatus(ctx, res); err != nil { + logger.Error(err, "failed to update reservation after removing non-eligible VMs", "reservationName", res.Name) + } + } + summary.reservationsUpdated += len(nonEligibleReservationsToUpdate) + + // 5. Remove empty failover reservations + failoverReservations, emptyReservationsToDelete := reconcileRemoveEmptyReservations(ctx, failoverReservations) + + for _, res := range emptyReservationsToDelete { + if err := c.Delete(ctx, res); err != nil { + logger.Error(err, "failed to delete empty failover reservation", "reservationName", res.Name) + } else { + logger.V(1).Info("deleted empty failover reservation", "reservationName", res.Name, "hypervisor", res.Status.Host) + } + } + summary.reservationsDeleted = len(emptyReservationsToDelete) + + // 6. Create and assign reservations for VMs that need them + assignSummary, hitMaxVMsLimit := c.reconcileCreateAndAssignReservations(ctx, vms, failoverReservations, allHypervisors) + summary.vmsProcessed = assignSummary.vmsProcessed + summary.reservationsNeeded = assignSummary.reservationsNeeded + summary.totalReused = assignSummary.totalReused + summary.totalCreated = assignSummary.totalCreated + summary.totalFailed = assignSummary.totalFailed + + // Log summary + duration := time.Since(startTime) + logger.Info("periodic reconciliation completed", + "reconcileCount", c.reconcileCount, + "duration", duration.Round(time.Millisecond), + "vmsProcessed", summary.vmsProcessed, + "reservationsNeeded", summary.reservationsNeeded, + "reused", summary.totalReused, + "created", summary.totalCreated, + "failed", summary.totalFailed, + "updated", summary.reservationsUpdated, + "deleted", summary.reservationsDeleted) + + if hitMaxVMsLimit && c.Config.ShortReconcileInterval > 0 { + logger.Info("requeuing with short interval due to MaxVMsToProcess limit", "shortReconcileInterval", c.Config.ShortReconcileInterval) + return ctrl.Result{RequeueAfter: c.Config.ShortReconcileInterval}, nil + } + + return ctrl.Result{RequeueAfter: c.Config.ReconcileInterval}, nil +} + +// reconcileRemoveInvalidVMFromReservations removes VMs from reservation allocations if: +// - The VM no longer exists +// - The VM has moved to a different host +// Returns the updated list of reservations (with modifications applied in-memory). +// The caller is responsible for persisting any changes to the cluster. +func reconcileRemoveInvalidVMFromReservations( + ctx context.Context, + vms []VM, + failoverReservations []v1alpha1.Reservation, +) (updatedReservations []v1alpha1.Reservation, reservationsToUpdate []*v1alpha1.Reservation) { + + logger := LoggerFromContext(ctx) + + vmToHypervisor := make(map[string]string) + for _, vm := range vms { + vmToHypervisor[vm.UUID] = vm.CurrentHypervisor + } + + updatedReservations = make([]v1alpha1.Reservation, 0, len(failoverReservations)) + + for _, res := range failoverReservations { + allocations := getFailoverAllocations(&res) + updatedAllocations := make(map[string]string) + needsUpdate := false + + for vmUUID, allocatedHypervisor := range allocations { + vmCurrentHypervisor, vmExists := vmToHypervisor[vmUUID] + if !vmExists { + logger.Info("removing VM from reservation allocations because VM no longer exists", + "vmUUID", vmUUID, "reservation", res.Name) + needsUpdate = true + continue + } + if vmCurrentHypervisor != allocatedHypervisor { + logger.Info("removing VM from reservation allocations because hypervisor has changed", + "vmUUID", vmUUID, "reservation", res.Name, + "allocatedHypervisor", allocatedHypervisor, "currentHypervisor", vmCurrentHypervisor) + needsUpdate = true + continue + } + updatedAllocations[vmUUID] = allocatedHypervisor + } + + if needsUpdate { + updatedRes := res.DeepCopy() + if updatedRes.Status.FailoverReservation == nil { + updatedRes.Status.FailoverReservation = &v1alpha1.FailoverReservationStatus{} + } + updatedRes.Status.FailoverReservation.Allocations = updatedAllocations + now := metav1.Now() + updatedRes.Status.FailoverReservation.LastChanged = &now + updatedRes.Status.FailoverReservation.AcknowledgedAt = nil + updatedReservations = append(updatedReservations, *updatedRes) + reservationsToUpdate = append(reservationsToUpdate, updatedRes) + } else { + updatedReservations = append(updatedReservations, res) + } + } + + return updatedReservations, reservationsToUpdate +} + +// reconcileRemoveNoneligibleVMFromReservations removes VMs from reservation allocations if +// they no longer meet eligibility criteria. +func reconcileRemoveNoneligibleVMFromReservations( + ctx context.Context, + vms []VM, + failoverReservations []v1alpha1.Reservation, +) (updatedReservations []v1alpha1.Reservation, reservationsToUpdate []*v1alpha1.Reservation) { + + logger := LoggerFromContext(ctx) + + vmByUUID := make(map[string]VM) + for _, vm := range vms { + vmByUUID[vm.UUID] = vm + } + + updatedReservations = make([]v1alpha1.Reservation, 0, len(failoverReservations)) + + for _, res := range failoverReservations { + allocations := getFailoverAllocations(&res) + updatedAllocations := make(map[string]string) + needsUpdate := false + + for vmUUID, allocatedHypervisor := range allocations { + vm, vmExists := vmByUUID[vmUUID] + if !vmExists { + updatedAllocations[vmUUID] = allocatedHypervisor + continue + } + + tempRes := res.DeepCopy() + delete(tempRes.Status.FailoverReservation.Allocations, vmUUID) + + tempReservations := make([]v1alpha1.Reservation, 0, len(failoverReservations)) + for _, r := range failoverReservations { + if r.Name == res.Name { + tempReservations = append(tempReservations, *tempRes) + } else { + tempReservations = append(tempReservations, r) + } + } + + if !IsVMEligibleForReservation(vm, *tempRes, tempReservations) { + logger.Info("removing VM from reservation allocations because it no longer meets eligibility criteria", + "vmUUID", vmUUID, "reservation", res.Name, + "vmHypervisor", vm.CurrentHypervisor, "reservationHypervisor", res.Status.Host) + needsUpdate = true + continue + } + updatedAllocations[vmUUID] = allocatedHypervisor + } + + if needsUpdate { + updatedRes := res.DeepCopy() + if updatedRes.Status.FailoverReservation == nil { + updatedRes.Status.FailoverReservation = &v1alpha1.FailoverReservationStatus{} + } + updatedRes.Status.FailoverReservation.Allocations = updatedAllocations + now := metav1.Now() + updatedRes.Status.FailoverReservation.LastChanged = &now + updatedRes.Status.FailoverReservation.AcknowledgedAt = nil + updatedReservations = append(updatedReservations, *updatedRes) + reservationsToUpdate = append(reservationsToUpdate, updatedRes) + } else { + updatedReservations = append(updatedReservations, res) + } + } + + return updatedReservations, reservationsToUpdate +} + +// reconcileRemoveEmptyReservations removes failover reservations that have no allocated VMs. +func reconcileRemoveEmptyReservations( + ctx context.Context, + failoverReservations []v1alpha1.Reservation, +) (updatedReservations []v1alpha1.Reservation, reservationsToDelete []*v1alpha1.Reservation) { + + logger := LoggerFromContext(ctx) + + updatedReservations = make([]v1alpha1.Reservation, 0, len(failoverReservations)) + + for _, res := range failoverReservations { + allocations := getFailoverAllocations(&res) + if len(allocations) == 0 { + resCopy := res.DeepCopy() + reservationsToDelete = append(reservationsToDelete, resCopy) + logger.Info("marking empty failover reservation for deletion", "reservationName", res.Name, "hypervisor", res.Status.Host) + } else { + updatedReservations = append(updatedReservations, res) + } + } + + return updatedReservations, reservationsToDelete +} + +// selectVMsToProcess selects a subset of VMs to process based on MaxVMsToProcess limit. +func (c *FailoverReservationController) selectVMsToProcess( + ctx context.Context, + vmsMissingFailover []vmFailoverNeed, + maxToProcess int, +) (selected []vmFailoverNeed, hitLimit bool) { + + logger := LoggerFromContext(ctx) + + if len(vmsMissingFailover) == 0 { + return vmsMissingFailover, false + } + + sortVMsByMemory(vmsMissingFailover) + + if maxToProcess <= 0 || len(vmsMissingFailover) <= maxToProcess { + return vmsMissingFailover, false + } + + offset := 0 + if c.reconcileCount%4 == 0 { + offset = int(c.reconcileCount) % len(vmsMissingFailover) + } + + selected = make([]vmFailoverNeed, 0, maxToProcess) + for i := range maxToProcess { + idx := (offset + i) % len(vmsMissingFailover) + selected = append(selected, vmsMissingFailover[idx]) + } + + logger.Info("selected VMs to process (sorted by memory, with rotation)", + "totalVMsMissingFailover", len(vmsMissingFailover), + "maxToProcess", maxToProcess, + "offset", offset, + "reconcileCount", c.reconcileCount) + + return selected, true +} + +// sortVMsByMemory sorts VMs by memory in descending order (largest first). +func sortVMsByMemory(vms []vmFailoverNeed) { + sort.Slice(vms, func(i, j int) bool { + var memI, memJ int64 + if mem, ok := vms[i].VM.Resources["memory"]; ok { + memI = mem.Value() + } + if mem, ok := vms[j].VM.Resources["memory"]; ok { + memJ = mem.Value() + } + return memI > memJ + }) +} + +// reconcileCreateAndAssignReservations creates and assigns failover reservations for VMs that need them. +func (c *FailoverReservationController) reconcileCreateAndAssignReservations( + ctx context.Context, + vms []VM, + failoverReservations []v1alpha1.Reservation, + allHypervisors []string, +) (reconcileSummary, bool) { + + logger := LoggerFromContext(ctx) + + vmsMissingFailover := c.calculateVMsMissingFailover(ctx, vms, failoverReservations) + logger.V(1).Info("VMs missing failover reservations", "count", len(vmsMissingFailover)) + + vmsMissingFailover, hitMaxVMsLimit := c.selectVMsToProcess(ctx, vmsMissingFailover, c.Config.MaxVMsToProcess) + + logger.V(1).Info("found hypervisors and vm missing failover reservation", + "countHypervisors", len(allHypervisors), + "countVMsMissingFailover", len(vmsMissingFailover)) + + totalReservationsNeeded := 0 + for _, need := range vmsMissingFailover { + totalReservationsNeeded += need.Count + } + + var totalReused, totalCreated, totalFailed int + + for _, need := range vmsMissingFailover { + vmReused := 0 + vmCreated := 0 + vmFailed := 0 + + reqID := uuid.New().String() + vmCtx := reservations.WithRequestID(ctx, reqID) + vmLogger := LoggerFromContext(vmCtx).WithValues("vmUUID", need.VM.UUID) + vmLogger.Info("processing VM for failover reservation") + + for i := range need.Count { + reusedRes := c.tryReuseExistingReservation(vmCtx, need.VM, failoverReservations, allHypervisors) + + if reusedRes != nil { + if err := c.patchReservationStatus(vmCtx, reusedRes); err != nil { + vmLogger.Error(err, "failed to persist reused reservation", "reservationName", reusedRes.Name) + vmFailed++ + continue + } + vmReused++ + for j := range failoverReservations { + if failoverReservations[j].Name == reusedRes.Name { + failoverReservations[j] = *reusedRes + break + } + } + continue + } + + newRes, err := c.scheduleAndBuildNewFailoverReservation(vmCtx, need.VM, allHypervisors, failoverReservations) + if err != nil { + vmLogger.V(1).Info("failed to schedule failover reservation", "error", err, "iteration", i+1, "needed", need.Count) + vmFailed++ + break + } + + savedStatus := newRes.Status.DeepCopy() + + if err := c.Create(vmCtx, newRes); err != nil { + vmLogger.Error(err, "failed to create failover reservation", "reservationName", newRes.Name) + vmFailed++ + break + } + + newRes.Status = *savedStatus + + if err := c.patchReservationStatus(vmCtx, newRes); err != nil { + vmLogger.V(1).Info("failed to update failover reservation status", "error", err, "reservationName", newRes.Name, "status", newRes.Status) + } else { + vmLogger.Info("successfully updated failover reservation status", + "reservationName", newRes.Name, + "host", newRes.Status.Host, + "allocations", newRes.Status.FailoverReservation.Allocations) + } + + vmCreated++ + failoverReservations = append(failoverReservations, *newRes) + } + + vmLogger.Info("processed VM failover reservations", + "flavorName", need.VM.FlavorName, + "needed", need.Count, + "reused", vmReused, + "created", vmCreated, + "failed", vmFailed) + + totalReused += vmReused + totalCreated += vmCreated + totalFailed += vmFailed + } + + return reconcileSummary{ + vmsProcessed: len(vmsMissingFailover), + reservationsNeeded: totalReservationsNeeded, + totalReused: totalReused, + totalCreated: totalCreated, + totalFailed: totalFailed, + }, hitMaxVMsLimit +} + +// calculateVMsMissingFailover calculates which VMs need failover reservations and how many. +func (c *FailoverReservationController) calculateVMsMissingFailover( + ctx context.Context, + vms []VM, + failoverReservations []v1alpha1.Reservation, +) []vmFailoverNeed { + + logger := LoggerFromContext(ctx) + + var result []vmFailoverNeed + totalReservationsNeeded := 0 + + for _, vm := range vms { + requiredCount := c.getRequiredFailoverCount(vm.FlavorName) + if requiredCount == 0 { + continue + } + + currentCount := countReservationsForVM(failoverReservations, vm.UUID) + + if currentCount >= requiredCount { + continue + } + + needed := requiredCount - currentCount + totalReservationsNeeded += needed + + logger.V(2).Info("VM needs more failover reservations", + "vmUUID", vm.UUID, + "flavorName", vm.FlavorName, + "currentCount", currentCount, + "requiredCount", requiredCount, + "needed", needed) + + result = append(result, vmFailoverNeed{ + VM: vm, + Count: needed, + }) + } + + if len(result) > 0 { + logger.Info("VMs missing failover reservations summary", + "vmCount", len(result), + "totalReservationsNeeded", totalReservationsNeeded) + } + + return result +} + +// getRequiredFailoverCount returns the number of failover reservations required for a flavor. +// It supports glob patterns in the configuration. +// If multiple patterns match, the pattern with the highest count is used. +func (c *FailoverReservationController) getRequiredFailoverCount(flavorName string) int { + if flavorName == "" { + return 0 + } + + maxCount := 0 + for pattern, count := range c.Config.FlavorFailoverRequirements { + matched, err := filepath.Match(pattern, flavorName) + if err != nil { + log.Error(err, "invalid pattern in FlavorFailoverRequirements", "pattern", pattern) + continue + } + if matched && count > maxCount { + maxCount = count + } + } + return maxCount +} + +// patchReservationStatus patches the status of a reservation using MergeFrom. +func (c *FailoverReservationController) patchReservationStatus(ctx context.Context, res *v1alpha1.Reservation) error { + logger := LoggerFromContext(ctx).WithValues("reservationName", res.Name) + + current := &v1alpha1.Reservation{} + if err := c.Get(ctx, client.ObjectKeyFromObject(res), current); err != nil { + logger.Error(err, "failed to get current reservation state") + return fmt.Errorf("failed to get current reservation state: %w", err) + } + + old := current.DeepCopy() + current.Status = res.Status + + patch := client.MergeFrom(old) + if err := c.Status().Patch(ctx, current, patch); err != nil { + logger.Error(err, "failed to patch reservation status") + return fmt.Errorf("failed to patch reservation status: %w", err) + } + + return nil +} + +// ============================================================================ +// Manager Setup +// ============================================================================ + +// SetupWithManager sets up the watch-based reconciler with the Manager. +// This handles per-reservation reconciliation triggered by CRD changes. +func (c *FailoverReservationController) SetupWithManager(mgr ctrl.Manager, mcl *multicluster.Client) error { + c.Recorder = mgr.GetEventRecorder("failover-reservation-controller") + + return multicluster.BuildController(mcl, mgr). + For(&v1alpha1.Reservation{}). + WithEventFilter(failoverReservationPredicate). + Named("failover-reservation"). + WithOptions(controller.Options{ + MaxConcurrentReconciles: 1, + }). + Complete(c) +} + +// Start implements manager.Runnable. +// It runs the periodic reconciliation loop at the configured interval. +// This can be called directly when the controller is created after the manager starts. +func (c *FailoverReservationController) Start(ctx context.Context) error { + log.Info("starting failover reservation controller (periodic)", + "reconcileInterval", c.Config.ReconcileInterval, + "shortReconcileInterval", c.Config.ShortReconcileInterval, + "creator", c.Config.Creator, + "datasourceName", c.Config.DatasourceName, + "schedulerURL", c.Config.SchedulerURL, + "flavorFailoverRequirements", c.Config.FlavorFailoverRequirements, + "maxVMsToProcess", c.Config.MaxVMsToProcess) + + timer := time.NewTimer(c.Config.ReconcileInterval) + defer timer.Stop() + + for { + select { + case <-ctx.Done(): + log.Info("stopping failover reservation controller") + return nil + case <-timer.C: + result, err := c.ReconcilePeriodic(ctx) + if err != nil { + log.Error(err, "failover reconciliation failed") + } + next := c.Config.ReconcileInterval + if result.RequeueAfter > 0 { + next = result.RequeueAfter + } + timer.Reset(next) + } + } +} + +// failoverReservationPredicate filters to only process failover reservations. +var failoverReservationPredicate = predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + res, ok := e.Object.(*v1alpha1.Reservation) + return ok && res.Spec.Type == v1alpha1.ReservationTypeFailover + }, + UpdateFunc: func(e event.UpdateEvent) bool { + res, ok := e.ObjectNew.(*v1alpha1.Reservation) + return ok && res.Spec.Type == v1alpha1.ReservationTypeFailover + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return false + }, + GenericFunc: func(e event.GenericEvent) bool { + res, ok := e.Object.(*v1alpha1.Reservation) + return ok && res.Spec.Type == v1alpha1.ReservationTypeFailover + }, +} diff --git a/internal/scheduling/reservations/failover/controller_test.go b/internal/scheduling/reservations/failover/controller_test.go new file mode 100644 index 000000000..19982d405 --- /dev/null +++ b/internal/scheduling/reservations/failover/controller_test.go @@ -0,0 +1,1035 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failover + +import ( + "context" + "testing" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ============================================================================ +// Test: reconcileRemoveNoneligibleVMFromReservations +// ============================================================================ + +func TestReconcileRemoveNoneligibleVMFromReservations(t *testing.T) { + tests := []struct { + name string + vms []VM + reservations []v1alpha1.Reservation + expectedUpdatedCount int + expectedToUpdateCount int + expectedAllocationsPerRes map[string]map[string]string + }{ + { + name: "no changes needed - all VMs eligible", + vms: []VM{ + newTestVMWithResources("vm-1", "host1"), + newTestVMWithResources("vm-2", "host2"), + }, + reservations: []v1alpha1.Reservation{ + newTestReservationWithResources("res-1", "host3", map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + }), + }, + expectedUpdatedCount: 1, + expectedToUpdateCount: 0, + expectedAllocationsPerRes: map[string]map[string]string{ + "res-1": {"vm-1": "host1", "vm-2": "host2"}, + }, + }, + { + name: "VM on same host as reservation - remove", + vms: []VM{ + newTestVMWithResources("vm-1", "host3"), // VM moved to host3 (same as reservation) + }, + reservations: []v1alpha1.Reservation{ + newTestReservationWithResources("res-1", "host3", map[string]string{ + "vm-1": "host1", // allocation says host1, but VM is now on host3 + }), + }, + expectedUpdatedCount: 1, + expectedToUpdateCount: 1, + expectedAllocationsPerRes: map[string]map[string]string{ + "res-1": {}, // vm-1 removed because it's on the same host as reservation + }, + }, + { + name: "multiple ineligible VMs - all processed", + vms: []VM{ + newTestVMWithResources("vm-1", "host3"), // ineligible (on same host as res-1) + newTestVMWithResources("vm-2", "host4"), // ineligible (on same host as res-2) + }, + reservations: []v1alpha1.Reservation{ + newTestReservationWithResources("res-1", "host3", map[string]string{ + "vm-1": "host1", + }), + newTestReservationWithResources("res-2", "host4", map[string]string{ + "vm-2": "host2", + }), + }, + expectedUpdatedCount: 2, + expectedToUpdateCount: 2, // Both reservations updated + expectedAllocationsPerRes: map[string]map[string]string{ + "res-1": {}, // vm-1 removed + "res-2": {}, // vm-2 removed + }, + }, + { + name: "VM not in list - keep in allocations", + vms: []VM{ + newTestVMWithResources("vm-1", "host1"), + // vm-2 not in list + }, + reservations: []v1alpha1.Reservation{ + newTestReservationWithResources("res-1", "host3", map[string]string{ + "vm-1": "host1", + "vm-2": "host2", // vm-2 not in VMs list - handled by reconcileRemoveInvalidVMFromReservations + }), + }, + expectedUpdatedCount: 1, + expectedToUpdateCount: 0, + expectedAllocationsPerRes: map[string]map[string]string{ + "res-1": {"vm-1": "host1", "vm-2": "host2"}, // vm-2 kept (not our responsibility) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + updatedReservations, reservationsToUpdate := reconcileRemoveNoneligibleVMFromReservations( + ctx, + tt.vms, + tt.reservations, + ) + + if len(updatedReservations) != tt.expectedUpdatedCount { + t.Errorf("expected %d updated reservations, got %d", + tt.expectedUpdatedCount, len(updatedReservations)) + } + + if len(reservationsToUpdate) != tt.expectedToUpdateCount { + t.Errorf("expected %d reservations to update, got %d", + tt.expectedToUpdateCount, len(reservationsToUpdate)) + } + + for _, res := range updatedReservations { + expectedAllocs, ok := tt.expectedAllocationsPerRes[res.Name] + if !ok { + t.Errorf("unexpected reservation %s in result", res.Name) + continue + } + + actualAllocs := getAllocations(&res) + if len(actualAllocs) != len(expectedAllocs) { + t.Errorf("reservation %s: expected %d allocations, got %d (%v)", + res.Name, len(expectedAllocs), len(actualAllocs), actualAllocs) + continue + } + + for vmUUID, expectedHost := range expectedAllocs { + actualHost, exists := actualAllocs[vmUUID] + if !exists { + t.Errorf("reservation %s: expected VM %s in allocations, but not found", + res.Name, vmUUID) + continue + } + if actualHost != expectedHost { + t.Errorf("reservation %s: VM %s expected host %s, got %s", + res.Name, vmUUID, expectedHost, actualHost) + } + } + } + }) + } +} + +// ============================================================================ +// Test: filterFailoverReservations +// ============================================================================ + +func TestFilterFailoverReservations(t *testing.T) { + tests := []struct { + name string + reservations []v1alpha1.Reservation + expectedCount int + expectedNames []string + }{ + { + name: "empty list", + reservations: []v1alpha1.Reservation{}, + expectedCount: 0, + expectedNames: nil, + }, + { + name: "all failover reservations", + reservations: []v1alpha1.Reservation{ + {ObjectMeta: metav1.ObjectMeta{Name: "res-1"}, Spec: v1alpha1.ReservationSpec{Type: v1alpha1.ReservationTypeFailover}}, + {ObjectMeta: metav1.ObjectMeta{Name: "res-2"}, Spec: v1alpha1.ReservationSpec{Type: v1alpha1.ReservationTypeFailover}}, + }, + expectedCount: 2, + expectedNames: []string{"res-1", "res-2"}, + }, + { + name: "mixed types - only failover returned", + reservations: []v1alpha1.Reservation{ + {ObjectMeta: metav1.ObjectMeta{Name: "res-1"}, Spec: v1alpha1.ReservationSpec{Type: v1alpha1.ReservationTypeFailover}}, + {ObjectMeta: metav1.ObjectMeta{Name: "res-2"}, Spec: v1alpha1.ReservationSpec{Type: "committed"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "res-3"}, Spec: v1alpha1.ReservationSpec{Type: v1alpha1.ReservationTypeFailover}}, + }, + expectedCount: 2, + expectedNames: []string{"res-1", "res-3"}, + }, + { + name: "no failover reservations", + reservations: []v1alpha1.Reservation{ + {ObjectMeta: metav1.ObjectMeta{Name: "res-1"}, Spec: v1alpha1.ReservationSpec{Type: "committed"}}, + }, + expectedCount: 0, + expectedNames: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := filterFailoverReservations(tt.reservations) + + if len(result) != tt.expectedCount { + t.Errorf("expected %d reservations, got %d", tt.expectedCount, len(result)) + } + + for i, name := range tt.expectedNames { + if i >= len(result) { + t.Errorf("missing expected reservation %s", name) + continue + } + if result[i].Name != name { + t.Errorf("expected reservation %s at index %d, got %s", name, i, result[i].Name) + } + } + }) + } +} + +// ============================================================================ +// Test: filterVMsOnKnownHypervisors +// ============================================================================ + +func TestFilterVMsOnKnownHypervisors(t *testing.T) { + tests := []struct { + name string + vms []VM + hypervisorList *hv1.HypervisorList + expectedCount int + expectedUUIDs []string + }{ + { + name: "empty VMs list", + vms: []VM{}, + hypervisorList: &hv1.HypervisorList{ + Items: []hv1.Hypervisor{ + newTestHypervisor("host1", []hv1.Instance{}), + newTestHypervisor("host2", []hv1.Instance{}), + }, + }, + expectedCount: 0, + expectedUUIDs: nil, + }, + { + name: "all VMs on known hypervisors and in instances", + vms: []VM{ + newTestVM("vm-1", "host1", "m1.large"), + newTestVM("vm-2", "host2", "m1.large"), + }, + hypervisorList: &hv1.HypervisorList{ + Items: []hv1.Hypervisor{ + newTestHypervisor("host1", []hv1.Instance{{ID: "vm-1", Name: "vm-1", Active: true}}), + newTestHypervisor("host2", []hv1.Instance{{ID: "vm-2", Name: "vm-2", Active: true}}), + newTestHypervisor("host3", []hv1.Instance{}), + }, + }, + expectedCount: 2, + expectedUUIDs: []string{"vm-1", "vm-2"}, + }, + { + name: "some VMs on unknown hypervisors", + vms: []VM{ + newTestVM("vm-1", "host1", "m1.large"), + newTestVM("vm-2", "unknown-host", "m1.large"), + newTestVM("vm-3", "host2", "m1.large"), + }, + hypervisorList: &hv1.HypervisorList{ + Items: []hv1.Hypervisor{ + newTestHypervisor("host1", []hv1.Instance{{ID: "vm-1", Name: "vm-1", Active: true}}), + newTestHypervisor("host2", []hv1.Instance{{ID: "vm-3", Name: "vm-3", Active: true}}), + }, + }, + expectedCount: 2, + expectedUUIDs: []string{"vm-1", "vm-3"}, + }, + { + name: "VM claims hypervisor but not in instances list - filter out", + vms: []VM{ + newTestVM("vm-1", "host1", "m1.large"), + newTestVM("vm-2", "host2", "m1.large"), // claims host2 but not in instances + }, + hypervisorList: &hv1.HypervisorList{ + Items: []hv1.Hypervisor{ + newTestHypervisor("host1", []hv1.Instance{{ID: "vm-1", Name: "vm-1", Active: true}}), + newTestHypervisor("host2", []hv1.Instance{}), // vm-2 not in instances + }, + }, + expectedCount: 1, + expectedUUIDs: []string{"vm-1"}, + }, + { + name: "VM on wrong hypervisor in instances - filter out", + vms: []VM{ + newTestVM("vm-1", "host1", "m1.large"), // claims host1 + }, + hypervisorList: &hv1.HypervisorList{ + Items: []hv1.Hypervisor{ + newTestHypervisor("host1", []hv1.Instance{}), // vm-1 not here + newTestHypervisor("host2", []hv1.Instance{{ID: "vm-1", Name: "vm-1", Active: true}}), // vm-1 is actually here + }, + }, + expectedCount: 0, + expectedUUIDs: nil, + }, + { + name: "inactive VM in instances - filter out", + vms: []VM{ + newTestVM("vm-1", "host1", "m1.large"), + }, + hypervisorList: &hv1.HypervisorList{ + Items: []hv1.Hypervisor{ + newTestHypervisor("host1", []hv1.Instance{{ID: "vm-1", Name: "vm-1", Active: false}}), // inactive + }, + }, + expectedCount: 0, + expectedUUIDs: nil, + }, + { + name: "no VMs on known hypervisors", + vms: []VM{ + newTestVM("vm-1", "unknown1", "m1.large"), + newTestVM("vm-2", "unknown2", "m1.large"), + }, + hypervisorList: &hv1.HypervisorList{ + Items: []hv1.Hypervisor{ + newTestHypervisor("host1", []hv1.Instance{}), + newTestHypervisor("host2", []hv1.Instance{}), + }, + }, + expectedCount: 0, + expectedUUIDs: nil, + }, + { + name: "empty hypervisors list", + vms: []VM{ + newTestVM("vm-1", "host1", "m1.large"), + }, + hypervisorList: &hv1.HypervisorList{ + Items: []hv1.Hypervisor{}, + }, + expectedCount: 0, + expectedUUIDs: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := filterVMsOnKnownHypervisors(tt.vms, tt.hypervisorList) + + if len(result) != tt.expectedCount { + t.Errorf("expected %d VMs, got %d", tt.expectedCount, len(result)) + } + + // Check that expected UUIDs are present (order may vary due to filtering) + resultUUIDs := make(map[string]bool) + for _, vm := range result { + resultUUIDs[vm.UUID] = true + } + + for _, uuid := range tt.expectedUUIDs { + if !resultUUIDs[uuid] { + t.Errorf("expected VM %s in result, but not found", uuid) + } + } + }) + } +} + +// newTestHypervisor creates a test hypervisor with the given instances +func newTestHypervisor(name string, instances []hv1.Instance) hv1.Hypervisor { + return hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Status: hv1.HypervisorStatus{ + Instances: instances, + }, + } +} + +// ============================================================================ +// Test: countReservationsForVM +// ============================================================================ + +func TestCountReservationsForVM(t *testing.T) { + tests := []struct { + name string + reservations []v1alpha1.Reservation + vmUUID string + expectedCount int + }{ + { + name: "empty reservations list", + reservations: []v1alpha1.Reservation{}, + vmUUID: "vm-1", + expectedCount: 0, + }, + { + name: "VM in one reservation", + reservations: []v1alpha1.Reservation{ + newTestReservation("res-1", "host1", map[string]string{"vm-1": "host2"}), + newTestReservation("res-2", "host3", map[string]string{"vm-2": "host4"}), + }, + vmUUID: "vm-1", + expectedCount: 1, + }, + { + name: "VM in multiple reservations", + reservations: []v1alpha1.Reservation{ + newTestReservation("res-1", "host1", map[string]string{"vm-1": "host2"}), + newTestReservation("res-2", "host3", map[string]string{"vm-1": "host2", "vm-2": "host4"}), + newTestReservation("res-3", "host5", map[string]string{"vm-1": "host2"}), + }, + vmUUID: "vm-1", + expectedCount: 3, + }, + { + name: "VM not in any reservation", + reservations: []v1alpha1.Reservation{ + newTestReservation("res-1", "host1", map[string]string{"vm-2": "host2"}), + newTestReservation("res-2", "host3", map[string]string{"vm-3": "host4"}), + }, + vmUUID: "vm-1", + expectedCount: 0, + }, + { + name: "reservation with nil allocations", + reservations: []v1alpha1.Reservation{ + { + ObjectMeta: metav1.ObjectMeta{Name: "res-1"}, + Spec: v1alpha1.ReservationSpec{Type: v1alpha1.ReservationTypeFailover}, + Status: v1alpha1.ReservationStatus{Host: "host1"}, + }, + }, + vmUUID: "vm-1", + expectedCount: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := countReservationsForVM(tt.reservations, tt.vmUUID) + + if result != tt.expectedCount { + t.Errorf("expected count %d, got %d", tt.expectedCount, result) + } + }) + } +} + +// ============================================================================ +// Test: getRequiredFailoverCount +// ============================================================================ + +func TestGetRequiredFailoverCount(t *testing.T) { + tests := []struct { + name string + config FailoverConfig + flavorName string + expectedCount int + }{ + { + name: "exact match", + config: FailoverConfig{ + FlavorFailoverRequirements: map[string]int{ + "m1.large": 2, + }, + }, + flavorName: "m1.large", + expectedCount: 2, + }, + { + name: "glob pattern match - prefix", + config: FailoverConfig{ + FlavorFailoverRequirements: map[string]int{ + "m1.*": 1, + }, + }, + flavorName: "m1.large", + expectedCount: 1, + }, + { + name: "pattern match - suffix", + config: FailoverConfig{ + FlavorFailoverRequirements: map[string]int{ + "*": 1, + "*hana": 3, + }, + }, + flavorName: "m1.hana", + expectedCount: 3, + }, + { + name: "pattern match other sorting - suffix", + config: FailoverConfig{ + FlavorFailoverRequirements: map[string]int{ + "*hana": 3, + "*": 1, + }, + }, + flavorName: "m1.hana", + expectedCount: 3, + }, + { + name: "glob pattern match - wildcard", + config: FailoverConfig{ + FlavorFailoverRequirements: map[string]int{ + "*": 1, + }, + }, + flavorName: "any-flavor", + expectedCount: 1, + }, + { + name: "no match", + config: FailoverConfig{ + FlavorFailoverRequirements: map[string]int{ + "m1.large": 2, + }, + }, + flavorName: "m2.small", + expectedCount: 0, + }, + { + name: "empty flavor name", + config: FailoverConfig{ + FlavorFailoverRequirements: map[string]int{ + "*": 1, + }, + }, + flavorName: "", + expectedCount: 0, + }, + { + name: "empty requirements", + config: FailoverConfig{ + FlavorFailoverRequirements: map[string]int{}, + }, + flavorName: "m1.large", + expectedCount: 0, + }, + { + name: "multiple patterns - highest count wins", + config: FailoverConfig{ + FlavorFailoverRequirements: map[string]int{ + "m1.large": 5, + "m1.*": 2, + "*": 1, + }, + }, + flavorName: "m1.large", + expectedCount: 5, // Note: map iteration order is not guaranteed, but exact match should be found + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + controller := &FailoverReservationController{ + Config: tt.config, + } + + result := controller.getRequiredFailoverCount(tt.flavorName) + + if result != tt.expectedCount { + t.Errorf("expected count %d, got %d", tt.expectedCount, result) + } + }) + } +} + +// ============================================================================ +// Test: reconcileRemoveInvalidVMFromReservations +// ============================================================================ + +func TestReconcileRemoveInvalidVMFromReservations(t *testing.T) { + tests := []struct { + name string + vms []VM + reservations []v1alpha1.Reservation + expectedUpdatedCount int // number of reservations in updatedReservations + expectedToUpdateCount int // number of reservations that need cluster update + expectedAllocationsPerRes map[string]map[string]string + }{ + { + name: "no changes needed - all VMs valid", + vms: []VM{ + newTestVM("vm-1", "host1", "flavor1"), + newTestVM("vm-2", "host2", "flavor1"), + }, + reservations: []v1alpha1.Reservation{ + newTestReservation("res-1", "host3", map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + }), + }, + expectedUpdatedCount: 1, + expectedToUpdateCount: 0, + expectedAllocationsPerRes: map[string]map[string]string{ + "res-1": {"vm-1": "host1", "vm-2": "host2"}, + }, + }, + { + name: "VM no longer exists - remove from allocations", + vms: []VM{ + newTestVM("vm-1", "host1", "flavor1"), + // vm-2 no longer exists + }, + reservations: []v1alpha1.Reservation{ + newTestReservation("res-1", "host3", map[string]string{ + "vm-1": "host1", + "vm-2": "host2", // vm-2 should be removed + }), + }, + expectedUpdatedCount: 1, + expectedToUpdateCount: 1, + expectedAllocationsPerRes: map[string]map[string]string{ + "res-1": {"vm-1": "host1"}, // vm-2 removed + }, + }, + { + name: "VM moved to different host - remove from allocations", + vms: []VM{ + newTestVM("vm-1", "host1", "flavor1"), + newTestVM("vm-2", "host4", "flavor1"), // moved from host2 to host4 + }, + reservations: []v1alpha1.Reservation{ + newTestReservation("res-1", "host3", map[string]string{ + "vm-1": "host1", + "vm-2": "host2", // vm-2 moved, should be removed + }), + }, + expectedUpdatedCount: 1, + expectedToUpdateCount: 1, + expectedAllocationsPerRes: map[string]map[string]string{ + "res-1": {"vm-1": "host1"}, // vm-2 removed because it moved + }, + }, + { + name: "multiple reservations - only affected ones updated", + vms: []VM{ + newTestVM("vm-1", "host1", "flavor1"), + newTestVM("vm-2", "host2", "flavor1"), + // vm-3 no longer exists + }, + reservations: []v1alpha1.Reservation{ + newTestReservation("res-1", "host3", map[string]string{ + "vm-1": "host1", + }), + newTestReservation("res-2", "host4", map[string]string{ + "vm-2": "host2", + "vm-3": "host3", // vm-3 should be removed + }), + }, + expectedUpdatedCount: 2, + expectedToUpdateCount: 1, // only res-2 needs update + expectedAllocationsPerRes: map[string]map[string]string{ + "res-1": {"vm-1": "host1"}, + "res-2": {"vm-2": "host2"}, // vm-3 removed + }, + }, + { + name: "all VMs removed from reservation - empty allocations", + vms: []VM{ + // no VMs exist + }, + reservations: []v1alpha1.Reservation{ + newTestReservation("res-1", "host3", map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + }), + }, + expectedUpdatedCount: 1, + expectedToUpdateCount: 1, + expectedAllocationsPerRes: map[string]map[string]string{ + "res-1": {}, // all VMs removed + }, + }, + { + name: "empty reservations list", + vms: []VM{ + newTestVM("vm-1", "host1", "flavor1"), + }, + reservations: []v1alpha1.Reservation{}, + expectedUpdatedCount: 0, + expectedToUpdateCount: 0, + expectedAllocationsPerRes: map[string]map[string]string{}, + }, + { + name: "reservation with no allocations - no changes", + vms: []VM{ + newTestVM("vm-1", "host1", "flavor1"), + }, + reservations: []v1alpha1.Reservation{ + newTestReservation("res-1", "host3", map[string]string{}), + }, + expectedUpdatedCount: 1, + expectedToUpdateCount: 0, + expectedAllocationsPerRes: map[string]map[string]string{ + "res-1": {}, + }, + }, + { + name: "mixed scenario - some VMs valid, some deleted, some moved", + vms: []VM{ + newTestVM("vm-1", "host1", "flavor1"), // valid + newTestVM("vm-2", "host5", "flavor1"), // moved from host2 to host5 + // vm-3 deleted + newTestVM("vm-4", "host4", "flavor1"), // valid + }, + reservations: []v1alpha1.Reservation{ + newTestReservation("res-1", "host6", map[string]string{ + "vm-1": "host1", // valid + "vm-2": "host2", // moved - remove + }), + newTestReservation("res-2", "host7", map[string]string{ + "vm-3": "host3", // deleted - remove + "vm-4": "host4", // valid + }), + }, + expectedUpdatedCount: 2, + expectedToUpdateCount: 2, // both need update + expectedAllocationsPerRes: map[string]map[string]string{ + "res-1": {"vm-1": "host1"}, + "res-2": {"vm-4": "host4"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + updatedReservations, reservationsToUpdate := reconcileRemoveInvalidVMFromReservations( + ctx, + tt.vms, + tt.reservations, + ) + + // Check count of updated reservations + if len(updatedReservations) != tt.expectedUpdatedCount { + t.Errorf("expected %d updated reservations, got %d", + tt.expectedUpdatedCount, len(updatedReservations)) + } + + // Check count of reservations that need cluster update + if len(reservationsToUpdate) != tt.expectedToUpdateCount { + t.Errorf("expected %d reservations to update, got %d", + tt.expectedToUpdateCount, len(reservationsToUpdate)) + } + + // Check allocations for each reservation + for _, res := range updatedReservations { + expectedAllocs, ok := tt.expectedAllocationsPerRes[res.Name] + if !ok { + t.Errorf("unexpected reservation %s in result", res.Name) + continue + } + + actualAllocs := getAllocations(&res) + if len(actualAllocs) != len(expectedAllocs) { + t.Errorf("reservation %s: expected %d allocations, got %d", + res.Name, len(expectedAllocs), len(actualAllocs)) + continue + } + + for vmUUID, expectedHost := range expectedAllocs { + actualHost, exists := actualAllocs[vmUUID] + if !exists { + t.Errorf("reservation %s: expected VM %s in allocations, but not found", + res.Name, vmUUID) + continue + } + if actualHost != expectedHost { + t.Errorf("reservation %s: VM %s expected host %s, got %s", + res.Name, vmUUID, expectedHost, actualHost) + } + } + } + }) + } +} + +// Test helper functions - local to this test file + +func newTestVM(uuid, currentHypervisor, flavorName string) VM { + return VM{ + UUID: uuid, + CurrentHypervisor: currentHypervisor, + FlavorName: flavorName, + ProjectID: "test-project", + } +} + +func newTestVMWithResources(uuid, currentHypervisor string) VM { + return VM{ + UUID: uuid, + CurrentHypervisor: currentHypervisor, + FlavorName: "m1.large", + ProjectID: "test-project", + Resources: map[string]resource.Quantity{ + "memory": *resource.NewQuantity(8192*1024*1024, resource.BinarySI), + "vcpus": *resource.NewQuantity(4, resource.DecimalSI), + }, + } +} + +func newTestReservation(name, host string, allocations map[string]string) v1alpha1.Reservation { + return v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailover, + }, + Status: v1alpha1.ReservationStatus{ + Host: host, + FailoverReservation: &v1alpha1.FailoverReservationStatus{ + Allocations: allocations, + }, + }, + } +} + +func newTestReservationWithResources(name, host string, allocations map[string]string) v1alpha1.Reservation { + return v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailover, + TargetHost: host, + Resources: map[hv1.ResourceName]resource.Quantity{ + "memory": *resource.NewQuantity(8192*1024*1024, resource.BinarySI), + "cpu": *resource.NewQuantity(4, resource.DecimalSI), + }, + }, + Status: v1alpha1.ReservationStatus{ + Host: host, + FailoverReservation: &v1alpha1.FailoverReservationStatus{ + Allocations: allocations, + }, + }, + } +} + +func getAllocations(res *v1alpha1.Reservation) map[string]string { + if res.Status.FailoverReservation == nil { + return nil + } + return res.Status.FailoverReservation.Allocations +} + +// ============================================================================ +// Test: selectVMsToProcess +// ============================================================================ + +func TestSelectVMsToProcess(t *testing.T) { + // Create 10 VMs with different memory sizes (sorted by memory descending) + createVMs := func(count int) []vmFailoverNeed { + vms := make([]vmFailoverNeed, count) + for i := range count { + vms[i] = vmFailoverNeed{ + VM: VM{ + UUID: "vm-" + string(rune('a'+i)), + CurrentHypervisor: "host" + string(rune('1'+i)), + Resources: map[string]resource.Quantity{ + "memory": *resource.NewQuantity(int64((count-i)*1024*1024*1024), resource.BinarySI), // Descending memory + }, + }, + Count: 1, + } + } + return vms + } + + tests := []struct { + name string + reconcileCount int64 + vmCount int + maxToProcess int + expectedOffset int // Expected starting offset in the VM list + expectedHit bool + }{ + // 3 out of 4 runs should start at offset 0 + { + name: "reconcile 1 - offset 0", + reconcileCount: 1, + vmCount: 10, + maxToProcess: 3, + expectedOffset: 0, + expectedHit: true, + }, + { + name: "reconcile 2 - offset 0", + reconcileCount: 2, + vmCount: 10, + maxToProcess: 3, + expectedOffset: 0, + expectedHit: true, + }, + { + name: "reconcile 3 - offset 0", + reconcileCount: 3, + vmCount: 10, + maxToProcess: 3, + expectedOffset: 0, + expectedHit: true, + }, + // Every 4th reconcile uses reconcileCount as offset (mod vmCount) + { + name: "reconcile 4 - offset 4", + reconcileCount: 4, + vmCount: 10, + maxToProcess: 3, + expectedOffset: 4, + expectedHit: true, + }, + { + name: "reconcile 5 - offset 0", + reconcileCount: 5, + vmCount: 10, + maxToProcess: 3, + expectedOffset: 0, + expectedHit: true, + }, + { + name: "reconcile 6 - offset 0", + reconcileCount: 6, + vmCount: 10, + maxToProcess: 3, + expectedOffset: 0, + expectedHit: true, + }, + { + name: "reconcile 7 - offset 0", + reconcileCount: 7, + vmCount: 10, + maxToProcess: 3, + expectedOffset: 0, + expectedHit: true, + }, + { + name: "reconcile 8 - offset 8", + reconcileCount: 8, + vmCount: 10, + maxToProcess: 3, + expectedOffset: 8, + expectedHit: true, + }, + // Test wrap-around when reconcileCount > vmCount + { + name: "reconcile 12 - offset 2 (12 mod 10)", + reconcileCount: 12, + vmCount: 10, + maxToProcess: 3, + expectedOffset: 2, // 12 % 10 = 2 + expectedHit: true, + }, + { + name: "reconcile 20 - offset 0 (20 mod 10)", + reconcileCount: 20, + vmCount: 10, + maxToProcess: 3, + expectedOffset: 0, // 20 % 10 = 0 + expectedHit: true, + }, + // Edge cases + { + name: "maxToProcess 0 - no limit, returns all", + reconcileCount: 4, + vmCount: 10, + maxToProcess: 0, + expectedOffset: 0, // No limit means all VMs returned starting from 0 + expectedHit: false, + }, + { + name: "maxToProcess >= vmCount - no limit hit", + reconcileCount: 4, + vmCount: 5, + maxToProcess: 10, + expectedOffset: 0, // All VMs fit, no rotation needed + expectedHit: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + controller := &FailoverReservationController{ + reconcileCount: tt.reconcileCount, + } + + vms := createVMs(tt.vmCount) + selected, hitLimit := controller.selectVMsToProcess(ctx, vms, tt.maxToProcess) + + if hitLimit != tt.expectedHit { + t.Errorf("expected hitLimit=%v, got %v", tt.expectedHit, hitLimit) + } + + if !tt.expectedHit { + // When no limit is hit, all VMs should be returned + if len(selected) != tt.vmCount { + t.Errorf("expected all %d VMs when no limit hit, got %d", tt.vmCount, len(selected)) + } + return + } + + // Verify the first selected VM is at the expected offset + if len(selected) == 0 { + t.Error("expected at least one VM selected") + return + } + + // The VMs are sorted by memory descending, so vm-a has most memory, vm-j has least + // After sorting, the order is: vm-a, vm-b, vm-c, ..., vm-j + // With offset, we should start at vms[offset] + expectedFirstVM := vms[tt.expectedOffset].VM.UUID + actualFirstVM := selected[0].VM.UUID + + if actualFirstVM != expectedFirstVM { + t.Errorf("expected first VM to be %s (offset %d), got %s", + expectedFirstVM, tt.expectedOffset, actualFirstVM) + } + + // Verify we got the expected number of VMs + expectedCount := tt.maxToProcess + if expectedCount > tt.vmCount { + expectedCount = tt.vmCount + } + if len(selected) != expectedCount { + t.Errorf("expected %d VMs selected, got %d", expectedCount, len(selected)) + } + }) + } +} diff --git a/internal/scheduling/reservations/failover/helpers.go b/internal/scheduling/reservations/failover/helpers.go new file mode 100644 index 000000000..f7efa135d --- /dev/null +++ b/internal/scheduling/reservations/failover/helpers.go @@ -0,0 +1,156 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failover + +import ( + "context" + "fmt" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// getFailoverAllocations safely returns the allocations map from a failover reservation. +// Returns an empty map if the reservation has no failover status or allocations. +func getFailoverAllocations(res *v1alpha1.Reservation) map[string]string { + if res.Status.FailoverReservation == nil || res.Status.FailoverReservation.Allocations == nil { + return map[string]string{} + } + return res.Status.FailoverReservation.Allocations +} + +// filterFailoverReservations filters a list of reservations to only include failover reservations. +func filterFailoverReservations(resList []v1alpha1.Reservation) []v1alpha1.Reservation { + var result []v1alpha1.Reservation + for _, res := range resList { + if res.Spec.Type == v1alpha1.ReservationTypeFailover { + result = append(result, res) + } + } + return result +} + +// countReservationsForVM counts how many reservations a VM is in. +func countReservationsForVM(resList []v1alpha1.Reservation, vmUUID string) int { + count := 0 + for _, res := range resList { + allocations := getFailoverAllocations(&res) + if _, exists := allocations[vmUUID]; exists { + count++ + } + } + return count +} + +// addVMToReservation creates a copy of a reservation with the VM added to its allocations. +// The original reservation is NOT modified. +func addVMToReservation(reservation v1alpha1.Reservation, vm VM) *v1alpha1.Reservation { + // Deep copy the reservation + updatedRes := reservation.DeepCopy() + + // Initialize the FailoverReservation status if needed + if updatedRes.Status.FailoverReservation == nil { + updatedRes.Status.FailoverReservation = &v1alpha1.FailoverReservationStatus{} + } + // Initialize the Allocations map if needed + if updatedRes.Status.FailoverReservation.Allocations == nil { + updatedRes.Status.FailoverReservation.Allocations = make(map[string]string) + } + // Add the VM to the allocations + updatedRes.Status.FailoverReservation.Allocations[vm.UUID] = vm.CurrentHypervisor + + // Mark the reservation as changed and not yet acknowledged + now := metav1.Now() + updatedRes.Status.FailoverReservation.LastChanged = &now + updatedRes.Status.FailoverReservation.AcknowledgedAt = nil + + return updatedRes +} + +// ValidateFailoverReservationResources validates that a failover reservation has valid resource keys. +// Returns an error if the reservation has invalid resource keys (only "cpu" and "memory" are allowed). +// This ensures reservations are properly considered by the scheduling filters. +func ValidateFailoverReservationResources(res *v1alpha1.Reservation) error { + if res.Spec.Resources == nil { + return nil // No resources is valid (will be caught elsewhere if needed) + } + + allowedKeys := map[hv1.ResourceName]bool{"cpu": true, "memory": true} + for key := range res.Spec.Resources { + if !allowedKeys[key] { + return fmt.Errorf("invalid resource key %q: only 'cpu' and 'memory' are allowed", key) + } + } + return nil +} + +// newFailoverReservation creates a new failover reservation for a VM on a specific hypervisor. +// This does NOT persist the reservation to the cluster - it only creates the in-memory object. +// The caller is responsible for persisting the reservation. +func newFailoverReservation(ctx context.Context, vm VM, hypervisor, creator string) *v1alpha1.Reservation { + logger := LoggerFromContext(ctx) + + // Build resources from VM's Resources map + // The VM struct uses "vcpus" and "memory" keys (see vm_source.go) + // We convert "vcpus" to "cpu" for the reservation because the scheduling capacity logic + // (e.g., nova filter_has_enough_capacity) uses "cpu" as the canonical key. + + // TODO we may want to use different resource (bigger) to enable better sharing + resources := make(map[hv1.ResourceName]resource.Quantity) + if memory, ok := vm.Resources["memory"]; ok { + resources["memory"] = memory + } + if vcpus, ok := vm.Resources["vcpus"]; ok { + // todo check if that is correct, i.e. that the cpu reported on e.g. hypervisors is vcpu and not pcpu + resources["cpu"] = vcpus + } + + reservation := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "failover-", + Labels: map[string]string{ + "cortex.cloud/creator": creator, + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelFailover, + }, + }, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailover, + Resources: resources, + TargetHost: hypervisor, // Set the desired hypervisor from scheduler response + FailoverReservation: &v1alpha1.FailoverReservationSpec{ + ResourceGroup: vm.FlavorName, + }, + }, + } + + // Set the status with the initial allocation (in-memory only) + now := metav1.Now() + reservation.Status.Host = hypervisor + reservation.Status.FailoverReservation = &v1alpha1.FailoverReservationStatus{ + Allocations: map[string]string{ + vm.UUID: vm.CurrentHypervisor, + }, + LastChanged: &now, + AcknowledgedAt: nil, + } + // Set the Ready condition + reservation.Status.Conditions = []metav1.Condition{ + { + Type: v1alpha1.ReservationConditionReady, + Status: metav1.ConditionTrue, + Reason: "ReservationActive", + Message: "Failover reservation is active and ready", + LastTransitionTime: metav1.Now(), + }, + } + + logger.V(1).Info("built new failover reservation", + "vmUUID", vm.UUID, + "hypervisor", hypervisor, + "resources", resources) + + return reservation +} diff --git a/internal/scheduling/reservations/failover/integration_test.go b/internal/scheduling/reservations/failover/integration_test.go new file mode 100644 index 000000000..b75e383e1 --- /dev/null +++ b/internal/scheduling/reservations/failover/integration_test.go @@ -0,0 +1,1513 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failover + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + "time" + + novaapi "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/filters" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/weighers" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +// ============================================================================ +// Integration Tests +// ============================================================================ + +// IntegrationTestCase defines a unified test case for all failover reservation integration tests. +type IntegrationTestCase struct { + Name string + Hypervisors []*hv1.Hypervisor + Reservations []*v1alpha1.Reservation + VMs []VM + FlavorRequirements map[string]int + + // Verification options + ExpectedMinRes int // Minimum expected reservations after reconcile + ExpectedMaxRes int // Maximum expected reservations after reconcile (0 = no max check) + VerifyVMReservation []string // VM UUIDs to verify have reservations + VMsToRemove map[string]map[string]string // reservationName -> vmUUID -> expectedHost (verify VM removed from reservation) + + // Test behavior options + ReconcileCount int // Number of reconciles to run (default: 1) + SkipFailureSimulation bool // Set to true to skip failure simulation tests (default: run simulation) + UseTraitsFilter bool // Use traits filter pipeline instead of default +} + +func TestIntegration(t *testing.T) { + testCases := []IntegrationTestCase{ + // ===================================================================== + // Basic Failover Reservation Tests + // ===================================================================== + { + Name: "2 VMs on 2 hosts, scheduler decides placement", + Hypervisors: []*hv1.Hypervisor{ + newHypervisor("host1", 8, 16, 4, 8, []hv1.Instance{{ID: "vm-1", Name: "vm-1", Active: true}}, nil), + newHypervisor("host2", 8, 16, 4, 8, []hv1.Instance{{ID: "vm-2", Name: "vm-2", Active: true}}, nil), + newHypervisor("host3", 8, 16, 0, 0, nil, nil), + newHypervisor("host4", 8, 16, 0, 0, nil, nil), + }, + VMs: []VM{ + newVM("vm-1", "m1.large", "project-A", "host1", 8192, 4), + newVM("vm-2", "m1.large", "project-A", "host2", 8192, 4), + }, + FlavorRequirements: map[string]int{"m1.large": 1}, + ExpectedMinRes: 1, + ExpectedMaxRes: 2, + VerifyVMReservation: []string{"vm-1", "vm-2"}, + }, + { + Name: "1 VM already has reservation, create 1 new", + Hypervisors: []*hv1.Hypervisor{ + newHypervisor("host1", 8, 16, 4, 8, []hv1.Instance{{ID: "vm-1", Name: "vm-1", Active: true}}, nil), + newHypervisor("host2", 8, 16, 4, 8, []hv1.Instance{{ID: "vm-2", Name: "vm-2", Active: true}}, nil), + newHypervisor("host3", 8, 16, 0, 0, nil, nil), + newHypervisor("host4", 8, 16, 0, 0, nil, nil), + }, + Reservations: []*v1alpha1.Reservation{ + newReservation("existing-res-1", "host2", 8192, 4, map[string]string{"vm-1": "host1"}), + }, + VMs: []VM{ + newVM("vm-1", "m1.large", "project-A", "host1", 8192, 4), + newVM("vm-2", "m1.large", "project-A", "host2", 8192, 4), + }, + FlavorRequirements: map[string]int{"m1.large": 1}, + ExpectedMinRes: 2, + ExpectedMaxRes: 2, + }, + { + Name: "VM with non-matching flavor, no reservations created", + Hypervisors: []*hv1.Hypervisor{ + newHypervisor("host1", 8, 16, 2, 4, []hv1.Instance{{ID: "vm-1", Name: "vm-1", Active: true}}, nil), + newHypervisor("host2", 8, 16, 0, 0, nil, nil), + newHypervisor("host3", 8, 16, 0, 0, nil, nil), + }, + VMs: []VM{ + newVM("vm-1", "m1.small", "project-A", "host1", 4096, 2), + }, + FlavorRequirements: map[string]int{"m1.large": 1}, // m1.small not in requirements + ExpectedMinRes: 0, + ExpectedMaxRes: 0, + }, + { + Name: "Reuse existing reservation - VM3 can share reservation with VM1", + Hypervisors: []*hv1.Hypervisor{ + newHypervisor("host1", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-1", Name: "vm-1", Active: true}}, nil), + newHypervisor("host2", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-2", Name: "vm-2", Active: true}}, nil), + newHypervisor("host3", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-3", Name: "vm-3", Active: true}}, nil), + newHypervisor("host4", 16, 32, 0, 0, nil, nil), + newHypervisor("host5", 16, 32, 0, 0, nil, nil), + }, + Reservations: []*v1alpha1.Reservation{ + newReservation("existing-res-1", "host2", 8192, 4, map[string]string{"vm-1": "host1"}), + }, + VMs: []VM{ + newVM("vm-1", "m1.large", "project-A", "host1", 8192, 4), + newVM("vm-2", "m1.large", "project-A", "host2", 8192, 4), + newVM("vm-3", "m1.large", "project-A", "host3", 8192, 4), + }, + FlavorRequirements: map[string]int{"m1.large": 1}, + ExpectedMinRes: 2, + ExpectedMaxRes: 3, + }, + + // ===================================================================== + // Availability Zone Tests + // ===================================================================== + { + Name: "VMs in different AZs get reservations only in their own AZ", + Hypervisors: []*hv1.Hypervisor{ + // AZ-A hosts + newHypervisorWithAZ("host-a1", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-a1", Name: "vm-a1", Active: true}}, nil, "az-a"), + newHypervisorWithAZ("host-a2", 16, 32, 0, 0, nil, nil, "az-a"), // Empty host for failover in AZ-A + // AZ-B hosts + newHypervisorWithAZ("host-b1", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-b1", Name: "vm-b1", Active: true}}, nil, "az-b"), + newHypervisorWithAZ("host-b2", 16, 32, 0, 0, nil, nil, "az-b"), // Empty host for failover in AZ-B + }, + VMs: []VM{ + newVMWithAZ("vm-a1", "m1.large", "project-A", "host-a1", 8192, 4, "az-a"), + newVMWithAZ("vm-b1", "m1.large", "project-A", "host-b1", 8192, 4, "az-b"), + }, + FlavorRequirements: map[string]int{"m1.large": 1}, + ExpectedMinRes: 2, // Each VM gets a reservation in its own AZ + ExpectedMaxRes: 2, + VerifyVMReservation: []string{"vm-a1", "vm-b1"}, + }, + { + Name: "VM in AZ-A cannot get reservation on AZ-B host (only AZ-B hosts available)", + Hypervisors: []*hv1.Hypervisor{ + // AZ-A hosts - only one host with VM, no empty hosts for failover + newHypervisorWithAZ("host-a1", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-a1", Name: "vm-a1", Active: true}}, nil, "az-a"), + // AZ-B hosts - empty hosts available but wrong AZ for vm-a1 + newHypervisorWithAZ("host-b1", 16, 32, 0, 0, nil, nil, "az-b"), + newHypervisorWithAZ("host-b2", 16, 32, 0, 0, nil, nil, "az-b"), + }, + VMs: []VM{ + newVMWithAZ("vm-a1", "m1.large", "project-A", "host-a1", 8192, 4, "az-a"), + }, + FlavorRequirements: map[string]int{}, // Empty - don't require failover for this test + ExpectedMinRes: 0, // No reservation can be created - no hosts in AZ-A + ExpectedMaxRes: 0, + SkipFailureSimulation: true, // Skip failure simulation since no reservations + }, + { + Name: "Multiple VMs in same AZ share reservations correctly", + Hypervisors: []*hv1.Hypervisor{ + // AZ-A hosts + newHypervisorWithAZ("host-a1", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-a1", Name: "vm-a1", Active: true}}, nil, "az-a"), + newHypervisorWithAZ("host-a2", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-a2", Name: "vm-a2", Active: true}}, nil, "az-a"), + newHypervisorWithAZ("host-a3", 16, 32, 0, 0, nil, nil, "az-a"), // Empty host for failover + // AZ-B hosts + newHypervisorWithAZ("host-b1", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-b1", Name: "vm-b1", Active: true}}, nil, "az-b"), + newHypervisorWithAZ("host-b2", 16, 32, 0, 0, nil, nil, "az-b"), // Empty host for failover + }, + VMs: []VM{ + newVMWithAZ("vm-a1", "m1.large", "project-A", "host-a1", 8192, 4, "az-a"), + newVMWithAZ("vm-a2", "m1.large", "project-A", "host-a2", 8192, 4, "az-a"), + newVMWithAZ("vm-b1", "m1.large", "project-A", "host-b1", 8192, 4, "az-b"), + }, + FlavorRequirements: map[string]int{"m1.large": 1}, + ExpectedMinRes: 2, // VMs in AZ-A can share, VM in AZ-B gets its own + ExpectedMaxRes: 3, + VerifyVMReservation: []string{"vm-a1", "vm-a2", "vm-b1"}, + }, + + // ===================================================================== + // Multi-Host Failure Tolerance Tests (n=2) + // ===================================================================== + { + Name: "4 VMs on 4 hosts with n=2 failure tolerance", + Hypervisors: []*hv1.Hypervisor{ + newHypervisor("host1", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-1", Name: "vm-1", Active: true}}, nil), + newHypervisor("host2", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-2", Name: "vm-2", Active: true}}, nil), + newHypervisor("host3", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-3", Name: "vm-3", Active: true}}, nil), + newHypervisor("host4", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-4", Name: "vm-4", Active: true}}, nil), + newHypervisor("host5", 32, 64, 0, 0, nil, nil), + newHypervisor("host6", 32, 64, 0, 0, nil, nil), + newHypervisor("host7", 32, 64, 0, 0, nil, nil), + newHypervisor("host8", 32, 64, 0, 0, nil, nil), + }, + VMs: []VM{ + newVM("vm-1", "m1.large", "project-A", "host1", 8192, 4), + newVM("vm-2", "m1.large", "project-A", "host2", 8192, 4), + newVM("vm-3", "m1.large", "project-A", "host3", 8192, 4), + newVM("vm-4", "m1.large", "project-A", "host4", 8192, 4), + }, + FlavorRequirements: map[string]int{"m1.large": 2}, + ExpectedMinRes: 4, + ExpectedMaxRes: 8, + }, + { + Name: "5 VMs on 5 hosts with n=2 failure tolerance", + Hypervisors: []*hv1.Hypervisor{ + newHypervisor("host1", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-1", Name: "vm-1", Active: true}}, nil), + newHypervisor("host2", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-2", Name: "vm-2", Active: true}}, nil), + newHypervisor("host3", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-3", Name: "vm-3", Active: true}}, nil), + newHypervisor("host4", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-4", Name: "vm-4", Active: true}}, nil), + newHypervisor("host5", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-5", Name: "vm-5", Active: true}}, nil), + newHypervisor("host6", 32, 64, 0, 0, nil, nil), + newHypervisor("host7", 32, 64, 0, 0, nil, nil), + }, + VMs: []VM{ + newVM("vm-1", "m1.large", "project-A", "host1", 8192, 4), + newVM("vm-2", "m1.large", "project-A", "host2", 8192, 4), + newVM("vm-3", "m1.large", "project-A", "host3", 8192, 4), + newVM("vm-4", "m1.large", "project-A", "host4", 8192, 4), + newVM("vm-5", "m1.large", "project-A", "host5", 8192, 4), + }, + FlavorRequirements: map[string]int{"m1.large": 2}, + ExpectedMinRes: 5, + ExpectedMaxRes: 10, + }, + { + Name: "5 hosts with existing reservations and n=2 failure tolerance", + Hypervisors: []*hv1.Hypervisor{ + newHypervisor("host1", 32, 64, 4, 8, []hv1.Instance{{ID: "vm-1", Name: "vm-1", Active: true}}, nil), + newHypervisor("host2", 32, 64, 4, 8, []hv1.Instance{{ID: "vm-2", Name: "vm-2", Active: true}}, nil), + newHypervisor("host3", 32, 64, 4, 8, []hv1.Instance{{ID: "vm-3", Name: "vm-3", Active: true}}, nil), + newHypervisor("host4", 32, 64, 0, 0, nil, nil), + newHypervisor("host5", 32, 64, 0, 0, nil, nil), + newHypervisor("host6", 32, 64, 0, 0, nil, nil), + }, + Reservations: []*v1alpha1.Reservation{ + newReservation("existing-res-1", "host4", 8192, 4, map[string]string{"vm-1": "host1"}), + newReservation("existing-res-2", "host5", 8192, 4, map[string]string{"vm-2": "host2"}), + }, + VMs: []VM{ + newVM("vm-1", "m1.large", "project-A", "host1", 8192, 4), + newVM("vm-2", "m1.large", "project-A", "host2", 8192, 4), + newVM("vm-3", "m1.large", "project-A", "host3", 8192, 4), + }, + FlavorRequirements: map[string]int{"m1.large": 2}, + ExpectedMinRes: 4, + ExpectedMaxRes: 6, + }, + + // ===================================================================== + // Incorrect Reservation Cleanup Tests + // ===================================================================== + { + Name: "VM deleted - remove from reservation allocations", + Hypervisors: []*hv1.Hypervisor{ + newHypervisor("host1", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-1", Name: "vm-1", Active: true}}, nil), + newHypervisor("host2", 16, 32, 0, 0, nil, nil), + newHypervisor("host3", 16, 32, 0, 0, nil, nil), + }, + Reservations: []*v1alpha1.Reservation{ + newReservation("res-1", "host2", 8192, 4, map[string]string{ + "vm-1": "host1", + "vm-deleted": "host3", // This VM no longer exists + }), + }, + VMs: []VM{ + newVM("vm-1", "m1.large", "project-A", "host1", 8192, 4), + }, + FlavorRequirements: map[string]int{"m1.large": 1}, + VMsToRemove: map[string]map[string]string{ + "res-1": {"vm-deleted": "host3"}, + }, + ReconcileCount: 2, // First removes incorrect, second creates new + SkipFailureSimulation: false, + }, + { + Name: "VM moved to different hypervisor - remove from reservation allocations", + Hypervisors: []*hv1.Hypervisor{ + newHypervisor("host1", 16, 32, 0, 0, nil, nil), + newHypervisor("host2", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-1", Name: "vm-1", Active: true}}, nil), + newHypervisor("host3", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-2", Name: "vm-2", Active: true}}, nil), + newHypervisor("host4", 16, 32, 0, 0, nil, nil), + }, + Reservations: []*v1alpha1.Reservation{ + newReservation("res-1", "host4", 8192, 4, map[string]string{ + "vm-1": "host1", // vm-1 was on host1, but now it's on host2 + "vm-2": "host3", + }), + }, + VMs: []VM{ + newVM("vm-1", "m1.large", "project-A", "host2", 8192, 4), // Moved from host1 to host2 + newVM("vm-2", "m1.large", "project-A", "host3", 8192, 4), + }, + FlavorRequirements: map[string]int{"m1.large": 1}, + VMsToRemove: map[string]map[string]string{ + "res-1": {"vm-1": "host1"}, + }, + ReconcileCount: 2, + SkipFailureSimulation: false, + }, + { + Name: "VM on same host as reservation - remove due to eligibility", + Hypervisors: []*hv1.Hypervisor{ + newHypervisor("host1", 16, 32, 0, 0, nil, nil), + newHypervisor("host2", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-2", Name: "vm-2", Active: true}}, nil), + newHypervisor("host3", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-1", Name: "vm-1", Active: true}}, nil), + newHypervisor("host4", 16, 32, 0, 0, nil, nil), + }, + Reservations: []*v1alpha1.Reservation{ + newReservation("res-1", "host3", 8192, 4, map[string]string{ + "vm-1": "host1", // vm-1 was on host1, but now it's on host3 (same as reservation!) + "vm-2": "host2", + }), + }, + VMs: []VM{ + newVM("vm-1", "m1.large", "project-A", "host3", 8192, 4), // Same as reservation! + newVM("vm-2", "m1.large", "project-A", "host2", 8192, 4), + }, + FlavorRequirements: map[string]int{"m1.large": 1}, + VMsToRemove: map[string]map[string]string{ + "res-1": {"vm-1": "host1"}, + }, + ReconcileCount: 2, + SkipFailureSimulation: false, + }, + { + Name: "Mixed scenario - deleted VM, moved VM, and valid VM", + Hypervisors: []*hv1.Hypervisor{ + newHypervisor("host1", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-1", Name: "vm-1", Active: true}}, nil), + newHypervisor("host2", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-2", Name: "vm-2", Active: true}}, nil), + newHypervisor("host3", 16, 32, 0, 0, nil, nil), + newHypervisor("host4", 16, 32, 0, 0, nil, nil), + }, + Reservations: []*v1alpha1.Reservation{ + newReservation("res-1", "host3", 8192, 4, map[string]string{ + "vm-1": "host1", // Valid + "vm-2": "host3", // Moved from host3 to host2 + "vm-deleted": "host4", // Deleted + }), + }, + VMs: []VM{ + newVM("vm-1", "m1.large", "project-A", "host1", 8192, 4), + newVM("vm-2", "m1.large", "project-A", "host2", 8192, 4), // Moved from host3 to host2 + }, + FlavorRequirements: map[string]int{"m1.large": 1}, + VMsToRemove: map[string]map[string]string{ + "res-1": {"vm-2": "host3", "vm-deleted": "host4"}, + }, + ReconcileCount: 2, + SkipFailureSimulation: false, + }, + + // ===================================================================== + // Traits Filter Tests + // ===================================================================== + { + Name: "HANA VM gets reservation on HANA host, regular VM on any host", + Hypervisors: []*hv1.Hypervisor{ + newHypervisor("host1", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-hana-1", Name: "vm-hana-1", Active: true}}, []string{"CUSTOM_HANA"}), + newHypervisor("host2", 16, 32, 0, 0, nil, []string{"CUSTOM_HANA"}), + newHypervisor("host3", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-regular-1", Name: "vm-regular-1", Active: true}}, nil), + newHypervisor("host4", 16, 32, 0, 0, nil, nil), + }, + VMs: []VM{ + newVMWithExtraSpecs("vm-hana-1", "m1.hana", "project-A", "host1", 8192, 4, map[string]string{"trait:CUSTOM_HANA": "required"}), + newVMWithExtraSpecs("vm-regular-1", "m1.large", "project-A", "host3", 8192, 4, nil), + }, + FlavorRequirements: map[string]int{"m1.hana": 1, "m1.large": 1}, + ExpectedMinRes: 1, + UseTraitsFilter: true, + }, + { + Name: "VM with forbidden trait cannot use host with that trait", + Hypervisors: []*hv1.Hypervisor{ + newHypervisor("host1", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-no-hana-1", Name: "vm-no-hana-1", Active: true}}, nil), + newHypervisor("host2", 16, 32, 0, 0, nil, []string{"CUSTOM_HANA"}), // Has HANA trait + newHypervisor("host3", 16, 32, 0, 0, nil, nil), // No HANA trait + }, + VMs: []VM{ + newVMWithExtraSpecs("vm-no-hana-1", "m1.no-hana", "project-A", "host1", 8192, 4, map[string]string{"trait:CUSTOM_HANA": "forbidden"}), + }, + FlavorRequirements: map[string]int{"m1.no-hana": 1}, + ExpectedMinRes: 1, + UseTraitsFilter: true, + }, + { + Name: "Mixed required and forbidden traits - VMs placed on correct hosts", + Hypervisors: []*hv1.Hypervisor{ + newHypervisor("host1", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-hana-1", Name: "vm-hana-1", Active: true}}, []string{"CUSTOM_HANA"}), + newHypervisor("host2", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-no-hana-1", Name: "vm-no-hana-1", Active: true}}, nil), + newHypervisor("host3", 16, 32, 0, 0, nil, []string{"CUSTOM_HANA"}), // HANA host for failover + newHypervisor("host4", 16, 32, 0, 0, nil, nil), // Non-HANA host for failover + }, + VMs: []VM{ + newVMWithExtraSpecs("vm-hana-1", "m1.hana", "project-A", "host1", 8192, 4, map[string]string{"trait:CUSTOM_HANA": "required"}), + newVMWithExtraSpecs("vm-no-hana-1", "m1.no-hana", "project-A", "host2", 8192, 4, map[string]string{"trait:CUSTOM_HANA": "forbidden"}), + }, + FlavorRequirements: map[string]int{"m1.hana": 1, "m1.no-hana": 1}, + ExpectedMinRes: 2, // Each VM needs its own reservation on compatible host + UseTraitsFilter: true, + }, + { + Name: "Multiple HANA VMs share reservation on HANA host", + Hypervisors: []*hv1.Hypervisor{ + newHypervisor("host1", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-hana-1", Name: "vm-hana-1", Active: true}}, []string{"CUSTOM_HANA"}), + newHypervisor("host2", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-hana-2", Name: "vm-hana-2", Active: true}}, []string{"CUSTOM_HANA"}), + newHypervisor("host3", 16, 32, 0, 0, nil, []string{"CUSTOM_HANA"}), // Empty HANA host for failover + newHypervisor("host4", 16, 32, 0, 0, nil, nil), // Non-HANA host (not usable for HANA VMs) + }, + VMs: []VM{ + newVMWithExtraSpecs("vm-hana-1", "m1.hana", "project-A", "host1", 8192, 4, map[string]string{"trait:CUSTOM_HANA": "required"}), + newVMWithExtraSpecs("vm-hana-2", "m1.hana", "project-A", "host2", 8192, 4, map[string]string{"trait:CUSTOM_HANA": "required"}), + }, + FlavorRequirements: map[string]int{"m1.hana": 1}, + ExpectedMinRes: 1, // Both HANA VMs can share reservation on host3 + UseTraitsFilter: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + runIntegrationTest(t, tc) + }) + } +} + +// runIntegrationTest executes a single integration test case. +func runIntegrationTest(t *testing.T, tc IntegrationTestCase) { + t.Helper() + + // Create test environment + var env *IntegrationTestEnv + if tc.UseTraitsFilter { + env = newIntegrationTestEnvWithTraitsFilter(t, tc.VMs, tc.Hypervisors, tc.Reservations) + } else { + env = newIntegrationTestEnv(t, tc.VMs, tc.Hypervisors, tc.Reservations) + } + defer env.Close() + + t.Log("Initial state:") + env.LogStateSummary() + + // Determine number of reconciles (default: 1) + reconcileCount := tc.ReconcileCount + if reconcileCount == 0 { + reconcileCount = 1 + } + + // Run reconciles + for i := 1; i <= reconcileCount; i++ { + if err := env.TriggerFailoverReconcile(tc.FlavorRequirements); err != nil { + t.Logf("Reconcile %d returned error (may be expected): %v", i, err) + } + } + t.Logf("State after reconcile") + env.LogStateSummary() + if len(tc.VMsToRemove) > 0 { + for resName, vmHosts := range tc.VMsToRemove { + for vmUUID, expectedHost := range vmHosts { + env.VerifyVMRemovedFromReservation(resName, vmUUID, expectedHost) + } + } + } + + // Verify reservation count + if tc.ExpectedMaxRes > 0 { + env.VerifyReservationCountInRange(tc.ExpectedMinRes, tc.ExpectedMaxRes) + } else if tc.ExpectedMinRes > 0 { + reservations := env.ListReservations() + if len(reservations) < tc.ExpectedMinRes { + t.Errorf("Expected at least %d reservation(s), got %d", tc.ExpectedMinRes, len(reservations)) + } + } + + // Verify specific VMs have reservations + for _, vmUUID := range tc.VerifyVMReservation { + for _, vm := range tc.VMs { + if vm.UUID == vmUUID { + env.VerifyVMHasFailoverReservation(vmUUID, vm.CurrentHypervisor) + break + } + } + } + + // Verify all VMs have required reservations + env.VerifyVMsHaveRequiredReservations(tc.FlavorRequirements) + + // Run failure simulation unless skipped + if !tc.SkipFailureSimulation { + allHosts := make([]string, len(tc.Hypervisors)) + for i, hv := range tc.Hypervisors { + allHosts[i] = hv.Name + } + env.VerifyEvacuationForAllFailureCombinations(tc.FlavorRequirements, allHosts, 8192, 4) + } +} + +// ============================================================================ +// Test Environment +// ============================================================================ + +// IntegrationTestEnv provides a test environment with: +// - Fake k8s client for CRD operations (reservations, hypervisors) +// - Real HTTP server for NovaExternalScheduler endpoint +// - MockVMSource for listing VMs +type IntegrationTestEnv struct { + T *testing.T + Scheme *runtime.Scheme + K8sClient client.Client + Server *httptest.Server + NovaController *nova.FilterWeigherPipelineController + VMSource VMSource + SchedulerBaseURL string +} + +func (env *IntegrationTestEnv) Close() { + env.Server.Close() +} + +// ============================================================================ +// Environment Helper Methods +// ============================================================================ + +// SendPlacementRequest sends a placement request to the scheduler and returns the response. +func (env *IntegrationTestEnv) SendPlacementRequest(req novaapi.ExternalSchedulerRequest) novaapi.ExternalSchedulerResponse { + env.T.Helper() + + body, err := json.Marshal(req) + if err != nil { + env.T.Fatalf("Failed to marshal request: %v", err) + } + + resp, err := http.Post(env.SchedulerBaseURL+"/scheduler/nova/external", "application/json", bytes.NewReader(body)) + if err != nil { + env.T.Fatalf("Failed to send request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + env.T.Fatalf("Expected status 200, got %d", resp.StatusCode) + } + + var response novaapi.ExternalSchedulerResponse + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + env.T.Fatalf("Failed to decode response: %v", err) + } + + return response +} + +// ListVMs returns all VMs from the VMSource. +func (env *IntegrationTestEnv) ListVMs() []VM { + vms, err := env.VMSource.ListVMs(context.Background()) + if err != nil { + env.T.Fatalf("Failed to list VMs: %v", err) + } + return vms +} + +// ListReservations returns all reservations. +func (env *IntegrationTestEnv) ListReservations() []v1alpha1.Reservation { + var list v1alpha1.ReservationList + if err := env.K8sClient.List(context.Background(), &list); err != nil { + env.T.Fatalf("Failed to list reservations: %v", err) + } + return list.Items +} + +// ListHypervisors returns all hypervisors. +func (env *IntegrationTestEnv) ListHypervisors() []hv1.Hypervisor { + var list hv1.HypervisorList + if err := env.K8sClient.List(context.Background(), &list); err != nil { + env.T.Fatalf("Failed to list hypervisors: %v", err) + } + return list.Items +} + +// LogStateSummary logs a summary of the current state. +func (env *IntegrationTestEnv) LogStateSummary() { + env.T.Helper() + + hypervisors := env.ListHypervisors() + vms := env.ListVMs() + reservationsList := env.ListReservations() + + vmsByHypervisor := make(map[string][]VM) + for _, vm := range vms { + vmsByHypervisor[vm.CurrentHypervisor] = append(vmsByHypervisor[vm.CurrentHypervisor], vm) + } + + resByHypervisor := make(map[string][]v1alpha1.Reservation) + for _, res := range reservationsList { + if res.Status.Host != "" { + resByHypervisor[res.Status.Host] = append(resByHypervisor[res.Status.Host], res) + } + } + + env.T.Log("=== State Summary ===") + for _, hv := range hypervisors { + hypervisorName := hv.Name + + var vmParts []string + for _, vm := range vmsByHypervisor[hypervisorName] { + memoryMB := int64(0) + vcpus := int64(0) + if mem, ok := vm.Resources["memory"]; ok { + memoryMB = mem.Value() / (1024 * 1024) + } + if cpu, ok := vm.Resources["vcpus"]; ok { + vcpus = cpu.Value() + } + vmParts = append(vmParts, fmt.Sprintf("%s(%dMB,%dCPU)", vm.UUID, memoryMB, vcpus)) + } + + var resParts []string + for _, res := range resByHypervisor[hypervisorName] { + memoryMB := int64(0) + vcpus := int64(0) + if mem, ok := res.Spec.Resources[hv1.ResourceMemory]; ok { + memoryMB = mem.Value() / (1024 * 1024) + } + if cpu, ok := res.Spec.Resources[hv1.ResourceCPU]; ok { + vcpus = cpu.Value() + } + + var usedByParts []string + if res.Status.FailoverReservation != nil { + for vmID, vmHost := range res.Status.FailoverReservation.Allocations { + usedByParts = append(usedByParts, fmt.Sprintf("%s@%s", vmID, vmHost)) + } + } + usedByStr := "" + if len(usedByParts) > 0 { + usedByStr = fmt.Sprintf("; used_by=[%s]", strings.Join(usedByParts, ",")) + } + + resParts = append(resParts, fmt.Sprintf("%s(%dMB;%dCPU%s)", res.Name, memoryMB, vcpus, usedByStr)) + } + + var parts []string + if len(vmParts) > 0 { + parts = append(parts, strings.Join(vmParts, "; ")) + } + if len(resParts) > 0 { + parts = append(parts, strings.Join(resParts, "; ")) + } + + summary := "(empty)" + if len(parts) > 0 { + summary = strings.Join(parts, "; ") + } + + env.T.Logf("%s: %s", hypervisorName, summary) + } + env.T.Log("=====================") +} + +// TriggerFailoverReconcile creates a FailoverReservationController and triggers its Reconcile method. +func (env *IntegrationTestEnv) TriggerFailoverReconcile(flavorRequirements map[string]int) error { + env.T.Helper() + + schedulerClient := reservations.NewSchedulerClient(env.SchedulerBaseURL + "/scheduler/nova/external") + + config := FailoverConfig{ + ReconcileInterval: time.Minute, + Creator: "test-failover-controller", + FlavorFailoverRequirements: flavorRequirements, + } + + controller := NewFailoverReservationController( + env.K8sClient, + env.VMSource, + config, + schedulerClient, + ) + + _, err := controller.ReconcilePeriodic(context.Background()) + return err +} + +// ============================================================================ +// Verification Helpers +// ============================================================================ + +// VerifyReservationCountInRange checks that the number of reservations is within the expected range. +func (env *IntegrationTestEnv) VerifyReservationCountInRange(minCount, maxCount int) { + env.T.Helper() + reservationsList := env.ListReservations() + count := len(reservationsList) + if count < minCount || count > maxCount { + env.T.Errorf("Expected %d-%d reservations, got %d", minCount, maxCount, count) + } else { + env.T.Logf("Reservation count %d is within expected range [%d, %d]", count, minCount, maxCount) + } +} + +// VerifyVMHasFailoverReservation checks that a VM has a failover reservation on a different hypervisor +// and that the reservation host is in the same availability zone as the VM. +func (env *IntegrationTestEnv) VerifyVMHasFailoverReservation(vmUUID, vmCurrentHypervisor string) { + env.T.Helper() + reservationsList := env.ListReservations() + hypervisors := env.ListHypervisors() + vms := env.ListVMs() + + // Find the VM to get its AZ + var vmAZ string + for _, vm := range vms { + if vm.UUID == vmUUID { + vmAZ = vm.AvailabilityZone + break + } + } + + // Build a map of hypervisor name -> AZ for quick lookup + hypervisorAZ := make(map[string]string) + for _, hv := range hypervisors { + if az, ok := hv.Labels[corev1.LabelTopologyZone]; ok { + hypervisorAZ[hv.Name] = az + } + } + + for _, res := range reservationsList { + if res.Spec.Type != v1alpha1.ReservationTypeFailover { + continue + } + if res.Status.FailoverReservation != nil { + if _, exists := res.Status.FailoverReservation.Allocations[vmUUID]; exists { + if res.Status.Host == vmCurrentHypervisor { + env.T.Errorf("Failover reservation for VM %s is on the same hypervisor %s", vmUUID, vmCurrentHypervisor) + } + + // Verify the reservation host is in the same AZ as the VM + resHostAZ := hypervisorAZ[res.Status.Host] + if vmAZ != resHostAZ { + env.T.Errorf("Failover reservation for VM %s (AZ: %s) is on hypervisor %s which is in wrong AZ: %s", + vmUUID, vmAZ, res.Status.Host, resHostAZ) + } + + env.T.Logf("VM %s (AZ: %s) has failover reservation %s on hypervisor %s (AZ: %s)", + vmUUID, vmAZ, res.Name, res.Status.Host, resHostAZ) + return + } + } + } + env.T.Errorf("No failover reservation found for VM %s", vmUUID) +} + +// VerifyVMRemovedFromReservation checks that a specific VM with a specific host allocation +// is not in a specific reservation's allocations. The expectedHost parameter allows +// verifying that a VM was removed from a reservation where it was allocated with a specific host, +// while allowing the VM to be re-added with a different host. +func (env *IntegrationTestEnv) VerifyVMRemovedFromReservation(reservationName, vmUUID, expectedHost string) { + env.T.Helper() + reservationsList := env.ListReservations() + + for _, res := range reservationsList { + if res.Name != reservationName { + continue + } + if res.Status.FailoverReservation == nil { + env.T.Logf("VM %s correctly removed from reservation %s (no allocations)", vmUUID, reservationName) + return + } + if allocatedHost, exists := res.Status.FailoverReservation.Allocations[vmUUID]; exists { + if allocatedHost == expectedHost { + env.T.Errorf("VM %s should have been removed from reservation %s (was allocated with host %s) but is still present with same host", vmUUID, reservationName, expectedHost) + } else { + env.T.Logf("VM %s was re-added to reservation %s with different host (old: %s, new: %s) - this is allowed", vmUUID, reservationName, expectedHost, allocatedHost) + } + } else { + env.T.Logf("VM %s correctly removed from reservation %s", vmUUID, reservationName) + } + return + } + env.T.Logf("Reservation %s not found (may have been deleted)", reservationName) +} + +// VerifyVMsHaveRequiredReservations checks that each VM has the required number of failover reservations. +func (env *IntegrationTestEnv) VerifyVMsHaveRequiredReservations(flavorRequirements map[string]int) bool { + env.T.Helper() + + vms := env.ListVMs() + reservationsList := env.ListReservations() + + vmReservationHosts := make(map[string][]string) + for _, res := range reservationsList { + if res.Spec.Type != v1alpha1.ReservationTypeFailover { + continue + } + if res.Status.FailoverReservation != nil { + for vmUUID := range res.Status.FailoverReservation.Allocations { + vmReservationHosts[vmUUID] = append(vmReservationHosts[vmUUID], res.Status.Host) + } + } + } + + env.T.Log("╔══════════════════════════════════════════════════════════════════╗") + env.T.Log("║ SANITY CHECK: Verifying each VM has required reservations ║") + env.T.Log("╠══════════════════════════════════════════════════════════════════╣") + + allPassed := true + for _, vm := range vms { + requiredCount, needsFailover := flavorRequirements[vm.FlavorName] + if !needsFailover || requiredCount == 0 { + env.T.Logf("║ ⏭️ %s: flavor %s doesn't require failover", vm.UUID, vm.FlavorName) + continue + } + + actualCount := len(vmReservationHosts[vm.UUID]) + if actualCount >= requiredCount { + env.T.Logf("║ ✅ %s: has %d/%d reservations on hosts %v", vm.UUID, actualCount, requiredCount, vmReservationHosts[vm.UUID]) + } else { + env.T.Logf("║ ❌ %s: has %d/%d reservations (MISSING %d)", vm.UUID, actualCount, requiredCount, requiredCount-actualCount) + env.T.Errorf("VM %s has %d reservations but needs %d", vm.UUID, actualCount, requiredCount) + allPassed = false + } + } + + env.T.Log("╚══════════════════════════════════════════════════════════════════╝") + return allPassed +} + +// VerifyEvacuationForAllFailureCombinations tests that VMs can be evacuated for all +// possible host failure combinations up to the configured failure tolerance. +func (env *IntegrationTestEnv) VerifyEvacuationForAllFailureCombinations( + flavorRequirements map[string]int, + allHosts []string, + memoryMB, vcpus uint64, +) { + + env.T.Helper() + + if !env.VerifyVMsHaveRequiredReservations(flavorRequirements) { + env.T.Error("Sanity check failed: not all VMs have required reservations, skipping failure simulations") + return + } + + toleranceGroups := make(map[int][]string) + for fn, tolerance := range flavorRequirements { + toleranceGroups[tolerance] = append(toleranceGroups[tolerance], fn) + } + + for tolerance, flavorNames := range toleranceGroups { + if tolerance == 0 { + continue + } + + env.T.Logf("Testing failure tolerance %d for flavors: %v", tolerance, flavorNames) + + hostCombinations := generateHostCombinations(allHosts, tolerance) + env.T.Logf("Generated %d host failure combinations for tolerance %d", len(hostCombinations), tolerance) + + for _, failedHosts := range hostCombinations { + env.T.Run("FailedHosts_"+strings.Join(failedHosts, "_"), func(t *testing.T) { + env.simulateHostFailure(failedHosts, allHosts, memoryMB, vcpus, flavorRequirements) + }) + } + } +} + +// simulateHostFailure simulates a host failure by sending evacuation requests for all VMs +// on the failed hosts and verifying they can be placed on reservation hosts. +func (env *IntegrationTestEnv) simulateHostFailure(failedHosts, allHosts []string, memoryMB, vcpus uint64, flavorRequirements map[string]int) { + env.T.Helper() + + vms := env.ListVMs() + reservationsList := env.ListReservations() + + reservationsByHost := make(map[string][]v1alpha1.Reservation) + for _, res := range reservationsList { + if res.Spec.Type == v1alpha1.ReservationTypeFailover { + reservationsByHost[res.Status.Host] = append(reservationsByHost[res.Status.Host], res) + } + } + + failedHostSet := make(map[string]bool) + for _, h := range failedHosts { + failedHostSet[h] = true + } + + availableHosts := make([]string, 0) + for _, h := range allHosts { + if !failedHostSet[h] { + availableHosts = append(availableHosts, h) + } + } + + affectedVMs := make([]VM, 0) + for _, vm := range vms { + if failedHostSet[vm.CurrentHypervisor] { + affectedVMs = append(affectedVMs, vm) + } + } + + usedReservations := make(map[string]bool) + + env.T.Log("╔══════════════════════════════════════════════════════════════════╗") + env.T.Logf("║ FAILURE SCENARIO: Hosts %v failed", failedHosts) + env.T.Log("╠══════════════════════════════════════════════════════════════════╣") + env.T.Logf("║ AFFECTED VMs: %d VMs need evacuation", len(affectedVMs)) + env.T.Log("╠══════════════════════════════════════════════════════════════════╣") + env.T.Log("║ EVACUATION MOVES:") + + successful := 0 + failed := 0 + + for _, vm := range affectedVMs { + if _, needsFailover := flavorRequirements[vm.FlavorName]; !needsFailover { + env.T.Logf("║ ⏭️ %s: flavor %s doesn't require failover, skipping", vm.UUID, vm.FlavorName) + continue + } + + externalHosts := make([]novaapi.ExternalSchedulerHost, len(availableHosts)) + weights := make(map[string]float64) + for i, h := range availableHosts { + externalHosts[i] = novaapi.ExternalSchedulerHost{ComputeHost: h} + weights[h] = 1.0 + } + + // Build extra specs including VM's flavor extra specs (for traits) + extraSpecs := map[string]string{ + "capabilities:hypervisor_type": "qemu", + } + for k, v := range vm.FlavorExtraSpecs { + extraSpecs[k] = v + } + + request := novaapi.ExternalSchedulerRequest{ + Pipeline: "nova-external-scheduler-kvm-all-filters-enabled", + Hosts: externalHosts, + Weights: weights, + Spec: novaapi.NovaObject[novaapi.NovaSpec]{ + Data: novaapi.NovaSpec{ + InstanceUUID: vm.UUID, + ProjectID: vm.ProjectID, + Flavor: novaapi.NovaObject[novaapi.NovaFlavor]{ + Data: novaapi.NovaFlavor{ + Name: vm.FlavorName, + VCPUs: vcpus, + MemoryMB: memoryMB, + ExtraSpecs: extraSpecs, + }, + }, + }, + }, + } + + response := env.SendPlacementRequest(request) + + if len(response.Hosts) == 0 { + env.T.Logf("║ ❌ %s: %s → NO HOSTS AVAILABLE", vm.UUID, vm.CurrentHypervisor) + env.T.Errorf("No hypervisors available for evacuating VM %s from failed hypervisor %s", vm.UUID, vm.CurrentHypervisor) + failed++ + continue + } + + selectedHost := "" + selectedReservation := "" + for _, candidateHost := range response.Hosts { + for _, res := range reservationsByHost[candidateHost] { + if res.Status.FailoverReservation != nil { + if _, vmUsesThis := res.Status.FailoverReservation.Allocations[vm.UUID]; vmUsesThis { + if !usedReservations[res.Name] { + usedReservations[res.Name] = true + selectedHost = candidateHost + selectedReservation = res.Name + break + } + } + } + } + if selectedHost != "" { + break + } + } + + if selectedHost == "" { + env.T.Logf("║ ❌ %s: %s → NO RESERVATION HOST FOUND", vm.UUID, vm.CurrentHypervisor) + env.T.Errorf("VM %s has no reservation hypervisor available for evacuation from %s", vm.UUID, vm.CurrentHypervisor) + failed++ + continue + } + + env.T.Logf("║ ✅ %s: %s → %s (using reservation %s)", vm.UUID, vm.CurrentHypervisor, selectedHost, selectedReservation) + successful++ + } + + env.T.Log("╠══════════════════════════════════════════════════════════════════╣") + env.T.Logf("║ RESULT: %d/%d VMs successfully evacuated", successful, len(affectedVMs)) + env.T.Log("╚══════════════════════════════════════════════════════════════════╝") + + if failed > 0 { + env.T.Errorf("Failed to evacuate %d VMs", failed) + } +} + +// ============================================================================ +// Helper Functions +// ============================================================================ + +var metricsRegistered sync.Once +var sharedMonitor lib.FilterWeigherPipelineMonitor + +func getSharedMonitor() lib.FilterWeigherPipelineMonitor { + metricsRegistered.Do(func() { + sharedMonitor = lib.NewPipelineMonitor() + }) + return sharedMonitor +} + +// MockVMSource implements VMSource for testing without requiring a database. +type MockVMSource struct { + VMs []VM +} + +// NewMockVMSource creates a new MockVMSource with the given VMs. +func NewMockVMSource(vms []VM) *MockVMSource { + return &MockVMSource{VMs: vms} +} + +// ListVMs returns the configured VMs. +func (s *MockVMSource) ListVMs(_ context.Context) ([]VM, error) { + return s.VMs, nil +} + +// ListVMsOnHypervisors returns VMs that are on the given hypervisors. +// For the mock, this simply returns all VMs (filtering is not needed for tests). +func (s *MockVMSource) ListVMsOnHypervisors(_ context.Context, _ *hv1.HypervisorList, _ bool) ([]VM, error) { + return s.VMs, nil +} + +// GetVM returns a specific VM by UUID. +// Returns nil, nil if the VM is not found. +func (s *MockVMSource) GetVM(_ context.Context, vmUUID string) (*VM, error) { + for i := range s.VMs { + if s.VMs[i].UUID == vmUUID { + return &s.VMs[i], nil + } + } + return nil, nil +} + +// newIntegrationTestEnv creates a complete test environment with HTTP server and VMSource. +func newIntegrationTestEnv(t *testing.T, vms []VM, hypervisors []*hv1.Hypervisor, reservations []*v1alpha1.Reservation) *IntegrationTestEnv { + t.Helper() + + // Combine hypervisors and reservations into a single objects slice + objects := make([]client.Object, 0, len(hypervisors)+len(reservations)) + for _, hv := range hypervisors { + objects = append(objects, hv) + } + for _, res := range reservations { + objects = append(objects, res) + } + + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("Failed to add v1alpha1 scheme: %v", err) + } + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("Failed to add hv1 scheme: %v", err) + } + + k8sClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + WithStatusSubresource(&v1alpha1.Reservation{}). + WithIndex(&v1alpha1.Reservation{}, "spec.type", func(obj client.Object) []string { + res := obj.(*v1alpha1.Reservation) + return []string{string(res.Spec.Type)} + }). + Build() + + novaController := &nova.FilterWeigherPipelineController{ + BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]]{ + Client: k8sClient, + Pipelines: make(map[string]lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]), + PipelineConfigs: make(map[string]v1alpha1.Pipeline), + }, + Monitor: getSharedMonitor(), + } + + // Register all pipelines needed for testing + pipelines := []v1alpha1.Pipeline{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "nova-external-scheduler-kvm-all-filters-enabled", + }, + Spec: v1alpha1.PipelineSpec{ + Type: v1alpha1.PipelineTypeFilterWeigher, + Filters: []v1alpha1.FilterSpec{ + {Name: "filter_has_enough_capacity"}, + {Name: "filter_correct_az"}, + }, + Weighers: []v1alpha1.WeigherSpec{ + {Name: "kvm_failover_evacuation"}, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: PipelineReuseFailoverReservation, + }, + Spec: v1alpha1.PipelineSpec{ + Type: v1alpha1.PipelineTypeFilterWeigher, + Filters: []v1alpha1.FilterSpec{ + {Name: "filter_has_requested_traits"}, + {Name: "filter_correct_az"}, + }, + Weighers: []v1alpha1.WeigherSpec{}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: PipelineNewFailoverReservation, + }, + Spec: v1alpha1.PipelineSpec{ + Type: v1alpha1.PipelineTypeFilterWeigher, + Filters: []v1alpha1.FilterSpec{ + {Name: "filter_has_enough_capacity"}, + {Name: "filter_has_requested_traits"}, + {Name: "filter_correct_az"}, + }, + Weighers: []v1alpha1.WeigherSpec{ + {Name: "kvm_failover_evacuation"}, + }, + }, + }, + } + + ctx := context.Background() + for _, pipeline := range pipelines { + result := lib.InitNewFilterWeigherPipeline( + ctx, k8sClient, pipeline.Name, + filters.Index, pipeline.Spec.Filters, + weighers.Index, pipeline.Spec.Weighers, + novaController.Monitor, + ) + if len(result.FilterErrors) > 0 || len(result.WeigherErrors) > 0 { + t.Fatalf("Failed to init pipeline %s: filters=%v, weighers=%v", pipeline.Name, result.FilterErrors, result.WeigherErrors) + } + novaController.Pipelines[pipeline.Name] = result.Pipeline + novaController.PipelineConfigs[pipeline.Name] = pipeline + } + + api := &testHTTPAPI{delegate: novaController} + mux := http.NewServeMux() + mux.HandleFunc("/scheduler/nova/external", api.NovaExternalScheduler) + server := httptest.NewServer(mux) + + return &IntegrationTestEnv{ + T: t, + Scheme: scheme, + K8sClient: k8sClient, + Server: server, + NovaController: novaController, + VMSource: NewMockVMSource(vms), + SchedulerBaseURL: server.URL, + } +} + +// newIntegrationTestEnvWithTraitsFilter creates a test environment with the filter_has_requested_traits filter enabled. +func newIntegrationTestEnvWithTraitsFilter(t *testing.T, vms []VM, hypervisors []*hv1.Hypervisor, reservations []*v1alpha1.Reservation) *IntegrationTestEnv { + t.Helper() + + // Combine hypervisors and reservations into a single objects slice + objects := make([]client.Object, 0, len(hypervisors)+len(reservations)) + for _, hv := range hypervisors { + objects = append(objects, hv) + } + for _, res := range reservations { + objects = append(objects, res) + } + + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("Failed to add v1alpha1 scheme: %v", err) + } + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("Failed to add hv1 scheme: %v", err) + } + + k8sClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + WithStatusSubresource(&v1alpha1.Reservation{}). + WithIndex(&v1alpha1.Reservation{}, "spec.type", func(obj client.Object) []string { + res := obj.(*v1alpha1.Reservation) + return []string{string(res.Spec.Type)} + }). + Build() + + novaController := &nova.FilterWeigherPipelineController{ + BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]]{ + Client: k8sClient, + Pipelines: make(map[string]lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]), + PipelineConfigs: make(map[string]v1alpha1.Pipeline), + }, + Monitor: getSharedMonitor(), + } + + // Register all pipelines needed for testing (with traits filter enabled) + pipelines := []v1alpha1.Pipeline{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "nova-external-scheduler-kvm-all-filters-enabled", + }, + Spec: v1alpha1.PipelineSpec{ + Type: v1alpha1.PipelineTypeFilterWeigher, + Filters: []v1alpha1.FilterSpec{ + {Name: "filter_has_enough_capacity"}, + {Name: "filter_has_requested_traits"}, + {Name: "filter_correct_az"}, + }, + Weighers: []v1alpha1.WeigherSpec{ + {Name: "kvm_failover_evacuation"}, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: PipelineReuseFailoverReservation, + }, + Spec: v1alpha1.PipelineSpec{ + Type: v1alpha1.PipelineTypeFilterWeigher, + Filters: []v1alpha1.FilterSpec{ + {Name: "filter_has_requested_traits"}, + {Name: "filter_correct_az"}, + }, + Weighers: []v1alpha1.WeigherSpec{}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: PipelineNewFailoverReservation, + }, + Spec: v1alpha1.PipelineSpec{ + Type: v1alpha1.PipelineTypeFilterWeigher, + Filters: []v1alpha1.FilterSpec{ + {Name: "filter_has_enough_capacity"}, + {Name: "filter_has_requested_traits"}, + {Name: "filter_correct_az"}, + }, + Weighers: []v1alpha1.WeigherSpec{ + {Name: "kvm_failover_evacuation"}, + }, + }, + }, + } + + ctx := context.Background() + for _, pipeline := range pipelines { + result := lib.InitNewFilterWeigherPipeline( + ctx, k8sClient, pipeline.Name, + filters.Index, pipeline.Spec.Filters, + weighers.Index, pipeline.Spec.Weighers, + novaController.Monitor, + ) + if len(result.FilterErrors) > 0 || len(result.WeigherErrors) > 0 { + t.Fatalf("Failed to init pipeline %s: filters=%v, weighers=%v", pipeline.Name, result.FilterErrors, result.WeigherErrors) + } + novaController.Pipelines[pipeline.Name] = result.Pipeline + novaController.PipelineConfigs[pipeline.Name] = pipeline + } + + api := &testHTTPAPI{delegate: novaController} + mux := http.NewServeMux() + mux.HandleFunc("/scheduler/nova/external", api.NovaExternalScheduler) + server := httptest.NewServer(mux) + + return &IntegrationTestEnv{ + T: t, + Scheme: scheme, + K8sClient: k8sClient, + Server: server, + NovaController: novaController, + VMSource: NewMockVMSource(vms), + SchedulerBaseURL: server.URL, + } +} + +// testHTTPAPI is a simplified HTTP API for testing that delegates to the controller. +type testHTTPAPI struct { + delegate nova.HTTPAPIDelegate +} + +// NovaExternalScheduler handles the POST request from the Nova scheduler. +func (api *testHTTPAPI) NovaExternalScheduler(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "invalid request method", http.StatusMethodNotAllowed) + return + } + + defer r.Body.Close() + + var requestData novaapi.ExternalSchedulerRequest + if err := json.NewDecoder(r.Body).Decode(&requestData); err != nil { + http.Error(w, "failed to decode request body", http.StatusBadRequest) + return + } + + rawBytes, err := json.Marshal(requestData) + if err != nil { + http.Error(w, "failed to marshal request", http.StatusInternalServerError) + return + } + raw := runtime.RawExtension{Raw: rawBytes} + + pipelineName := requestData.Pipeline + if pipelineName == "" { + pipelineName = "nova-external-scheduler-kvm-all-filters-enabled" + } + + decision := &v1alpha1.Decision{ + TypeMeta: metav1.TypeMeta{ + Kind: "Decision", + APIVersion: "cortex.cloud/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "nova-", + }, + Spec: v1alpha1.DecisionSpec{ + SchedulingDomain: v1alpha1.SchedulingDomainNova, + PipelineRef: corev1.ObjectReference{ + Name: pipelineName, + }, + ResourceID: requestData.Spec.Data.InstanceUUID, + NovaRaw: &raw, + }, + } + + ctx := r.Context() + if err := api.delegate.ProcessNewDecisionFromAPI(ctx, decision); err != nil { + http.Error(w, "failed to process scheduling decision", http.StatusInternalServerError) + return + } + + if decision.Status.Result == nil { + http.Error(w, "decision didn't produce a result", http.StatusInternalServerError) + return + } + + hosts := decision.Status.Result.OrderedHosts + response := novaapi.ExternalSchedulerResponse{Hosts: hosts} + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(response); err != nil { + http.Error(w, "failed to encode response", http.StatusInternalServerError) + return + } +} + +// ============================================================================ +// Object Creation Functions +// ============================================================================ + +// defaultTestAZ is the default availability zone used for tests when not explicitly specified. +const defaultTestAZ = "az-a" + +// newHypervisor creates a Hypervisor CRD with the given parameters. +// Uses defaultTestAZ as the availability zone. +func newHypervisor(name string, cpuCap, memoryGi, cpuAlloc, memoryGiAlloc int, instances []hv1.Instance, traits []string) *hv1.Hypervisor { + return newHypervisorWithAZ(name, cpuCap, memoryGi, cpuAlloc, memoryGiAlloc, instances, traits, defaultTestAZ) +} + +// newHypervisorWithAZ creates a Hypervisor CRD with the given parameters including availability zone. +func newHypervisorWithAZ(name string, cpuCap, memoryGi, cpuAlloc, memoryGiAlloc int, instances []hv1.Instance, traits []string, az string) *hv1.Hypervisor { + labels := make(map[string]string) + if az != "" { + labels[corev1.LabelTopologyZone] = az + } + capacity := map[hv1.ResourceName]resource.Quantity{ + "cpu": resource.MustParse(strconv.Itoa(cpuCap)), + "memory": resource.MustParse(strconv.Itoa(memoryGi) + "Gi"), + } + return &hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: labels, + }, + Spec: hv1.HypervisorSpec{ + Overcommit: map[hv1.ResourceName]float64{ + "cpu": 1.0, + "memory": 1.0, + }, + }, + Status: hv1.HypervisorStatus{ + Capacity: capacity, + EffectiveCapacity: capacity, + Allocation: map[hv1.ResourceName]resource.Quantity{"cpu": resource.MustParse(strconv.Itoa(cpuAlloc)), "memory": resource.MustParse(strconv.Itoa(memoryGiAlloc) + "Gi")}, + NumInstances: len(instances), + Instances: instances, + Traits: traits, + }, + } +} + +// newReservation creates a Reservation CRD with the given parameters. +func newReservation(name, host string, memoryMB, vcpus uint64, allocations map[string]string) *v1alpha1.Reservation { //nolint:unparam + return &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelFailover, + "cortex.cloud/creator": "test", + }, + }, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailover, + TargetHost: host, + Resources: map[hv1.ResourceName]resource.Quantity{ + "memory": resource.MustParse(strconv.FormatUint(memoryMB, 10) + "Mi"), + "cpu": resource.MustParse(strconv.FormatUint(vcpus, 10)), + }, + FailoverReservation: &v1alpha1.FailoverReservationSpec{ + ResourceGroup: "m1.large", + }, + }, + Status: v1alpha1.ReservationStatus{ + Conditions: []metav1.Condition{ + { + Type: v1alpha1.ReservationConditionReady, + Status: metav1.ConditionTrue, + Reason: "ReservationActive", + }, + }, + Host: host, + FailoverReservation: &v1alpha1.FailoverReservationStatus{ + Allocations: allocations, + }, + }, + } +} + +// newVM creates a VM with the given parameters. +// Uses defaultTestAZ as the availability zone. +func newVM(uuid, flavorName, projectID, host string, memoryMB, vcpus uint64) VM { //nolint:unparam + return newVMWithAZ(uuid, flavorName, projectID, host, memoryMB, vcpus, defaultTestAZ) +} + +// newVMWithAZ creates a VM with the given parameters including availability zone. +func newVMWithAZ(uuid, flavorName, projectID, host string, memoryMB, vcpus uint64, az string) VM { + return VM{ + UUID: uuid, + FlavorName: flavorName, + ProjectID: projectID, + CurrentHypervisor: host, + AvailabilityZone: az, + Resources: map[string]resource.Quantity{ + "memory": resource.MustParse(strconv.FormatUint(memoryMB, 10) + "Mi"), + "vcpus": resource.MustParse(strconv.FormatUint(vcpus, 10)), + }, + FlavorExtraSpecs: make(map[string]string), + } +} + +// newVMWithExtraSpecs creates a VM with the given parameters including extra specs. +// Uses defaultTestAZ as the availability zone. +func newVMWithExtraSpecs(uuid, flavorName, projectID, host string, memoryMB, vcpus uint64, extraSpecs map[string]string) VM { //nolint:unparam + return newVMWithExtraSpecsAndAZ(uuid, flavorName, projectID, host, memoryMB, vcpus, extraSpecs, defaultTestAZ) +} + +// newVMWithExtraSpecsAndAZ creates a VM with the given parameters including extra specs and availability zone. +func newVMWithExtraSpecsAndAZ(uuid, flavorName, projectID, host string, memoryMB, vcpus uint64, extraSpecs map[string]string, az string) VM { + vm := newVMWithAZ(uuid, flavorName, projectID, host, memoryMB, vcpus, az) + if extraSpecs != nil { + vm.FlavorExtraSpecs = extraSpecs + } + return vm +} + +// generateHostCombinations generates all combinations of hosts up to size n. +func generateHostCombinations(hosts []string, maxSize int) [][]string { + var result [][]string + for size := 1; size <= maxSize && size <= len(hosts); size++ { + result = append(result, combinations(hosts, size)...) + } + return result +} + +// combinations generates all combinations of size k from the given slice. +func combinations(items []string, k int) [][]string { + if k == 0 { + return [][]string{{}} + } + if len(items) < k { + return nil + } + + var result [][]string + for _, combo := range combinations(items[1:], k-1) { + result = append(result, append([]string{items[0]}, combo...)) + } + result = append(result, combinations(items[1:], k)...) + return result +} diff --git a/internal/scheduling/reservations/failover/reservation_eligibility.go b/internal/scheduling/reservations/failover/reservation_eligibility.go new file mode 100644 index 000000000..d62aee424 --- /dev/null +++ b/internal/scheduling/reservations/failover/reservation_eligibility.go @@ -0,0 +1,258 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failover + +import ( + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" +) + +// DependencyGraph encapsulates the data structures needed for eligibility checking. +// It tracks relationships between VMs, reservations, and hypervisors. +type DependencyGraph struct { + // vmToReservations maps vm_uuid -> set of reservation keys (namespace/name) the VM uses + vmToReservations map[string]map[string]bool + // vmToCurrentHypervisor maps vm_uuid -> current hypervisor (where VM is now) + vmToCurrentHypervisor map[string]string + // vmToReservationHosts maps vm_uuid -> set of reservation hosts (where VM could evacuate) + vmToReservationHosts map[string]map[string]bool + // reservationToVMs maps reservation key (namespace/name) -> set of vm_uuids using it + reservationToVMs map[string]map[string]bool + // reservationToHost maps reservation key (namespace/name) -> hypervisor where reservation is placed + reservationToHost map[string]string +} + +// reservationKey returns a unique key for a reservation (namespace/name). +// This prevents collisions between same-named reservations in different namespaces. +func reservationKey(namespace, name string) string { + return namespace + "/" + name +} + +// newDependencyGraph builds a DependencyGraph for eligibility checking. +// The VM is treated as already being part of the candidate reservation. +func newDependencyGraph( + vm VM, + candidateReservation v1alpha1.Reservation, + allFailoverReservations []v1alpha1.Reservation, +) *DependencyGraph { + + g := &DependencyGraph{ + vmToReservations: make(map[string]map[string]bool), + vmToCurrentHypervisor: make(map[string]string), + vmToReservationHosts: make(map[string]map[string]bool), + reservationToVMs: make(map[string]map[string]bool), + reservationToHost: make(map[string]string), + } + + // Process all reservations + for _, res := range allFailoverReservations { + resKey := reservationKey(res.Namespace, res.Name) + resHost := res.Status.Host + + g.ensureResInMaps(resKey) + g.reservationToHost[resKey] = resHost + + allocations := getFailoverAllocations(&res) + for vmUUID, vmHypervisor := range allocations { + g.ensureVMInMaps(vmUUID) + g.vmToReservations[vmUUID][resKey] = true + g.vmToCurrentHypervisor[vmUUID] = vmHypervisor + g.vmToReservationHosts[vmUUID][resHost] = true + g.reservationToVMs[resKey][vmUUID] = true + } + } + + // Process candidate reservation (may not be in allFailoverReservations) + candidateResKey := reservationKey(candidateReservation.Namespace, candidateReservation.Name) + candidateResHost := candidateReservation.Status.Host + + g.ensureResInMaps(candidateResKey) + g.reservationToHost[candidateResKey] = candidateResHost + + candidateAllocations := getFailoverAllocations(&candidateReservation) + for vmUUID, vmHypervisor := range candidateAllocations { + g.ensureVMInMaps(vmUUID) + g.vmToReservations[vmUUID][candidateResKey] = true + g.vmToCurrentHypervisor[vmUUID] = vmHypervisor + g.vmToReservationHosts[vmUUID][candidateResHost] = true + g.reservationToVMs[candidateResKey][vmUUID] = true + } + + // Add the VM we're checking with its current hypervisor + g.ensureVMInMaps(vm.UUID) + g.vmToCurrentHypervisor[vm.UUID] = vm.CurrentHypervisor + + // KEY: Treat VM as already in the candidate reservation + g.vmToReservations[vm.UUID][candidateResKey] = true + g.vmToReservationHosts[vm.UUID][candidateResHost] = true + g.reservationToVMs[candidateResKey][vm.UUID] = true + + return g +} + +func (g *DependencyGraph) ensureVMInMaps(vmUUID string) { + if g.vmToReservations[vmUUID] == nil { + g.vmToReservations[vmUUID] = make(map[string]bool) + } + if g.vmToReservationHosts[vmUUID] == nil { + g.vmToReservationHosts[vmUUID] = make(map[string]bool) + } +} + +func (g *DependencyGraph) ensureResInMaps(resName string) { + if g.reservationToVMs[resName] == nil { + g.reservationToVMs[resName] = make(map[string]bool) + } +} + +// checkAllVMConstraints checks if a single VM satisfies all constraints (1-5). +// Returns true if the VM satisfies all constraints. +func (g *DependencyGraph) checkAllVMConstraints(vmUUID string) bool { + vmCurrentHypervisor := g.vmToCurrentHypervisor[vmUUID] + vmSlotHypervisors := g.vmToReservationHosts[vmUUID] + + // Constraint (1): A VM cannot reserve a slot on its own hypervisor + if vmSlotHypervisors[vmCurrentHypervisor] { + return false + } + + // Constraint (2): A VM's N reservation slots must be on N distinct hypervisors + numReservations := len(g.vmToReservations[vmUUID]) + numUniqueHosts := len(g.vmToReservationHosts[vmUUID]) + if numUniqueHosts < numReservations { + return false + } + + // Collect all VMs (other than this VM) that use any of this VM's slots + vmsUsingVMSlotsCount := make(map[string]int) + for resName := range g.vmToReservations[vmUUID] { + for otherVM := range g.reservationToVMs[resName] { + if otherVM != vmUUID { + vmsUsingVMSlotsCount[otherVM]++ + } + } + } + + // Constraint (4): Any other VM that uses this VM's slots must not run on: + // - this VM's current hypervisor + // - any of this VM's slot hypervisors + for otherVM := range vmsUsingVMSlotsCount { + otherVMCurrentHypervisor := g.vmToCurrentHypervisor[otherVM] + + if otherVMCurrentHypervisor == vmCurrentHypervisor { + return false + } + + if vmSlotHypervisors[otherVMCurrentHypervisor] { + return false + } + } + + // Constraint (5): No two VMs (other than this VM) using this VM's slots can be on the same hypervisor + hypervisorToVM := make(map[string]string) + for otherVM, slotCount := range vmsUsingVMSlotsCount { + if slotCount > 1 { + return false + } + + otherVMCurrentHypervisor := g.vmToCurrentHypervisor[otherVM] + if existingVM, exists := hypervisorToVM[otherVMCurrentHypervisor]; exists && existingVM != otherVM { + return false + } + hypervisorToVM[otherVMCurrentHypervisor] = otherVM + } + + return true +} + +// isVMEligibleForReservation checks if all VMs in the candidate reservation satisfy constraints. +func (g *DependencyGraph) isVMEligibleForReservation(candidateResName string) bool { + for vmUUID := range g.reservationToVMs[candidateResName] { + if !g.checkAllVMConstraints(vmUUID) { + return false + } + } + return true +} + +// IsVMEligibleForReservation checks if a VM is eligible to use a specific reservation. +// A VM is eligible if it satisfies all the following constraints: +// (1) A VM cannot reserve a slot on its own hypervisor. +// (2) A VM's N reservation slots must be placed on N distinct hypervisors (no hypervisor overlap among slots). +// (3) For any reservation r, no two VMs that use r may be on the same hypervisor (directly or potentially via a reservation). +// (4) For VM v with slots R = {r1..rn}, any other VM vi that uses any rj must not run on hs(v) nor on any hs(rj). +// (5) For VM v with slots R = {r1..rn}, there exist no vm_1, vm_2 (vm_1 != v and vm_2 != v) +// +// with vm_1 uses r_j and vm_2 uses r_k and hypervisor(vm_1) = hypervisor(vm_2). +func IsVMEligibleForReservation(vm VM, reservation v1alpha1.Reservation, allFailoverReservations []v1alpha1.Reservation) bool { + // Check if VM is already using this reservation + resAllocations := getFailoverAllocations(&reservation) + if _, exists := resAllocations[vm.UUID]; exists { + return false + } + + // Ensure the candidate reservation is included in allFailoverReservations + reservationInList := false + for _, res := range allFailoverReservations { + if res.Name == reservation.Name && res.Namespace == reservation.Namespace { + reservationInList = true + break + } + } + if !reservationInList { + allFailoverReservations = append(append([]v1alpha1.Reservation{}, allFailoverReservations...), reservation) + } + + // Build dependency graph (with VM already in the candidate reservation) + graph := newDependencyGraph(vm, reservation, allFailoverReservations) + + // Check all constraints for all VMs in the reservation + return graph.isVMEligibleForReservation(reservationKey(reservation.Namespace, reservation.Name)) +} + +// doesVMFitInReservation checks if a VM's resources fit within a reservation's resources. +// Returns true if all VM resources are less than or equal to the reservation's resources. +func doesVMFitInReservation(vm VM, reservation v1alpha1.Reservation) bool { + // Check memory: VM's memory must be <= reservation's memory + if vmMemory, ok := vm.Resources["memory"]; ok { + if resMemory, ok := reservation.Spec.Resources[hv1.ResourceMemory]; ok { + if vmMemory.Cmp(resMemory) > 0 { + return false // VM memory exceeds reservation memory + } + } else { + return false // Reservation has no memory resource defined + } + } + + // Check CPU: VM's vcpus must be <= reservation's cpu + // Note: VM uses "vcpus" key, but reservations use "cpu" as the canonical key. + if vmVCPUs, ok := vm.Resources["vcpus"]; ok { + if resCPU, ok := reservation.Spec.Resources[hv1.ResourceCPU]; ok { + if vmVCPUs.Cmp(resCPU) > 0 { + return false // VM vcpus exceeds reservation cpu + } + } else { + return false // Reservation has no cpu resource defined + } + } + + return true +} + +// FindEligibleReservations finds all reservations that a VM is eligible to use. +// It checks both resource fit and eligibility constraints. +func FindEligibleReservations( + vm VM, + failoverReservations []v1alpha1.Reservation, +) []v1alpha1.Reservation { + //TODO: we create data mappings inside IsVMEligibleForReservation those should probably be done already on this level to avoid redundant work + var eligible []v1alpha1.Reservation + for _, res := range failoverReservations { + if doesVMFitInReservation(vm, res) && IsVMEligibleForReservation(vm, res, failoverReservations) { + eligible = append(eligible, res) + } + } + + return eligible +} diff --git a/internal/scheduling/reservations/failover/reservation_eligibility_test.go b/internal/scheduling/reservations/failover/reservation_eligibility_test.go new file mode 100644 index 000000000..ddae548cc --- /dev/null +++ b/internal/scheduling/reservations/failover/reservation_eligibility_test.go @@ -0,0 +1,1425 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failover + +import ( + "testing" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Default resources for test VMs and reservations (4GB memory, 2 vcpus) +// Note: Reservations use "cpu" as the canonical key, VMs use "vcpus" +var defaultResources = map[hv1.ResourceName]resource.Quantity{ + "memory": resource.MustParse("4Gi"), + "cpu": resource.MustParse("2"), +} + +var defaultVMResources = map[string]resource.Quantity{ + "memory": resource.MustParse("4Gi"), + "vcpus": resource.MustParse("2"), +} + +// makeReservation creates a test reservation with the given parameters. +func makeReservation(name, host string, usedBy map[string]string) v1alpha1.Reservation { + return v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailover, + TargetHost: host, + Resources: defaultResources, + }, + Status: v1alpha1.ReservationStatus{ + Host: host, + FailoverReservation: &v1alpha1.FailoverReservationStatus{ + Allocations: usedBy, + }, + }, + } +} + +// makeReservationWithResources creates a test reservation with custom resources. +func makeReservationWithResources(name, host string, usedBy map[string]string, resources map[hv1.ResourceName]resource.Quantity) v1alpha1.Reservation { //nolint:unparam // name is always "res-1" in tests but kept for clarity + return v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailover, + TargetHost: host, + Resources: resources, + }, + Status: v1alpha1.ReservationStatus{ + Host: host, + FailoverReservation: &v1alpha1.FailoverReservationStatus{ + Allocations: usedBy, + }, + }, + } +} + +// makeVM creates a test VM with the given parameters. +func makeVM(uuid, hypervisor string) VM { + return VM{ + UUID: uuid, + CurrentHypervisor: hypervisor, + Resources: defaultVMResources, + } +} + +// makeVMWithResources creates a test VM with custom resources. +func makeVMWithResources(uuid, hypervisor string, resources map[string]resource.Quantity) VM { //nolint:unparam // uuid is always "vm-1" in tests but kept for clarity + return VM{ + UUID: uuid, + CurrentHypervisor: hypervisor, + Resources: resources, + } +} + +// buildVMHypervisorsMap builds a map of VM UUID to their hypervisors from failover reservations. +// It also includes the VM we are checking (vm) with its current hypervisor, +// and the candidate reservation (which may have VMs not in allFailoverReservations). +// This is a test helper function used to verify data structure consistency. +func buildVMHypervisorsMap(vm VM, candidateReservation v1alpha1.Reservation, allFailoverReservations []v1alpha1.Reservation) map[string]map[string]bool { + vmHypervisorsMap := make(map[string]map[string]bool) + + vmHypervisorsMap[vm.UUID] = make(map[string]bool) + vmHypervisorsMap[vm.UUID][vm.CurrentHypervisor] = true + + // Add VMs from reservation allocations + for _, res := range allFailoverReservations { + allocations := getFailoverAllocations(&res) + for vmUUID, vmHypervisor := range allocations { + if vmHypervisorsMap[vmUUID] == nil { + vmHypervisorsMap[vmUUID] = make(map[string]bool) + } + vmHypervisorsMap[vmUUID][vmHypervisor] = true + } + } + + // Add VMs from the candidate reservation + candidateAllocations := getFailoverAllocations(&candidateReservation) + for vmUUID, vmHypervisor := range candidateAllocations { + if vmHypervisorsMap[vmUUID] == nil { + vmHypervisorsMap[vmUUID] = make(map[string]bool) + } + vmHypervisorsMap[vmUUID][vmHypervisor] = true + } + + return vmHypervisorsMap +} + +// TestIsVMEligibleForReservation tests the IsVMEligibleForReservation function with various scenarios. +func TestIsVMEligibleForReservation(t *testing.T) { + testCases := []struct { + name string + vm VM + reservation v1alpha1.Reservation + vmHostMap map[string]string + allReservations []v1alpha1.Reservation + expected bool + }{ + // ============================================================================ + // Basic eligibility tests + // ============================================================================ + { + name: "eligible: empty reservation on different host", + vm: makeVM("vm-1", "host1"), + reservation: makeReservation("res-1", "host2", map[string]string{}), + vmHostMap: map[string]string{ + "vm-1": "host1", + }, + allReservations: []v1alpha1.Reservation{}, + expected: true, + }, + { + name: "eligible: reservation not in allReservations list", + vm: makeVM("vm-1", "host1"), + reservation: makeReservation("res-new", "host2", map[string]string{}), + vmHostMap: map[string]string{ + "vm-1": "host1", + }, + allReservations: []v1alpha1.Reservation{ + makeReservation("res-other", "host3", map[string]string{"vm-2": "host2"}), + }, + expected: true, + }, + { + name: "eligible: empty allReservations with non-empty candidate", + vm: makeVM("vm-2", "host2"), + reservation: makeReservation("res-1", "host3", map[string]string{"vm-1": "host1"}), + vmHostMap: map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + }, + allReservations: []v1alpha1.Reservation{}, + expected: true, + }, + { + name: "ineligible: VM already uses this reservation", + vm: makeVM("vm-1", "host1"), + reservation: makeReservation("res-1", "host2", map[string]string{"vm-1": "host1"}), + vmHostMap: map[string]string{ + "vm-1": "host1", + }, + allReservations: []v1alpha1.Reservation{}, + expected: false, + }, + // ============================================================================ + // Constraint 1: VM cannot reserve on its own host + // ============================================================================ + { + name: "C1: ineligible - reservation on VM's own host", + vm: makeVM("vm-1", "host1"), + reservation: makeReservation("res-1", "host1", map[string]string{}), + vmHostMap: map[string]string{ + "vm-1": "host1", + }, + allReservations: []v1alpha1.Reservation{}, + expected: false, + }, + { + name: "C1: ineligible - reservation on VM's own host with other VMs", + vm: makeVM("vm-1", "host1"), + reservation: makeReservation("res-1", "host1", map[string]string{"vm-2": "host2"}), + vmHostMap: map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + }, + allReservations: []v1alpha1.Reservation{}, + expected: false, + }, + // ============================================================================ + // Constraint 2: VM's reservations must be on distinct hosts + // ============================================================================ + { + name: "C2: ineligible - VM already has reservation on same host", + vm: makeVM("vm-1", "host1"), + reservation: makeReservation("res-2", "host3", map[string]string{}), + vmHostMap: map[string]string{ + "vm-1": "host1", + }, + allReservations: []v1alpha1.Reservation{ + makeReservation("res-1", "host3", map[string]string{"vm-1": "host1"}), + }, + expected: false, + }, + { + name: "C2: eligible - VM has reservations on different hosts", + vm: makeVM("vm-1", "host1"), + reservation: makeReservation("res-2", "host4", map[string]string{}), + vmHostMap: map[string]string{ + "vm-1": "host1", + }, + allReservations: []v1alpha1.Reservation{ + makeReservation("res-1", "host3", map[string]string{"vm-1": "host1"}), + }, + expected: true, + }, + { + name: "C2: ineligible - VM has 2 reservations, third would be on duplicate host", + vm: makeVM("vm-1", "host1"), + reservation: makeReservation("res-3", "host3", map[string]string{}), + vmHostMap: map[string]string{ + "vm-1": "host1", + }, + allReservations: []v1alpha1.Reservation{ + makeReservation("res-1", "host3", map[string]string{"vm-1": "host1"}), + makeReservation("res-2", "host4", map[string]string{"vm-1": "host1"}), + }, + expected: false, + }, + { + name: "C2: eligible - VM has 2 reservations on different hosts, third on new host", + vm: makeVM("vm-1", "host1"), + reservation: makeReservation("res-3", "host5", map[string]string{}), + vmHostMap: map[string]string{ + "vm-1": "host1", + }, + allReservations: []v1alpha1.Reservation{ + makeReservation("res-1", "host3", map[string]string{"vm-1": "host1"}), + makeReservation("res-2", "host4", map[string]string{"vm-1": "host1"}), + }, + expected: true, + }, + { + name: "C3: eligible - VM can share reservation with VM on different host", + vm: makeVM("vm-3", "host3"), + reservation: makeReservation("res-1", "host2", map[string]string{"vm-1": "host1"}), + vmHostMap: map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + "vm-3": "host3", + }, + allReservations: []v1alpha1.Reservation{ + makeReservation("res-1", "host2", map[string]string{"vm-1": "host1"}), + }, + expected: true, + }, + // ============================================================================ + // Constraint 3 extended: VMs cannot share if one has reservation on other's host + // ============================================================================ + { + name: "C3ext: ineligible - VM has reservation on other VM's host", + vm: makeVM("vm-3", "host3"), + reservation: makeReservation("res-2", "host5", map[string]string{"vm-1": "host1"}), + vmHostMap: map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + "vm-3": "host3", + "vm-4": "host4", + }, + allReservations: []v1alpha1.Reservation{ + makeReservation("res-1", "host1", map[string]string{"vm-3": "host3"}), + makeReservation("res-2", "host5", map[string]string{"vm-1": "host1"}), + }, + expected: false, + }, + // ============================================================================ + // Constraint 4: VMs using shared reservation cannot run on VM's reservation hosts + // ============================================================================ + { + name: "C4: ineligible - reservation user runs on VM's reservation host", + vm: makeVM("vm-1", "host1"), + reservation: makeReservation("res-2", "host3", map[string]string{"vm-2": "host2"}), + vmHostMap: map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + }, + allReservations: []v1alpha1.Reservation{ + makeReservation("res-1", "host2", map[string]string{"vm-1": "host1"}), + }, + expected: false, + }, + { + name: "C4: ineligible - vm_b runs on vm_a's reservation host", + vm: makeVM("vm_a", "host1"), + reservation: makeReservation("res_k", "host3", map[string]string{"vm_b": "host2"}), + vmHostMap: map[string]string{ + "vm_a": "host1", + "vm_b": "host2", + }, + allReservations: []v1alpha1.Reservation{ + makeReservation("res_l", "host2", map[string]string{"vm_a": "host1"}), + makeReservation("res_k", "host3", map[string]string{"vm_b": "host2"}), + makeReservation("res_m", "host4", map[string]string{"vm_b": "host2"}), + }, + expected: false, + }, + // ============================================================================ + // Constraint 5: No two VMs (other than v) using v's slots can have same host + // For VM v with slots R = {r1..rn}, there exist no vm_j, vm_k (vm_j != v and vm_k != v) + // with vm_j uses r_j and vm_k uses r_k and host(vm_j) = host(vm_k). + // Note: vm_j and vm_k CAN be the same VM (same VM using multiple slots violates this) + // ============================================================================ + { + name: "C5: ineligible - two different VMs using v's slots on same host", + vm: makeVM("vm-1", "host1"), + reservation: makeReservation("res-2", "host4", map[string]string{"vm-3": "host2"}), + vmHostMap: map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + "vm-3": "host2", + }, + allReservations: []v1alpha1.Reservation{ + makeReservation("res-1", "host3", map[string]string{"vm-1": "host1", "vm-2": "host2"}), + }, + expected: false, + }, + { + name: "C5: ineligible - vm_b and vm_c both use vm_a's slots and are on same host", + vm: makeVM("vm_a", "host1"), + reservation: makeReservation("res_k", "host2", map[string]string{"vm_b": "host4"}), + vmHostMap: map[string]string{ + "vm_a": "host1", + "vm_b": "host4", + "vm_c": "host4", + }, + allReservations: []v1alpha1.Reservation{ + makeReservation("res_m", "host1", map[string]string{"vm_b": "host4"}), + makeReservation("res_n", "host1", map[string]string{"vm_c": "host4"}), + makeReservation("res_k", "host2", map[string]string{"vm_b": "host4"}), + makeReservation("res_l", "host3", map[string]string{"vm_a": "host1", "vm_c": "host4"}), + }, + expected: false, + }, + { + name: "C5: ineligible - vm-1 would use multiple of vm-2's slots", + vm: makeVM("vm-2", "host2"), + reservation: makeReservation("res-5", "host5", map[string]string{ + "vm-1": "host1", + }), + vmHostMap: map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + "vm-3": "host3", + "vm-4": "host4", + }, + allReservations: []v1alpha1.Reservation{ + makeReservation("res-4", "host4", map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + "vm-3": "host3", + }), + makeReservation("res-5", "host5", map[string]string{ + "vm-1": "host1", + }), + }, + // vm-1 would use both res-4 and res-5 (two of vm-2's slots) + // vm_j = vm-1 uses res-4, vm_k = vm-1 uses res-5, host(vm_j) = host(vm_k) = host1 → VIOLATION + expected: false, + }, + { + name: "C5: ineligible - vm-1 would use both res-3 and res-4 (vm-2's slots)", + vm: makeVM("vm-2", "host2"), + reservation: makeReservation("res-4", "host4", map[string]string{ + "vm-1": "host1", + }), + vmHostMap: map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + }, + allReservations: []v1alpha1.Reservation{ + makeReservation("res-3", "host3", map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + }), + makeReservation("res-4", "host4", map[string]string{ + "vm-1": "host1", + }), + }, + // vm-1 would use both res-3 and res-4 (two of vm-2's slots) + // vm_j = vm-1 uses res-3, vm_k = vm-1 uses res-4, host(vm_j) = host(vm_k) = host1 → VIOLATION + expected: false, + }, + { + name: "C5: eligible - vm-1 only uses one of vm-2's slots", + vm: makeVM("vm-2", "host2"), + reservation: makeReservation("res-4", "host4", map[string]string{ + "vm-1": "host1", + }), + vmHostMap: map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + }, + allReservations: []v1alpha1.Reservation{ + makeReservation("res-3", "host3", map[string]string{ + "vm-1": "host1", + }), + makeReservation("res-4", "host4", map[string]string{ + "vm-1": "host1", + }), + makeReservation("res-5", "host5", map[string]string{ + "vm-2": "host2", + }), + }, + expected: true, + }, + { + name: "C5: ineligible - vm-1 and vm-3 both use vm-2's slots and vm-1 is on host1", + vm: makeVM("vm-2", "host2"), + reservation: makeReservation("res-1", "host1", map[string]string{ + "vm-3": "host3", + }), + vmHostMap: map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + "vm-3": "host3", + "vm-4": "host4", + }, + allReservations: []v1alpha1.Reservation{ + makeReservation("res-3", "host3", map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + }), + makeReservation("res-6", "host6", map[string]string{ + "vm-1": "host1", + "vm-3": "host3", + "vm-4": "host4", + }), + makeReservation("res-1", "host1", map[string]string{ + "vm-3": "host3", + }), + }, + expected: false, + }, + // ============================================================================ + // Constraint 3: VMs sharing a reservation cannot be on the same host + // ============================================================================ + { + name: "C3: ineligible - another VM on same host uses reservation", + vm: makeVM("vm-1", "host1"), + reservation: makeReservation("res-1", "host3", map[string]string{"vm-2": "host1"}), + vmHostMap: map[string]string{ + "vm-1": "host1", + "vm-2": "host1", + }, + allReservations: []v1alpha1.Reservation{}, + expected: false, + }, + { + name: "C3: eligible - other VMs on different hosts", + vm: makeVM("vm-1", "host1"), + reservation: makeReservation("res-1", "host3", map[string]string{"vm-2": "host2"}), + vmHostMap: map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + }, + allReservations: []v1alpha1.Reservation{}, + expected: true, + }, + { + // vm-2 wants to use res-1 (empty) on host1. If vm-2 uses res-1: + // - vm-2's slots: res-3 on host3 (existing), res-1 on host1 (candidate) + // - vm-2's slot hosts: {host3, host1} + // - VMs using vm-2's slots: vm-1 uses res-3 (on host1) + // - Constraint 4: vm-1 is on host1, which is in vm-2's slot hosts → VIOLATION! + name: "C4: ineligible - vm-1 uses vm-2's slot and runs on candidate reservation's host (empty res)", + vm: makeVM("vm-2", "host2"), + reservation: makeReservation("res-1", "host1", map[string]string{}), + vmHostMap: map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + "vm-3": "host3", + "vm-4": "host4", + }, + allReservations: []v1alpha1.Reservation{ + makeReservation("res-3", "host3", map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + }), + makeReservation("res-4", "host4", map[string]string{ + "vm-1": "host1", + }), + makeReservation("res-1", "host1", map[string]string{}), + }, + expected: false, + }, + // ============================================================================ + // Integration test scenario: vm-3 should be able to use reservation on host1 + // ============================================================================ + { + // Scenario from integration test: + // - host1 and host3 failed + // - vm-1 on host1, vm-3 on host3 need evacuation + // - existing-res-1 on host4 has: vm-1, vm-3 + // - existing-res-2 on host5 has: vm-1, vm-2 + // - failover-zxmbh on host1 has: vm-2, vm-3 + // should be able to use failover-zxmbh on host1 (but host1 failed, so this is moot) + // vm-3 should be able to use existing-res-1 on host4 + // But vm-1 is also using existing-res-1, and both are evacuating + // This test checks if vm-3 can use the reservation on host1 when vm-3 is NOT yet in it + name: "integration: vm-3 ineligible for reservation on host1 (constraint violation)", + vm: makeVM("vm-3", "host3"), + reservation: makeReservation("failover-zxmbh", "host1", map[string]string{"vm-2": "host2"}), + vmHostMap: map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + "vm-3": "host3", + }, + allReservations: []v1alpha1.Reservation{ + makeReservation("existing-res-1", "host4", map[string]string{"vm-1": "host1", "vm-3": "host3"}), + makeReservation("existing-res-2", "host5", map[string]string{"vm-1": "host1", "vm-2": "host2"}), + makeReservation("failover-zxmbh", "host1", map[string]string{"vm-2": "host2"}), + }, + // vm-3 wants to use failover-zxmbh on host1 + // Constraint 1: host1 != host3 ✓ + // Constraint 2: vm-3 already has reservations on host4, not host1 ✓ + // Constraint 3: vm-2 uses failover-zxmbh, vm-2 is on host2, vm-3 is on host3 ✓ + // Constraint 4: vm-2 (using failover-zxmbh) is on host2, vm-3's reservation hosts are [host4] + // vm-2 is not on host3 (vm-3's host) ✓ + // vm-2 is not on host4 (vm-3's reservation host) ✓ + // Constraint 5: VMs using vm-3's slots (existing-res-1, failover-zxmbh): + // existing-res-1: vm-1 on host1 + // failover-zxmbh: vm-2 on host2 + // vm-1 and vm-2 are on different hosts ✓ + expected: false, + }, + // ============================================================================ + // Circular dependency scenarios + // ============================================================================ + { + name: "circular: ineligible - vm-3 has res on vm-1's host, vm-1 has res on vm-3's host", + vm: makeVM("vm-3", "host3"), + reservation: makeReservation("res-2", "host2", map[string]string{ + "vm-1": "host1", + }), + vmHostMap: map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + "vm-3": "host3", + "vm-4": "host4", + }, + allReservations: []v1alpha1.Reservation{ + makeReservation("res-1", "host1", map[string]string{ + "vm-3": "host3", + }), + makeReservation("res-3", "host3", map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + "vm-4": "host4", + }), + makeReservation("res-2", "host2", map[string]string{ + "vm-1": "host1", + }), + }, + expected: false, + }, + { + name: "circular: ineligible - vm-3 has res on vm-1's host, wants to share with vm-1", + vm: makeVM("vm-3", "host3"), + reservation: makeReservation("res-2", "host2", map[string]string{ + "vm-1": "host1", + "vm-4": "host4", + }), + vmHostMap: map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + "vm-3": "host3", + "vm-4": "host4", + }, + allReservations: []v1alpha1.Reservation{ + makeReservation("res-1", "host1", map[string]string{ + "vm-3": "host3", + }), + makeReservation("res-2", "host2", map[string]string{ + "vm-1": "host1", + "vm-4": "host4", + }), + makeReservation("res-6", "host6", map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + }), + }, + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // The new API builds VMHostsMap from the VM and allReservations + // No need to add temp reservations - the VM's host is included automatically + result := IsVMEligibleForReservation(tc.vm, tc.reservation, tc.allReservations) + + if result != tc.expected { + t.Errorf("IsVMEligibleForReservation() = %v, expected %v", result, tc.expected) + } + }) + } +} + +// TestDoesVMFitInReservation tests the doesVMFitInReservation function. +func TestDoesVMFitInReservation(t *testing.T) { + testCases := []struct { + name string + vm VM + reservation v1alpha1.Reservation + expected bool + }{ + { + name: "fits: VM fits exactly in reservation", + vm: makeVMWithResources("vm-1", "host1", map[string]resource.Quantity{ + "memory": resource.MustParse("4Gi"), + "vcpus": resource.MustParse("2"), + }), + reservation: makeReservationWithResources("res-1", "host2", map[string]string{}, map[hv1.ResourceName]resource.Quantity{ + "memory": resource.MustParse("4Gi"), + "cpu": resource.MustParse("2"), + }), + expected: true, + }, + { + name: "fits: VM is smaller than reservation", + vm: makeVMWithResources("vm-1", "host1", map[string]resource.Quantity{ + "memory": resource.MustParse("2Gi"), + "vcpus": resource.MustParse("1"), + }), + reservation: makeReservationWithResources("res-1", "host2", map[string]string{}, map[hv1.ResourceName]resource.Quantity{ + "memory": resource.MustParse("4Gi"), + "cpu": resource.MustParse("2"), + }), + expected: true, + }, + { + name: "exceeds: VM memory exceeds reservation", + vm: makeVMWithResources("vm-1", "host1", map[string]resource.Quantity{ + "memory": resource.MustParse("8Gi"), + "vcpus": resource.MustParse("2"), + }), + reservation: makeReservationWithResources("res-1", "host2", map[string]string{}, map[hv1.ResourceName]resource.Quantity{ + "memory": resource.MustParse("4Gi"), + "cpu": resource.MustParse("2"), + }), + expected: false, + }, + { + name: "exceeds: VM vcpus exceeds reservation cpu", + vm: makeVMWithResources("vm-1", "host1", map[string]resource.Quantity{ + "memory": resource.MustParse("4Gi"), + "vcpus": resource.MustParse("4"), + }), + reservation: makeReservationWithResources("res-1", "host2", map[string]string{}, map[hv1.ResourceName]resource.Quantity{ + "memory": resource.MustParse("4Gi"), + "cpu": resource.MustParse("2"), + }), + expected: false, + }, + { + name: "fits: VM has no resources defined", + vm: makeVMWithResources("vm-1", "host1", map[string]resource.Quantity{}), + reservation: makeReservationWithResources("res-1", "host2", map[string]string{}, map[hv1.ResourceName]resource.Quantity{ + "memory": resource.MustParse("4Gi"), + "cpu": resource.MustParse("2"), + }), + expected: true, + }, + { + name: "exceeds: reservation has no memory resource", + vm: makeVMWithResources("vm-1", "host1", map[string]resource.Quantity{ + "memory": resource.MustParse("4Gi"), + "vcpus": resource.MustParse("2"), + }), + reservation: makeReservationWithResources("res-1", "host2", map[string]string{}, map[hv1.ResourceName]resource.Quantity{ + "cpu": resource.MustParse("2"), + }), + expected: false, + }, + { + name: "exceeds: reservation has no cpu resource", + vm: makeVMWithResources("vm-1", "host1", map[string]resource.Quantity{ + "memory": resource.MustParse("4Gi"), + "vcpus": resource.MustParse("2"), + }), + reservation: makeReservationWithResources("res-1", "host2", map[string]string{}, map[hv1.ResourceName]resource.Quantity{ + "memory": resource.MustParse("4Gi"), + }), + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := doesVMFitInReservation(tc.vm, tc.reservation) + + if result != tc.expected { + t.Errorf("doesVMFitInReservation() = %v, expected %v", result, tc.expected) + } + }) + } +} + +// updateReservationInList returns a new slice with the given reservation updated. +func updateReservationInList(reservations []v1alpha1.Reservation, updated v1alpha1.Reservation) []v1alpha1.Reservation { + result := make([]v1alpha1.Reservation, len(reservations)) + for i, res := range reservations { + if res.Name == updated.Name { + result[i] = updated + } else { + result[i] = res + } + } + return result +} + +// checkAllExistingVMsRemainEligible checks that after adding newVM to a reservation, +// all existing VMs in that reservation remain eligible. +// Returns (allEligible, failedVMUUID, reason). +func checkAllExistingVMsRemainEligible( + newVM VM, + reservation v1alpha1.Reservation, + allReservations []v1alpha1.Reservation, +) (allEligible bool, failedVMUUID, reason string) { + // Get existing allocations + existingAllocations := reservation.Status.FailoverReservation.Allocations + if existingAllocations == nil { + return true, "", "" // No existing VMs to check + } + + // Simulate adding newVM to the reservation + updatedRes := reservation.DeepCopy() + if updatedRes.Status.FailoverReservation == nil { + updatedRes.Status.FailoverReservation = &v1alpha1.FailoverReservationStatus{} + } + if updatedRes.Status.FailoverReservation.Allocations == nil { + updatedRes.Status.FailoverReservation.Allocations = make(map[string]string) + } + updatedRes.Status.FailoverReservation.Allocations[newVM.UUID] = newVM.CurrentHypervisor + + // Update allReservations with the modified reservation + updatedAllRes := updateReservationInList(allReservations, *updatedRes) + + // Check each existing VM in the reservation + for vmUUID, vmHost := range existingAllocations { + existingVM := VM{UUID: vmUUID, CurrentHypervisor: vmHost, Resources: defaultVMResources} + + // Temporarily remove the VM to check if it can be "re-added" + // This mimics what reconcileRemoveNoneligibleVMFromReservations does + tempRes := updatedRes.DeepCopy() + delete(tempRes.Status.FailoverReservation.Allocations, vmUUID) + tempAllRes := updateReservationInList(updatedAllRes, *tempRes) + + if !IsVMEligibleForReservation(existingVM, *tempRes, tempAllRes) { + return false, vmUUID, "VM became ineligible after adding " + newVM.UUID + } + } + return true, "", "" +} + +// TestAddingVMDoesNotMakeOthersIneligible tests that when a VM is eligible to be added +// to a reservation, adding it does not make existing VMs in that reservation ineligible. +// This is a critical invariant - if violated, the system will oscillate between adding +// and removing VMs from reservations. +func TestAddingVMDoesNotMakeOthersIneligible(t *testing.T) { + testCases := []struct { + name string + vm VM + reservation v1alpha1.Reservation + allReservations []v1alpha1.Reservation + vmIsEligible bool // Expected result of IsVMEligibleForReservation + existingVMsStayEligible bool // Expected: all existing VMs should stay eligible + failingVM string // If existingVMsStayEligible is false, which VM fails + }{ + // ============================================================================ + // Cases where VM is eligible and existing VMs should stay eligible + // ============================================================================ + { + name: "simple: add VM to empty reservation", + vm: makeVM("vm-2", "host2"), + reservation: makeReservation("res-1", "host3", map[string]string{}), + allReservations: []v1alpha1.Reservation{ + makeReservation("res-1", "host3", map[string]string{}), + }, + vmIsEligible: true, + existingVMsStayEligible: true, + }, + { + name: "simple: add VM to reservation with one VM on different host", + vm: makeVM("vm-2", "host2"), + reservation: makeReservation("res-1", "host3", map[string]string{"vm-1": "host1"}), + allReservations: []v1alpha1.Reservation{ + makeReservation("res-1", "host3", map[string]string{"vm-1": "host1"}), + }, + vmIsEligible: true, + existingVMsStayEligible: true, + }, + // ============================================================================ + // Cases where VM would make existing VMs ineligible if added + // ============================================================================ + { + // Scenario: vm-3 is eligible to join res-A, but vm-3 also uses res-B. + // vm-1 already uses res-A. If vm-3 joins res-A, then vm-1's slots include res-A. + // vm-3 uses res-A (one of vm-1's slots) and res-B. + // Actually, vm-3 already uses res-B which vm-1 also uses, so vm-3 would use + // two of vm-1's slots (res-A and res-B) -> constraint 5 violation! + name: "ineligible: vm-3 already shares res-B with vm-1, cannot join res-A (constraint 5)", + vm: makeVM("vm-3", "host3"), + reservation: makeReservation("res-A", "host4", map[string]string{ + "vm-1": "host1", + }), + allReservations: []v1alpha1.Reservation{ + makeReservation("res-A", "host4", map[string]string{"vm-1": "host1"}), + makeReservation("res-B", "host5", map[string]string{"vm-1": "host1", "vm-3": "host3"}), + }, + vmIsEligible: false, // vm-3 would use two of vm-1's slots (res-A and res-B) + existingVMsStayEligible: true, // N/A since vm-3 can't join + }, + { + // Scenario with n=2: Each VM needs 2 reservations + // vm-1 on host1 has res-A (host3) and res-B (host4) + // vm-2 on host2 has res-A (host3) and res-C (host5) + // vm-3 on host2 wants to join res-B + // After vm-3 joins res-B: + // - vm-1's slots: res-A, res-B + // - VMs using vm-1's slots (other than vm-1): vm-2 (uses res-A), vm-3 (uses res-B) + // - vm-2 is on host2, vm-3 is on host2 -> SAME HOST! + // - Constraint 5 violated for vm-1! + name: "ineligible: vm-3 on same host as vm-2 cannot join res-B (constraint 5)", + vm: makeVM("vm-3", "host2"), // Same host as vm-2! + reservation: makeReservation("res-B", "host4", map[string]string{ + "vm-1": "host1", + }), + allReservations: []v1alpha1.Reservation{ + makeReservation("res-A", "host3", map[string]string{"vm-1": "host1", "vm-2": "host2"}), + makeReservation("res-B", "host4", map[string]string{"vm-1": "host1"}), + makeReservation("res-C", "host5", map[string]string{"vm-2": "host2"}), + }, + vmIsEligible: false, // EXPECTED: vm-3 should NOT be eligible (would break vm-1) + existingVMsStayEligible: true, // If vm-3 can't join, existing VMs stay eligible + }, + { + // Another scenario: vm-3 joins res-A where vm-1 and vm-2 already are + // vm-1 on host1, vm-2 on host2, vm-3 on host3 + // res-A on host4 has vm-1 and vm-2 + // vm-3 wants to join res-A + // vm-3 also has res-B on host5 + // After vm-3 joins: + // - For vm-1: vm-1's slots include res-A + // - VMs using res-A (other than vm-1): vm-2, vm-3 + // - vm-2 on host2, vm-3 on host3 -> different hosts, OK + name: "OK: vm-3 joins res-A with vm-1 and vm-2, all on different hosts", + vm: makeVM("vm-3", "host3"), + reservation: makeReservation("res-A", "host4", map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + }), + allReservations: []v1alpha1.Reservation{ + makeReservation("res-A", "host4", map[string]string{"vm-1": "host1", "vm-2": "host2"}), + makeReservation("res-B", "host5", map[string]string{"vm-3": "host3"}), + }, + vmIsEligible: true, + existingVMsStayEligible: true, + }, + { + // Constraint 1 violation scenario: + // vm-1 on host1 has res-A (host3) + // vm-2 on host3 (same as res-A's host!) wants to join res-A + // Constraint 1 says VM cannot reserve on its own host + // vm-2 is on host3, res-A is on host3 -> vm-2 is NOT eligible! + name: "ineligible: vm-2 on reservation host cannot join res-A (constraint 1)", + vm: makeVM("vm-2", "host3"), // Same as res-A's host! + reservation: makeReservation("res-A", "host3", map[string]string{ + "vm-1": "host1", + }), + allReservations: []v1alpha1.Reservation{ + makeReservation("res-A", "host3", map[string]string{"vm-1": "host1"}), + }, + // Constraint 1: VM cannot reserve on its own host + // vm-2 is on host3, res-A is on host3 -> vm-2 is NOT eligible + vmIsEligible: false, // Constraint 1 catches this + existingVMsStayEligible: true, // N/A since vm-2 can't join + }, + { + // Constraint 4 violation scenario: + // vm-1 on host1 has res-A (host3) and res-B (host4) + // vm-2 on host4 (same as res-B's host!) wants to join res-A + // After adding vm-2 to res-A: + // - For vm-1: vm-1's slots are res-A (host3) and res-B (host4) + // - VMs using vm-1's slots (other than vm-1): vm-2 uses res-A + // - Constraint 4: vm-2 must not run on vm-1's host (host1) or vm-1's slot hosts (host3, host4) + // - vm-2 is on host4, which is vm-1's slot host -> VIOLATION! + name: "ineligible: vm-2 on vm-1's slot host cannot join res-A (constraint 4)", + vm: makeVM("vm-2", "host4"), // Same as vm-1's res-B host! + reservation: makeReservation("res-A", "host3", map[string]string{ + "vm-1": "host1", + }), + allReservations: []v1alpha1.Reservation{ + makeReservation("res-A", "host3", map[string]string{"vm-1": "host1"}), + makeReservation("res-B", "host4", map[string]string{"vm-1": "host1"}), + }, + vmIsEligible: false, // EXPECTED: vm-2 should NOT be eligible (would break vm-1) + existingVMsStayEligible: true, // If vm-2 can't join, existing VMs stay eligible + }, + // ============================================================================ + // Edge case: VM in OTHER reservations (not candidate) becomes ineligible + // Must check VMs in allFailoverReservations, not just candidate reservation + // ============================================================================ + { + // Scenario: + // vm-1 on host1 has res-A (host3) and res-B (host4) + // vm-2 on host2 has res-B (host4) - shares res-B with vm-1 + // vm-3 on host2 wants to join res-A (which only has vm-1) + // + // After vm-3 joins res-A: + // - For vm-1: vm-1's slots are res-A and res-B + // - VMs using vm-1's slots: vm-3 (uses res-A), vm-2 (uses res-B) + // - vm-3 is on host2, vm-2 is on host2 -> SAME HOST! + // - Constraint 5 violated for vm-1! + // + // Must check vm-1 (in candidate res-A), but vm-2 is NOT in res-A. + // vm-2 is in res-B, which is in allFailoverReservations. + name: "ineligible: vm-3 on same host as vm-2 who shares vm-1's slot (constraint 5)", + vm: makeVM("vm-3", "host2"), // Same host as vm-2! + reservation: makeReservation("res-A", "host3", map[string]string{ + "vm-1": "host1", + }), + allReservations: []v1alpha1.Reservation{ + makeReservation("res-A", "host3", map[string]string{"vm-1": "host1"}), + makeReservation("res-B", "host4", map[string]string{"vm-1": "host1", "vm-2": "host2"}), + }, + vmIsEligible: false, // EXPECTED: vm-3 should NOT be eligible (would break vm-1) + existingVMsStayEligible: true, // If vm-3 can't join, existing VMs stay eligible + }, + // ============================================================================ + // Complex scenarios with n=3 (3 reservations per VM) + // ============================================================================ + { + // Scenario with n=3: + // vm-1 on host1 has res-A (host4), res-B (host5), res-C (host6) + // vm-2 on host2 has res-A (host4), res-D (host7), res-E (host8) + // vm-3 on host3 wants to join res-B (which only has vm-1) + // + // After vm-3 joins res-B: + // - For vm-1: vm-1's slots are res-A, res-B, res-C + // - VMs using vm-1's slots: vm-2 (uses res-A), vm-3 (uses res-B) + // - vm-2 on host2, vm-3 on host3 -> different hosts, OK + name: "n=3: OK - vm-3 joins res-B, vm-2 uses res-A, different hosts", + vm: makeVM("vm-3", "host3"), + reservation: makeReservation("res-B", "host5", map[string]string{ + "vm-1": "host1", + }), + allReservations: []v1alpha1.Reservation{ + makeReservation("res-A", "host4", map[string]string{"vm-1": "host1", "vm-2": "host2"}), + makeReservation("res-B", "host5", map[string]string{"vm-1": "host1"}), + makeReservation("res-C", "host6", map[string]string{"vm-1": "host1"}), + makeReservation("res-D", "host7", map[string]string{"vm-2": "host2"}), + makeReservation("res-E", "host8", map[string]string{"vm-2": "host2"}), + }, + vmIsEligible: true, + existingVMsStayEligible: true, + }, + { + // Scenario with n=3 - constraint 5 violation: + // vm-1 on host1 has res-A (host4), res-B (host5), res-C (host6) + // vm-2 on host2 has res-A (host4) + // vm-3 on host2 wants to join res-B (which only has vm-1) + // + // After vm-3 joins res-B: + // - For vm-1: vm-1's slots are res-A, res-B, res-C + // - VMs using vm-1's slots: vm-2 (uses res-A), vm-3 (uses res-B) + // - vm-2 on host2, vm-3 on host2 -> SAME HOST! + // - Constraint 5 violated for vm-1! + name: "n=3: ineligible - vm-3 on same host as vm-2 who uses vm-1's slot (constraint 5)", + vm: makeVM("vm-3", "host2"), // Same host as vm-2! + reservation: makeReservation("res-B", "host5", map[string]string{ + "vm-1": "host1", + }), + allReservations: []v1alpha1.Reservation{ + makeReservation("res-A", "host4", map[string]string{"vm-1": "host1", "vm-2": "host2"}), + makeReservation("res-B", "host5", map[string]string{"vm-1": "host1"}), + makeReservation("res-C", "host6", map[string]string{"vm-1": "host1"}), + }, + vmIsEligible: false, // EXPECTED: vm-3 should NOT be eligible (would break vm-1) + existingVMsStayEligible: true, + }, + { + // Scenario with n=3 - constraint 4 violation: + // vm-1 on host1 has res-A (host4), res-B (host5), res-C (host6) + // vm-2 on host5 (same as res-B!) wants to join res-A + // + // After vm-2 joins res-A: + // - For vm-1: vm-1's slots are res-A, res-B, res-C + // - VMs using vm-1's slots: vm-2 (uses res-A) + // - Constraint 4: vm-2 must not run on vm-1's slot hosts (host4, host5, host6) + // - vm-2 is on host5, which is vm-1's slot host -> VIOLATION! + name: "n=3: ineligible - vm-2 on vm-1's slot host cannot join res-A (constraint 4)", + vm: makeVM("vm-2", "host5"), // Same as vm-1's res-B host! + reservation: makeReservation("res-A", "host4", map[string]string{ + "vm-1": "host1", + }), + allReservations: []v1alpha1.Reservation{ + makeReservation("res-A", "host4", map[string]string{"vm-1": "host1"}), + makeReservation("res-B", "host5", map[string]string{"vm-1": "host1"}), + makeReservation("res-C", "host6", map[string]string{"vm-1": "host1"}), + }, + vmIsEligible: false, // EXPECTED: vm-2 should NOT be eligible (would break vm-1) + existingVMsStayEligible: true, + }, + // ============================================================================ + // Edge case: VM NOT in candidate reservation is affected + // This tests if the fix correctly handles VMs that share slots with VMs in the candidate + // ============================================================================ + { + // Scenario: + // vm-1 on host1 has res-A (host4) and res-B (host5) + // vm-2 on host2 has res-A (host4) and res-C (host6) + // vm-3 on host1 (same as vm-1!) wants to join res-C (which only has vm-2) + // + // After vm-3 joins res-C: + // - For vm-2 (in res-C): vm-2's slots are res-A and res-C + // - VMs using vm-2's slots: vm-1 (uses res-A), vm-3 (uses res-C) + // - vm-1 on host1, vm-3 on host1 -> SAME HOST! + // - Constraint 5 violated for vm-2! + // + // This is caught because vm-2 is in res-C (the candidate reservation). + name: "edge: vm-3 joins res-C, makes vm-2 ineligible (vm-1 and vm-3 same host)", + vm: makeVM("vm-3", "host1"), // Same host as vm-1! + reservation: makeReservation("res-C", "host6", map[string]string{ + "vm-2": "host2", + }), + allReservations: []v1alpha1.Reservation{ + makeReservation("res-A", "host4", map[string]string{"vm-1": "host1", "vm-2": "host2"}), + makeReservation("res-B", "host5", map[string]string{"vm-1": "host1"}), + makeReservation("res-C", "host6", map[string]string{"vm-2": "host2"}), + }, + vmIsEligible: false, // EXPECTED: vm-3 should NOT be eligible (would break vm-2) + existingVMsStayEligible: true, + }, + { + // Scenario: 4 VMs, complex sharing + // vm-1 on host1 has res-A (host5), res-B (host6) + // vm-2 on host2 has res-A (host5), res-C (host7) + // vm-3 on host3 has res-B (host6), res-C (host7) + // vm-4 on host4 wants to join res-A + // + // After vm-4 joins res-A: + // - For vm-1 (in res-A): vm-1's slots are res-A, res-B + // - VMs using vm-1's slots: vm-2 (uses res-A), vm-3 (uses res-B), vm-4 (uses res-A) + // - vm-2 on host2, vm-3 on host3, vm-4 on host4 -> all different hosts, OK + // - For vm-2 (in res-A): vm-2's slots are res-A, res-C + // - VMs using vm-2's slots: vm-1 (uses res-A), vm-3 (uses res-C), vm-4 (uses res-A) + // - vm-1 on host1, vm-3 on host3, vm-4 on host4 -> all different hosts, OK + name: "complex: 4 VMs, vm-4 joins res-A, all different hosts - OK", + vm: makeVM("vm-4", "host4"), + reservation: makeReservation("res-A", "host5", map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + }), + allReservations: []v1alpha1.Reservation{ + makeReservation("res-A", "host5", map[string]string{"vm-1": "host1", "vm-2": "host2"}), + makeReservation("res-B", "host6", map[string]string{"vm-1": "host1", "vm-3": "host3"}), + makeReservation("res-C", "host7", map[string]string{"vm-2": "host2", "vm-3": "host3"}), + }, + vmIsEligible: true, + existingVMsStayEligible: true, + }, + { + // Scenario: 4 VMs, complex sharing - constraint 5 violation + // vm-1 on host1 has res-A (host5), res-B (host6) + // vm-2 on host2 has res-A (host5), res-C (host7) + // vm-3 on host3 has res-B (host6), res-C (host7) + // vm-4 on host3 (same as vm-3!) wants to join res-A + // + // After vm-4 joins res-A: + // - For vm-1 (in res-A): vm-1's slots are res-A, res-B + // - VMs using vm-1's slots: vm-2 (uses res-A), vm-3 (uses res-B), vm-4 (uses res-A) + // - vm-3 on host3, vm-4 on host3 -> SAME HOST! + // - Constraint 5 violated for vm-1! + name: "complex: ineligible - vm-4 on same host as vm-3 who uses vm-1's slot (constraint 5)", + vm: makeVM("vm-4", "host3"), // Same host as vm-3! + reservation: makeReservation("res-A", "host5", map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + }), + allReservations: []v1alpha1.Reservation{ + makeReservation("res-A", "host5", map[string]string{"vm-1": "host1", "vm-2": "host2"}), + makeReservation("res-B", "host6", map[string]string{"vm-1": "host1", "vm-3": "host3"}), + makeReservation("res-C", "host7", map[string]string{"vm-2": "host2", "vm-3": "host3"}), + }, + vmIsEligible: false, // EXPECTED: vm-4 should NOT be eligible (would break vm-1) + existingVMsStayEligible: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // First, verify the VM's eligibility matches expectation + isEligible := IsVMEligibleForReservation(tc.vm, tc.reservation, tc.allReservations) + if isEligible != tc.vmIsEligible { + t.Errorf("IsVMEligibleForReservation() = %v, expected %v", isEligible, tc.vmIsEligible) + return + } + + // If VM is not eligible, skip the "existing VMs stay eligible" check + if !isEligible { + return + } + + // Check that all existing VMs remain eligible after adding the new VM + allStayEligible, failedVM, reason := checkAllExistingVMsRemainEligible( + tc.vm, tc.reservation, tc.allReservations, + ) + + if allStayEligible != tc.existingVMsStayEligible { + if tc.existingVMsStayEligible { + t.Errorf("Expected all existing VMs to stay eligible, but %s failed: %s", failedVM, reason) + } else { + t.Errorf("Expected VM %s to become ineligible, but all VMs stayed eligible", tc.failingVM) + } + } + + if !allStayEligible && tc.failingVM != "" && failedVM != tc.failingVM { + t.Errorf("Expected VM %s to become ineligible, but VM %s failed instead", tc.failingVM, failedVM) + } + }) + } +} + +// TestSymmetryOfEligibility tests that eligibility constraints are symmetric. +// If vm-A can share a reservation with vm-B, then vm-B should be able to share with vm-A +// (assuming they have equivalent reservation setups). +func TestSymmetryOfEligibility(t *testing.T) { + testCases := []struct { + name string + vm1 VM + vm2 VM + // vm1Reservation is the reservation to check for vm1's eligibility + vm1Reservation v1alpha1.Reservation + // vm2Reservation is the reservation to check for vm2's eligibility + vm2Reservation v1alpha1.Reservation + // allReservationsForVM1 is the context when checking vm1's eligibility + allReservationsForVM1 []v1alpha1.Reservation + // allReservationsForVM2 is the context when checking vm2's eligibility + allReservationsForVM2 []v1alpha1.Reservation + vm1Eligible bool + vm2Eligible bool + }{ + { + name: "symmetric: both VMs can join empty reservation", + vm1: makeVM("vm-1", "host1"), + vm2: makeVM("vm-2", "host2"), + vm1Reservation: makeReservation("res-1", "host3", map[string]string{}), + vm2Reservation: makeReservation("res-1", "host3", map[string]string{}), + allReservationsForVM1: []v1alpha1.Reservation{ + makeReservation("res-1", "host3", map[string]string{}), + }, + allReservationsForVM2: []v1alpha1.Reservation{ + makeReservation("res-1", "host3", map[string]string{}), + }, + vm1Eligible: true, + vm2Eligible: true, + }, + { + name: "symmetric: vm-1 in res, vm-2 can join; vm-2 in res, vm-1 can join", + vm1: makeVM("vm-1", "host1"), + vm2: makeVM("vm-2", "host2"), + // Check if vm-1 can join res-1 when vm-2 is already in it + vm1Reservation: makeReservation("res-1", "host3", map[string]string{"vm-2": "host2"}), + // Check if vm-2 can join res-1 when vm-1 is already in it + vm2Reservation: makeReservation("res-1", "host3", map[string]string{"vm-1": "host1"}), + allReservationsForVM1: []v1alpha1.Reservation{ + makeReservation("res-1", "host3", map[string]string{"vm-2": "host2"}), + }, + allReservationsForVM2: []v1alpha1.Reservation{ + makeReservation("res-1", "host3", map[string]string{"vm-1": "host1"}), + }, + vm1Eligible: true, + vm2Eligible: true, + }, + { + name: "asymmetric: vm-1 has res on host2, vm-2 cannot join res on host3 with vm-1", + vm1: makeVM("vm-1", "host1"), + vm2: makeVM("vm-2", "host2"), + // vm-1 is already in res-2, so we check if vm-1 can join a different reservation + // For this test, vm-1 is already in res-2, so vm1Eligible is about whether vm-1 could join res-2 (it can't, it's already in it) + vm1Reservation: makeReservation("res-2", "host3", map[string]string{"vm-1": "host1"}), + vm2Reservation: makeReservation("res-2", "host3", map[string]string{"vm-1": "host1"}), + allReservationsForVM1: []v1alpha1.Reservation{ + makeReservation("res-1", "host2", map[string]string{"vm-1": "host1"}), + makeReservation("res-2", "host3", map[string]string{"vm-1": "host1"}), + }, + allReservationsForVM2: []v1alpha1.Reservation{ + makeReservation("res-1", "host2", map[string]string{"vm-1": "host1"}), + makeReservation("res-2", "host3", map[string]string{"vm-1": "host1"}), + }, + // vm-1 is already in res-2, so it's not eligible to join again + vm1Eligible: false, + // vm-2 wants to join res-2 which has vm-1 + // has res-1 on host2, vm-2 is on host2 + // Constraint 4: vm-2 runs on vm-1's slot host (host2) -> VIOLATION + vm2Eligible: false, + }, + { + name: "symmetric: both VMs on different hosts can share reservation", + vm1: makeVM("vm-1", "host1"), + vm2: makeVM("vm-2", "host2"), + // Check if vm-1 can join res-1 when vm-2 is already in it + vm1Reservation: makeReservation("res-1", "host3", map[string]string{"vm-2": "host2"}), + // Check if vm-2 can join res-1 when vm-1 is already in it + vm2Reservation: makeReservation("res-1", "host3", map[string]string{"vm-1": "host1"}), + allReservationsForVM1: []v1alpha1.Reservation{ + makeReservation("res-1", "host3", map[string]string{"vm-2": "host2"}), + }, + allReservationsForVM2: []v1alpha1.Reservation{ + makeReservation("res-1", "host3", map[string]string{"vm-1": "host1"}), + }, + vm1Eligible: true, + vm2Eligible: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Check vm1's eligibility to join the reservation + vm1Result := IsVMEligibleForReservation(tc.vm1, tc.vm1Reservation, tc.allReservationsForVM1) + if vm1Result != tc.vm1Eligible { + t.Errorf("vm1 eligibility: got %v, expected %v", vm1Result, tc.vm1Eligible) + } + + // Check vm2's eligibility to join the reservation + vm2Result := IsVMEligibleForReservation(tc.vm2, tc.vm2Reservation, tc.allReservationsForVM2) + if vm2Result != tc.vm2Eligible { + t.Errorf("vm2 eligibility: got %v, expected %v", vm2Result, tc.vm2Eligible) + } + }) + } +} + +// TestDataStructureConsistency tests that the internal data structures +// produce consistent results. This test will help verify the refactoring. +func TestDataStructureConsistency(t *testing.T) { + // This test verifies that the helper functions produce consistent results + // with the main IsVMEligibleForReservation function. + + testCases := []struct { + name string + vm VM + reservation v1alpha1.Reservation + allReservations []v1alpha1.Reservation + }{ + { + name: "simple case", + vm: makeVM("vm-1", "host1"), + reservation: makeReservation("res-1", "host2", map[string]string{}), + allReservations: []v1alpha1.Reservation{ + makeReservation("res-1", "host2", map[string]string{}), + }, + }, + { + name: "complex case with multiple VMs and reservations", + vm: makeVM("vm-3", "host3"), + reservation: makeReservation("res-A", "host4", map[string]string{"vm-1": "host1", "vm-2": "host2"}), + allReservations: []v1alpha1.Reservation{ + makeReservation("res-A", "host4", map[string]string{"vm-1": "host1", "vm-2": "host2"}), + makeReservation("res-B", "host5", map[string]string{"vm-1": "host1"}), + makeReservation("res-C", "host6", map[string]string{"vm-2": "host2"}), + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Get the result from the main function + result := IsVMEligibleForReservation(tc.vm, tc.reservation, tc.allReservations) + + // Verify helper functions produce consistent data + vmHypervisorsMap := buildVMHypervisorsMap(tc.vm, tc.reservation, tc.allReservations) + + // Verify the VM is in the map + if _, exists := vmHypervisorsMap[tc.vm.UUID]; !exists { + t.Errorf("VM %s not found in vmHypervisorsMap", tc.vm.UUID) + } + + // Verify the VM's current hypervisor is in the map + if !vmHypervisorsMap[tc.vm.UUID][tc.vm.CurrentHypervisor] { + t.Errorf("VM %s's current hypervisor %s not in vmHypervisorsMap", tc.vm.UUID, tc.vm.CurrentHypervisor) + } + + // Log the result for debugging + t.Logf("VM %s eligibility for %s: %v", tc.vm.UUID, tc.reservation.Name, result) + }) + } +} + +// TestFindEligibleReservations tests the FindEligibleReservations function. +func TestFindEligibleReservations(t *testing.T) { + testCases := []struct { + name string + vm VM + failoverReservations []v1alpha1.Reservation + vmHostMap map[string]string + expectedCount int + expectedHosts []string + }{ + { + name: "none: no reservations available", + vm: makeVM("vm-1", "host1"), + failoverReservations: []v1alpha1.Reservation{}, + vmHostMap: map[string]string{ + "vm-1": "host1", + }, + expectedCount: 0, + expectedHosts: nil, + }, + { + name: "one: single eligible reservation", + vm: makeVM("vm-2", "host2"), + failoverReservations: []v1alpha1.Reservation{ + makeReservation("res-1", "host3", map[string]string{"vm-1": "host1"}), + }, + vmHostMap: map[string]string{ + "vm-1": "host1", + "vm-2": "host2", + }, + expectedCount: 1, + expectedHosts: []string{"host3"}, + }, + { + name: "multiple: two eligible reservations", + vm: makeVM("vm-3", "host3"), + failoverReservations: []v1alpha1.Reservation{ + makeReservation("res-1", "host1", map[string]string{"vm-1": "host2"}), + makeReservation("res-2", "host2", map[string]string{"vm-2": "host1"}), + }, + vmHostMap: map[string]string{ + "vm-1": "host2", + "vm-2": "host1", + "vm-3": "host3", + }, + expectedCount: 2, + expectedHosts: []string{"host1", "host2"}, + }, + { + name: "none: all reservations on VM's host", + vm: makeVM("vm-1", "host1"), + failoverReservations: []v1alpha1.Reservation{ + makeReservation("res-1", "host1", map[string]string{}), + }, + vmHostMap: map[string]string{ + "vm-1": "host1", + }, + expectedCount: 0, + expectedHosts: nil, + }, + { + name: "none: VM already uses the reservation", + vm: makeVM("vm-1", "host1"), + failoverReservations: []v1alpha1.Reservation{ + makeReservation("res-1", "host2", map[string]string{"vm-1": "host1"}), + }, + vmHostMap: map[string]string{ + "vm-1": "host1", + }, + expectedCount: 0, + expectedHosts: nil, + }, + { + name: "filtered: one eligible after filtering", + vm: makeVM("vm-1", "host1"), + failoverReservations: []v1alpha1.Reservation{ + makeReservation("res-1", "host2", map[string]string{"vm-2": "host1"}), + makeReservation("res-2", "host3", map[string]string{"vm-3": "host2"}), + }, + vmHostMap: map[string]string{ + "vm-1": "host1", + "vm-2": "host1", + "vm-3": "host2", + }, + expectedCount: 1, + expectedHosts: []string{"host3"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // The new API builds VMHostsMap from the VM and failoverReservations + // No need to add temp reservations - the VM's host is included automatically + result := FindEligibleReservations(tc.vm, tc.failoverReservations) + + if len(result) != tc.expectedCount { + t.Errorf("FindEligibleReservations() returned %d reservations, expected %d", len(result), tc.expectedCount) + } + + if tc.expectedHosts != nil { + resultHosts := make([]string, len(result)) + for i, res := range result { + resultHosts[i] = res.Status.Host + } + + for _, expectedHost := range tc.expectedHosts { + found := false + for _, resultHost := range resultHosts { + if resultHost == expectedHost { + found = true + break + } + } + if !found { + t.Errorf("Expected host %s not found in results %v", expectedHost, resultHosts) + } + } + } + }) + } +} diff --git a/internal/scheduling/reservations/failover/reservation_scheduling.go b/internal/scheduling/reservations/failover/reservation_scheduling.go new file mode 100644 index 000000000..41ab34a14 --- /dev/null +++ b/internal/scheduling/reservations/failover/reservation_scheduling.go @@ -0,0 +1,300 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failover + +import ( + "context" + "fmt" + "slices" + "sort" + + api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" +) + +// Pipeline names for failover reservation scheduling +const ( + // PipelineReuseFailoverReservation is used to check if a VM can reuse an existing reservation. + // It validates host compatibility without checking capacity (since reservation already has capacity). + PipelineReuseFailoverReservation = "kvm-valid-host-reuse-failover-reservation" + + // PipelineNewFailoverReservation is used to find a host for creating a new reservation. + // It validates host compatibility AND checks capacity. + PipelineNewFailoverReservation = "kvm-new-failover-reservation" + + // PipelineAcknowledgeFailoverReservation is used to validate that a failover reservation + // is still valid for all its allocated VMs. It sends an evacuation-style scheduling request + // for each VM with only the reservation's host as the eligible target. + PipelineAcknowledgeFailoverReservation = "kvm-acknowledge-failover-reservation" +) + +func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx context.Context, vm VM, allHypervisors []string, pipeline string) ([]string, error) { + logger := LoggerFromContext(ctx) + + // Build list of eligible hypervisors (excluding VM's current hypervisor) + eligibleHypervisors := make([]api.ExternalSchedulerHost, 0, len(allHypervisors)) + for _, hypervisor := range allHypervisors { + if hypervisor == vm.CurrentHypervisor { + continue // VM's current hypervisor + } + eligibleHypervisors = append(eligibleHypervisors, api.ExternalSchedulerHost{ + ComputeHost: hypervisor, + }) + } + + if len(eligibleHypervisors) == 0 { + logger.Info("no eligible hypervisors for failover reservation", + "vmCurrentHypervisor", vm.CurrentHypervisor) + return nil, fmt.Errorf("no eligible hypervisors for failover reservation (VM is on %s)", vm.CurrentHypervisor) + } + + ignoreHypervisors := []string{vm.CurrentHypervisor} + + // Get memory and vcpus from VM resources + // The VM struct uses "vcpus" and "memory" keys (see vm_source.go) + var memoryMB uint64 + var vcpus uint64 + if memory, ok := vm.Resources["memory"]; ok { + // Convert from bytes to MB + memoryMB = uint64(memory.Value() / (1024 * 1024)) //nolint:gosec // memory values won't overflow + } + if vcpusRes, ok := vm.Resources["vcpus"]; ok { + vcpus = uint64(vcpusRes.Value()) //nolint:gosec // vcpus values won't overflow + } + + // Build flavor extra specs from VM's extra specs + // Start with the VM's actual extra specs, then ensure required defaults are set + flavorExtraSpecs := make(map[string]string) + for k, v := range vm.FlavorExtraSpecs { + flavorExtraSpecs[k] = v + } + // Ensure hypervisor_type is set for KVM scheduling if not already present + if _, ok := flavorExtraSpecs["capabilities:hypervisor_type"]; !ok { + flavorExtraSpecs["capabilities:hypervisor_type"] = "qemu" + } + + // Schedule the reservation using the SchedulerClient. + // Note: We pass all hypervisors (from all AZs) in EligibleHosts. The scheduler pipeline's + // filter_correct_az filter will exclude hosts that are not in the VM's availability zone. + scheduleReq := reservations.ScheduleReservationRequest{ + InstanceUUID: "failover-" + vm.UUID, + ProjectID: vm.ProjectID, + FlavorName: vm.FlavorName, + FlavorExtraSpecs: flavorExtraSpecs, + MemoryMB: memoryMB, + VCPUs: vcpus, + EligibleHosts: eligibleHypervisors, + IgnoreHosts: ignoreHypervisors, + Pipeline: pipeline, + AvailabilityZone: vm.AvailabilityZone, + } + + logger.V(1).Info("scheduling failover reservation", + "vmUUID", vm.UUID, + "pipeline", pipeline, + "eligibleHypervisors", len(eligibleHypervisors), + "ignoreHypervisors", ignoreHypervisors) + + scheduleResp, err := c.SchedulerClient.ScheduleReservation(ctx, scheduleReq) + if err != nil { + logger.Error(err, "failed to schedule failover reservation", "vmUUID", vm.UUID, "pipeline", pipeline) + return nil, fmt.Errorf("failed to schedule failover reservation: %w", err) + } + + logger.V(1).Info("scheduler returned hypervisors for failover reservation", + "vmUUID", vm.UUID, + "pipeline", pipeline, + "eligibleHypervisors", len(eligibleHypervisors), + "ignoreHypervisors", ignoreHypervisors, + "returnedHypervisors", scheduleResp.Hosts) + + return scheduleResp.Hosts, nil +} + +// tryReuseExistingReservation finds an existing reservation that can be reused for a VM. +// It returns a copy of the reservation with the VM added to its allocations (in-memory only, not persisted). +// The original reservation in the input slice is NOT modified. +// The caller is responsible for persisting the changes to the cluster. +func (c *FailoverReservationController) tryReuseExistingReservation( + ctx context.Context, + vm VM, + failoverReservations []v1alpha1.Reservation, + allHypervisors []string, +) *v1alpha1.Reservation { + + logger := LoggerFromContext(ctx) + + validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, PipelineReuseFailoverReservation) + if err != nil { + logger.Error(err, "failed to get potential hypervisors for VM", "vmUUID", vm.UUID) + return nil + } + if len(validHypervisors) == 0 { + logger.Info("no potential hypervisors returned by scheduler for VM", "vmUUID", vm.UUID) + return nil + } + + eligibleReservations := FindEligibleReservations(vm, failoverReservations) + if len(eligibleReservations) == 0 { + logger.Info("no eligible reservations found for VM", "vmUUID", vm.UUID) + return nil + } + + // Sort reservations by number of allocations (prefer reservations with more VMs for better sharing) + sort.Slice(eligibleReservations, func(i, j int) bool { + iAllocs := len(getFailoverAllocations(&eligibleReservations[i])) + jAllocs := len(getFailoverAllocations(&eligibleReservations[j])) + return iAllocs > jAllocs // Descending order - more allocations first + }) + + for _, reservation := range eligibleReservations { + logger.V(2).Info("checking existing reservation for eligibility", + "vmUUID", vm.UUID, + "reservationName", reservation.Name, + "reservationHypervisor", reservation.Status.Host) + if slices.Contains(validHypervisors, reservation.Status.Host) { + // Create a copy of the reservation with the VM added + updatedRes := addVMToReservation(reservation, vm) + logger.V(1).Info("found reusable reservation for VM", + "vmUUID", vm.UUID, + "reservationName", updatedRes.Name, + "hypervisor", updatedRes.Status.Host) + return updatedRes + } + } + + logger.V(1).Info("no reusable reservation found for VM", + "vmUUID", vm.UUID, + "eligibleReservationsCount", len(eligibleReservations), + "validHypervisorsCount", len(validHypervisors)) + return nil +} + +// validateVMViaSchedulerEvacuation sends an evacuation-style scheduling request to validate +// that a VM can use the reservation host. +// TODO this is a bit of a hack. Ideally we have a special kind of request for that which would also verify that we equally are using the reservation +func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( + ctx context.Context, + vm VM, + reservationHost string, +) (bool, error) { + + logger := LoggerFromContext(ctx) + + // Get memory and vcpus from VM resources + var memoryMB uint64 + var vcpus uint64 + if memory, ok := vm.Resources["memory"]; ok { + memoryMB = uint64(memory.Value() / (1024 * 1024)) //nolint:gosec // memory values won't overflow + } + if vcpusRes, ok := vm.Resources["vcpus"]; ok { + vcpus = uint64(vcpusRes.Value()) //nolint:gosec // vcpus values won't overflow + } + + // Build flavor extra specs from VM's extra specs + flavorExtraSpecs := make(map[string]string) + for k, v := range vm.FlavorExtraSpecs { + flavorExtraSpecs[k] = v + } + if _, ok := flavorExtraSpecs["capabilities:hypervisor_type"]; !ok { + flavorExtraSpecs["capabilities:hypervisor_type"] = "qemu" + } + + // Build a single-host request to validate the VM can use the reservation host + // Use vm.CurrentHypervisor directly instead of a separate parameter to avoid stale data + scheduleReq := reservations.ScheduleReservationRequest{ + InstanceUUID: "validate-" + vm.UUID, + ProjectID: vm.ProjectID, + FlavorName: vm.FlavorName, + FlavorExtraSpecs: flavorExtraSpecs, + MemoryMB: memoryMB, + VCPUs: vcpus, + EligibleHosts: []api.ExternalSchedulerHost{{ComputeHost: reservationHost}}, + IgnoreHosts: []string{vm.CurrentHypervisor}, + Pipeline: PipelineAcknowledgeFailoverReservation, + AvailabilityZone: vm.AvailabilityZone, + } + + logger.V(1).Info("validating VM via scheduler evacuation", + "vmUUID", vm.UUID, + "reservationHost", reservationHost, + "vmCurrentHost", vm.CurrentHypervisor, + "pipeline", PipelineAcknowledgeFailoverReservation) + + resp, err := c.SchedulerClient.ScheduleReservation(ctx, scheduleReq) + if err != nil { + logger.Error(err, "failed to validate VM for reservation host", "vmUUID", vm.UUID, "reservationHost", reservationHost) + return false, fmt.Errorf("failed to validate VM for reservation host: %w", err) + } + + // Handle empty response - no hosts returned + if len(resp.Hosts) < 1 { + logger.V(1).Info("scheduler returned no hosts for VM validation", "vmUUID", vm.UUID, "reservationHost", reservationHost) + return false, nil + } + + // Log unexpected scheduler responses + if len(resp.Hosts) > 1 || resp.Hosts[0] != reservationHost { + logger.Error(nil, "scheduler returned unexpected hosts for single-host validation request", + "vmUUID", vm.UUID, + "reservationHost", reservationHost, + "returnedHosts", resp.Hosts) + } + + // If the reservation host is returned, the VM can use it + return resp.Hosts[0] == reservationHost, nil +} + +// scheduleAndBuildNewFailoverReservation schedules a failover reservation for a VM. +// Returns the built reservation (in-memory only, not persisted). +// The caller is responsible for persisting the reservation to the cluster. +func (c *FailoverReservationController) scheduleAndBuildNewFailoverReservation( + ctx context.Context, + vm VM, + allHypervisors []string, + failoverReservations []v1alpha1.Reservation, +) (*v1alpha1.Reservation, error) { + + logger := LoggerFromContext(ctx) + + // Get potential hypervisors from scheduler + validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, PipelineNewFailoverReservation) + if err != nil { + return nil, fmt.Errorf("failed to get potential hypervisors for VM: %w", err) + } + + // Iterate through scheduler-returned hypervisors to find one that passes eligibility constraints + var selectedHypervisor string + for _, candidateHypervisor := range validHypervisors { + // Check if the VM can create a new reservation on this hypervisor + hypotheticalRes := v1alpha1.Reservation{ + Status: v1alpha1.ReservationStatus{ + Host: candidateHypervisor, + // Empty FailoverReservation status - new reservation has no allocations + }, + } + // todo we should update the API to not create a partial reservation object here + if IsVMEligibleForReservation(vm, hypotheticalRes, failoverReservations) { + selectedHypervisor = candidateHypervisor + logger.V(1).Info("VM can create new reservation on hypervisor", "vmUUID", vm.UUID, "hypervisor", candidateHypervisor) + break + } + } + + if selectedHypervisor == "" { + logger.Info("no eligible hypervisors after constraint checking", "vmUUID", vm.UUID, "schedulerReturnedCount", len(validHypervisors)) + return nil, fmt.Errorf("no eligible hypervisors after constraint checking (scheduler returned %d hypervisors, all rejected)", len(validHypervisors)) + } + + logger.V(1).Info("scheduler selected hypervisor for failover reservation", + "vmUUID", vm.UUID, + "selectedHypervisor", selectedHypervisor, + "allReturnedHypervisors", validHypervisors) + + // Build the failover reservation on the selected hypervisor (in-memory only) + reservation := newFailoverReservation(ctx, vm, selectedHypervisor, c.Config.Creator) + + return reservation, nil +} diff --git a/internal/scheduling/reservations/failover/reservation_scheduling_test.go b/internal/scheduling/reservations/failover/reservation_scheduling_test.go new file mode 100644 index 000000000..0ae69f8db --- /dev/null +++ b/internal/scheduling/reservations/failover/reservation_scheduling_test.go @@ -0,0 +1,293 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failover + +import ( + "context" + "testing" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ============================================================================ +// Test: buildReservationWithVM +// ============================================================================ + +func TestBuildReservationWithVM(t *testing.T) { + tests := []struct { + name string + reservation v1alpha1.Reservation + vm VM + wantVMInAllocations bool + wantAllocationsCount int + wantOriginalUnmodified bool + }{ + { + name: "adds VM to empty reservation", + reservation: buildSchedulingTestReservation("res-1", "host2", nil), + vm: buildSchedulingTestVM("vm-1", "host1"), + wantVMInAllocations: true, + wantAllocationsCount: 1, + wantOriginalUnmodified: true, + }, + { + name: "adds VM to reservation with existing allocations", + reservation: buildSchedulingTestReservation("res-1", "host2", map[string]string{"vm-2": "host3"}), + vm: buildSchedulingTestVM("vm-1", "host1"), + wantVMInAllocations: true, + wantAllocationsCount: 2, + wantOriginalUnmodified: true, + }, + { + name: "adds VM to reservation with nil FailoverReservation status", + reservation: buildSchedulingTestReservationNoStatus("res-1", "host2"), + vm: buildSchedulingTestVM("vm-1", "host1"), + wantVMInAllocations: true, + wantAllocationsCount: 1, + wantOriginalUnmodified: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Store original allocations count for verification + originalAllocCount := 0 + if tt.reservation.Status.FailoverReservation != nil { + originalAllocCount = len(tt.reservation.Status.FailoverReservation.Allocations) + } + + result := addVMToReservation(tt.reservation, tt.vm) + + // Verify result has FailoverReservation status + if result.Status.FailoverReservation == nil { + t.Fatal("result has no FailoverReservation status") + } + + // Verify VM is in allocations + allocatedHost, exists := result.Status.FailoverReservation.Allocations[tt.vm.UUID] + if exists != tt.wantVMInAllocations { + t.Errorf("VM in allocations = %v, want %v", exists, tt.wantVMInAllocations) + } + + // Verify allocated host matches VM's current hypervisor + if exists && allocatedHost != tt.vm.CurrentHypervisor { + t.Errorf("allocated host = %v, want %v", allocatedHost, tt.vm.CurrentHypervisor) + } + + // Verify allocations count + if len(result.Status.FailoverReservation.Allocations) != tt.wantAllocationsCount { + t.Errorf("allocations count = %d, want %d", + len(result.Status.FailoverReservation.Allocations), tt.wantAllocationsCount) + } + + // Verify original reservation is not modified + if tt.wantOriginalUnmodified { + currentOriginalCount := 0 + if tt.reservation.Status.FailoverReservation != nil { + currentOriginalCount = len(tt.reservation.Status.FailoverReservation.Allocations) + } + if currentOriginalCount != originalAllocCount { + t.Errorf("original reservation was modified: allocations count changed from %d to %d", + originalAllocCount, currentOriginalCount) + } + } + }) + } +} + +// ============================================================================ +// Test: buildNewFailoverReservation +// ============================================================================ + +func TestBuildNewFailoverReservation(t *testing.T) { + tests := []struct { + name string + vm VM + hypervisor string + wantHost string + wantTargetHost string + wantVMInAlloc bool + wantType v1alpha1.ReservationType + }{ + { + name: "creates reservation with correct host and VM", + vm: buildSchedulingTestVM("vm-1", "host1"), + hypervisor: "host2", + wantHost: "host2", + wantTargetHost: "host2", + wantVMInAlloc: true, + wantType: v1alpha1.ReservationTypeFailover, + }, + { + name: "creates reservation with VM resources", + vm: buildSchedulingTestVMWithResources("vm-2", "host3", 8192, 4), + hypervisor: "host4", + wantHost: "host4", + wantTargetHost: "host4", + wantVMInAlloc: true, + wantType: v1alpha1.ReservationTypeFailover, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + creator := "test-creator" + + result := newFailoverReservation(ctx, tt.vm, tt.hypervisor, creator) + + // Verify Status.Host + if result.Status.Host != tt.wantHost { + t.Errorf("Status.Host = %v, want %v", result.Status.Host, tt.wantHost) + } + + // Verify Spec.TargetHost + if result.Spec.TargetHost != tt.wantTargetHost { + t.Errorf("Spec.TargetHost = %v, want %v", result.Spec.TargetHost, tt.wantTargetHost) + } + + // Verify Type + if result.Spec.Type != tt.wantType { + t.Errorf("Spec.Type = %v, want %v", result.Spec.Type, tt.wantType) + } + + // Verify VM is in allocations + if result.Status.FailoverReservation == nil { + t.Fatal("result has no FailoverReservation status") + } + allocatedHost, exists := result.Status.FailoverReservation.Allocations[tt.vm.UUID] + if exists != tt.wantVMInAlloc { + t.Errorf("VM in allocations = %v, want %v", exists, tt.wantVMInAlloc) + } + if exists && allocatedHost != tt.vm.CurrentHypervisor { + t.Errorf("allocated host = %v, want %v", allocatedHost, tt.vm.CurrentHypervisor) + } + + // Verify resources are copied from VM + // Note: VM uses "vcpus" but reservation uses "cpu" as the canonical key + if tt.vm.Resources != nil { + if memory, ok := tt.vm.Resources["memory"]; ok { + if resMemory, ok := result.Spec.Resources[hv1.ResourceMemory]; !ok { + t.Error("reservation missing memory resource") + } else if !memory.Equal(resMemory) { + t.Errorf("memory resource = %v, want %v", resMemory, memory) + } + } + if vcpus, ok := tt.vm.Resources["vcpus"]; ok { + // VM uses "vcpus" but reservation should use "cpu" + if resCPU, ok := result.Spec.Resources[hv1.ResourceCPU]; !ok { + t.Error("reservation missing cpu resource") + } else if !vcpus.Equal(resCPU) { + t.Errorf("cpu resource = %v, want %v", resCPU, vcpus) + } + } + } + + // Verify labels + if result.Labels["cortex.cloud/creator"] != "test-creator" { + t.Errorf("creator label = %v, want %v", result.Labels["cortex.cloud/creator"], "test-creator") + } + if result.Labels[v1alpha1.LabelReservationType] != v1alpha1.ReservationTypeLabelFailover { + t.Errorf("type label = %v, want %v", result.Labels[v1alpha1.LabelReservationType], v1alpha1.ReservationTypeLabelFailover) + } + + // Verify GenerateName is set + if result.GenerateName != "failover-" { + t.Errorf("GenerateName = %v, want %v", result.GenerateName, "failover-") + } + + // Verify Ready condition is set + if len(result.Status.Conditions) == 0 { + t.Error("no conditions set on reservation") + } else { + foundReady := false + for _, cond := range result.Status.Conditions { + if cond.Type == v1alpha1.ReservationConditionReady { + foundReady = true + if cond.Status != metav1.ConditionTrue { + t.Errorf("Ready condition status = %v, want %v", cond.Status, metav1.ConditionTrue) + } + } + } + if !foundReady { + t.Error("Ready condition not found") + } + } + }) + } +} + +// ============================================================================ +// Test Helpers (local to this test file) +// ============================================================================ + +func buildSchedulingTestReservation(name, host string, allocations map[string]string) v1alpha1.Reservation { + res := v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailover, + TargetHost: host, + Resources: map[hv1.ResourceName]resource.Quantity{ + "memory": resource.MustParse("8Gi"), + "vcpus": resource.MustParse("4"), + }, + }, + Status: v1alpha1.ReservationStatus{ + Host: host, + }, + } + if allocations != nil { + res.Status.FailoverReservation = &v1alpha1.FailoverReservationStatus{ + Allocations: allocations, + } + } + return res +} + +func buildSchedulingTestReservationNoStatus(name, host string) v1alpha1.Reservation { + return v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailover, + TargetHost: host, + Resources: map[hv1.ResourceName]resource.Quantity{ + "memory": resource.MustParse("8Gi"), + "vcpus": resource.MustParse("4"), + }, + }, + Status: v1alpha1.ReservationStatus{ + Host: host, + // FailoverReservation is nil + }, + } +} + +func buildSchedulingTestVM(uuid, hypervisor string) VM { //nolint:unparam // uuid may vary in future tests + return VM{ + UUID: uuid, + CurrentHypervisor: hypervisor, + FlavorName: "m1.large", + ProjectID: "test-project", + Resources: map[string]resource.Quantity{ + "vcpus": resource.MustParse("4"), + "memory": resource.MustParse("8Gi"), + }, + } +} + +func buildSchedulingTestVMWithResources(uuid, hypervisor string, memoryMB, vcpus int64) VM { + return VM{ + UUID: uuid, + CurrentHypervisor: hypervisor, + FlavorName: "m1.large", + ProjectID: "test-project", + Resources: map[string]resource.Quantity{ + "vcpus": *resource.NewQuantity(vcpus, resource.DecimalSI), + "memory": *resource.NewQuantity(memoryMB*1024*1024, resource.BinarySI), + }, + } +} diff --git a/internal/scheduling/reservations/failover/vm_source.go b/internal/scheduling/reservations/failover/vm_source.go new file mode 100644 index 000000000..4d5c3f210 --- /dev/null +++ b/internal/scheduling/reservations/failover/vm_source.go @@ -0,0 +1,452 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failover + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/cobaltcore-dev/cortex/internal/scheduling/external" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// VM represents a virtual machine that may need failover reservations. +type VM struct { + // UUID is the unique identifier of the VM. + UUID string + // FlavorName is the name of the flavor used by the VM. + FlavorName string + // ProjectID is the OpenStack project ID that owns the VM. + ProjectID string + // CurrentHypervisor is the hypervisor where the VM is currently running. + CurrentHypervisor string + // AvailabilityZone is the availability zone where the VM is located. + // This is used to ensure failover reservations are created in the same AZ. + AvailabilityZone string + // Resources contains the VM's resource allocations (e.g., "memory", "vcpus"). + Resources map[string]resource.Quantity + // FlavorExtraSpecs contains the flavor's extra specifications (e.g., traits, capabilities). + // This is used by filters like filter_has_requested_traits and filter_capabilities. + FlavorExtraSpecs map[string]string +} + +// VMSource provides VMs that may need failover reservations. +// This interface allows swapping the implementation when a VM CRD arrives. +type VMSource interface { + // ListVMs returns all VMs that might need failover reservations. + ListVMs(ctx context.Context) ([]VM, error) + // ListVMsOnHypervisors returns VMs that are on the given hypervisors. + // If trustHypervisorLocation is true, uses hypervisor CRD as source of truth for VM location. + // If trustHypervisorLocation is false, uses postgres as source of truth but filters to VMs on known hypervisors. + // Also logs warnings about data sync issues between postgres and hypervisor CRD. + ListVMsOnHypervisors(ctx context.Context, hypervisorList *hv1.HypervisorList, trustHypervisorLocation bool) ([]VM, error) + // GetVM returns a specific VM by UUID. + // Returns nil, nil if the VM is not found (not an error, just doesn't exist). + GetVM(ctx context.Context, vmUUID string) (*VM, error) +} + +// DBVMSource implements VMSource by reading directly from the database. +// This is the preferred implementation as it avoids the size limitations of Knowledge CRDs. +type DBVMSource struct { + NovaReader external.NovaReaderInterface +} + +// NewDBVMSource creates a new DBVMSource. +func NewDBVMSource(novaReader external.NovaReaderInterface) *DBVMSource { + return &DBVMSource{NovaReader: novaReader} +} + +// ListVMs returns all VMs by joining server and flavor data from the database. +func (s *DBVMSource) ListVMs(ctx context.Context) ([]VM, error) { + // Fetch all servers + servers, err := s.NovaReader.GetAllServers(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get servers: %w", err) + } + + // Fetch all flavors and build a lookup map + flavors, err := s.NovaReader.GetAllFlavors(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get flavors: %w", err) + } + flavorByName := make(map[string]struct { + VCPUs uint64 + RAM uint64 + ExtraSpecs string + }) + for _, f := range flavors { + flavorByName[f.Name] = struct { + VCPUs uint64 + RAM uint64 + ExtraSpecs string + }{VCPUs: f.VCPUs, RAM: f.RAM, ExtraSpecs: f.ExtraSpecs} + } + + // Track filtering statistics + var skippedNoHost, skippedUnknownFlavor int + unknownFlavors := make(map[string]int) + + // Convert servers to VMs + vms := make([]VM, 0, len(servers)) + for _, server := range servers { + // Skip servers without a host (not yet scheduled) + if server.OSEXTSRVATTRHost == "" { + skippedNoHost++ + continue + } + + // Look up flavor resources + flavor, ok := flavorByName[server.FlavorName] + if !ok { + // Skip servers with unknown flavors + skippedUnknownFlavor++ + unknownFlavors[server.FlavorName]++ + continue + } + + // Build resources map + resources := map[string]resource.Quantity{ + "vcpus": *resource.NewQuantity(int64(flavor.VCPUs), resource.DecimalSI), //nolint:gosec // VCPUs won't overflow int64 + "memory": *resource.NewQuantity(int64(flavor.RAM)*1024*1024, resource.BinarySI), //nolint:gosec // RAM in MB won't overflow int64 + } + + // Parse extra specs from JSON string + extraSpecs := parseExtraSpecs(flavor.ExtraSpecs) + + vms = append(vms, VM{ + UUID: server.ID, + FlavorName: server.FlavorName, + ProjectID: server.TenantID, + CurrentHypervisor: server.OSEXTSRVATTRHost, + AvailabilityZone: server.OSEXTAvailabilityZone, + Resources: resources, + FlavorExtraSpecs: extraSpecs, + }) + } + + // Log filtering statistics at debug level (verbose) + log.V(1).Info("ListVMs filtering statistics", + "totalServersInDB", len(servers), + "skippedNoHost", skippedNoHost, + "skippedUnknownFlavor", skippedUnknownFlavor, + "totalFlavorsInDB", len(flavors), + "returnedVMs", len(vms)) + if len(unknownFlavors) > 0 { + log.V(1).Info("ListVMs unknown flavors", "unknownFlavors", unknownFlavors) + } + + return vms, nil +} + +// parseExtraSpecs parses a JSON string of extra specs into a map. +// Returns an empty map if the string is empty or invalid. +func parseExtraSpecs(extraSpecsJSON string) map[string]string { + if extraSpecsJSON == "" { + return make(map[string]string) + } + var extraSpecs map[string]string + if err := json.Unmarshal([]byte(extraSpecsJSON), &extraSpecs); err != nil { + // Log error but don't fail - return empty map + log.Error(err, "failed to parse flavor extra specs JSON", + "extraSpecsJSON", truncateString(extraSpecsJSON, 100)) + return make(map[string]string) + } + return extraSpecs +} + +// truncateString truncates a string to maxLen characters, adding "..." if truncated. +func truncateString(s string, maxLen int) string { + if len(s) <= maxLen { + return s + } + return s[:maxLen] + "..." +} + +// GetVM returns a specific VM by UUID. +// Returns nil, nil if the VM is not found (not an error, just doesn't exist). +func (s *DBVMSource) GetVM(ctx context.Context, vmUUID string) (*VM, error) { + // Fetch the server by UUID + server, err := s.NovaReader.GetServerByID(ctx, vmUUID) + if err != nil { + return nil, fmt.Errorf("failed to get server: %w", err) + } + if server == nil { + // Server not found + return nil, nil + } + + // Skip servers without a host (not yet scheduled) + if server.OSEXTSRVATTRHost == "" { + return nil, nil + } + + // Fetch the flavor for this server + flavor, err := s.NovaReader.GetFlavorByName(ctx, server.FlavorName) + if err != nil { + return nil, fmt.Errorf("failed to get flavor: %w", err) + } + if flavor == nil { + // Flavor not found + return nil, nil + } + + // Build resources map + resources := map[string]resource.Quantity{ + "vcpus": *resource.NewQuantity(int64(flavor.VCPUs), resource.DecimalSI), //nolint:gosec // VCPUs won't overflow int64 + "memory": *resource.NewQuantity(int64(flavor.RAM)*1024*1024, resource.BinarySI), //nolint:gosec // RAM in MB won't overflow int64 + } + + // Parse extra specs from JSON string + extraSpecs := parseExtraSpecs(flavor.ExtraSpecs) + + return &VM{ + UUID: server.ID, + FlavorName: server.FlavorName, + ProjectID: server.TenantID, + CurrentHypervisor: server.OSEXTSRVATTRHost, + AvailabilityZone: server.OSEXTAvailabilityZone, + Resources: resources, + FlavorExtraSpecs: extraSpecs, + }, nil +} + +// ListVMsOnHypervisors returns VMs that are on the given hypervisors. +// If trustHypervisorLocation is true, uses hypervisor CRD as source of truth for VM location. +// If trustHypervisorLocation is false, uses postgres as source of truth but filters to VMs on known hypervisors. +// Also logs warnings about data sync issues between postgres and hypervisor CRD. +func (s *DBVMSource) ListVMsOnHypervisors( + ctx context.Context, + hypervisorList *hv1.HypervisorList, + trustHypervisorLocation bool, +) ([]VM, error) { + // Get VMs from postgres + vms, err := s.ListVMs(ctx) + if err != nil { + return nil, err + } + + // Warn about data sync issues + warnUnknownVMsOnHypervisors(hypervisorList, vms) + + if trustHypervisorLocation { + // Use hypervisor CRD as source of truth for VM location + result := buildVMsFromHypervisors(hypervisorList, vms) + log.V(1).Info("built VMs from hypervisor instances (TrustHypervisorLocation=true)", + "count", len(result), + "knownHypervisors", len(hypervisorList.Items)) + return result, nil + } + + // Use postgres as source of truth, but filter to VMs on known hypervisors + result := filterVMsOnKnownHypervisors(vms, hypervisorList) + log.V(1).Info("filtered VMs to those on known hypervisors and in hypervisor instances", + "count", len(result), + "knownHypervisors", len(hypervisorList.Items)) + return result, nil +} + +// ============================================================================ +// VM/Hypervisor Processing (internal helpers) +// ============================================================================ + +// buildVMsFromHypervisors builds VMs from hypervisor instances, using the hypervisor CRD +// as the source of truth for VM location. It enriches VMs with data from postgres (flavor, size, extra specs, AZ). +// This is used when TrustHypervisorLocation is true. +// +// The function: +// 1. Iterates through all hypervisor instances to get VM UUIDs and their actual location +// 2. Looks up each VM in the postgres-sourced vms list to get flavor/size/extra specs/AZ +// 3. Returns VMs that exist in both hypervisor instances AND postgres (need postgres for scheduling data) +// 4. Deduplicates VMs that appear on multiple hypervisors (transient state during live migration) +func buildVMsFromHypervisors(hypervisorList *hv1.HypervisorList, postgresVMs []VM) []VM { + // Build a map of VM UUID -> VM data from postgres for quick lookup + vmDataByUUID := make(map[string]VM, len(postgresVMs)) + for _, vm := range postgresVMs { + vmDataByUUID[vm.UUID] = vm + } + + var result []VM + var enrichedCount, notInPostgresCount, duplicateCount int + + // Track seen UUIDs to deduplicate VMs that appear on multiple hypervisors + // This can happen transiently during live migration + seen := make(map[string]string) // vmUUID -> first hypervisor seen + + // Iterate through hypervisor instances + for _, hv := range hypervisorList.Items { + for _, inst := range hv.Status.Instances { + if !inst.Active { + continue + } + + // Check for duplicate UUIDs (same VM on multiple hypervisors) + if firstHypervisor, alreadySeen := seen[inst.ID]; alreadySeen { + log.Info("duplicate VM UUID on multiple hypervisors, skipping", + "uuid", inst.ID, + "hypervisor", hv.Name, + "firstSeenOn", firstHypervisor) + duplicateCount++ + continue + } + seen[inst.ID] = hv.Name + + // Look up VM data from postgres + pgVM, existsInPostgres := vmDataByUUID[inst.ID] + if !existsInPostgres { + // VM is on hypervisor but not in postgres - skip (need postgres for flavor/size) + notInPostgresCount++ + continue + } + + // Build VM with hypervisor location but postgres data (including AZ) + vm := VM{ + UUID: inst.ID, + FlavorName: pgVM.FlavorName, + ProjectID: pgVM.ProjectID, + CurrentHypervisor: hv.Name, // Use hypervisor CRD location, not postgres + AvailabilityZone: pgVM.AvailabilityZone, + Resources: pgVM.Resources, + FlavorExtraSpecs: pgVM.FlavorExtraSpecs, + } + result = append(result, vm) + enrichedCount++ + } + } + + log.V(1).Info("buildVMsFromHypervisors statistics", + "totalHypervisorInstances", enrichedCount+notInPostgresCount+duplicateCount, + "enrichedWithPostgresData", enrichedCount, + "notInPostgres", notInPostgresCount, + "duplicatesSkipped", duplicateCount) + + return result +} + +// filterVMsOnKnownHypervisors filters VMs to only include those that: +// 1. Are running on a known hypervisor +// 2. Are actually listed in that hypervisor's Status.Instances +// This removes VMs that are on hypervisors not managed by the hypervisor operator, +// or VMs that claim to be on a hypervisor but aren't in its instances list (data sync issue). +func filterVMsOnKnownHypervisors(vms []VM, hypervisorList *hv1.HypervisorList) []VM { + // Build a set of known hypervisors for O(1) lookup + hypervisorSet := make(map[string]bool, len(hypervisorList.Items)) + for _, hv := range hypervisorList.Items { + hypervisorSet[hv.Name] = true + } + + // Build a set of VM UUIDs that are actually in hypervisor instances + // Key: "vmUUID:hypervisorName" to ensure VM is on the correct hypervisor + vmOnHypervisor := make(map[string]bool) + // Also track all VMs on hypervisors (regardless of which hypervisor) + allVMsOnHypervisors := make(map[string]string) // vmUUID -> hypervisorName + totalVMsOnHypervisors := 0 + for _, hv := range hypervisorList.Items { + for _, inst := range hv.Status.Instances { + if inst.Active { + key := inst.ID + ":" + hv.Name + vmOnHypervisor[key] = true + allVMsOnHypervisors[inst.ID] = hv.Name + totalVMsOnHypervisors++ + } + } + } + + var result []VM + var filteredUnknownHypervisor int + var filteredNotInInstances int + var filteredWrongHypervisor int + for _, vm := range vms { + // Check if hypervisor is known + if !hypervisorSet[vm.CurrentHypervisor] { + filteredUnknownHypervisor++ + continue + } + // Check if VM is actually in the hypervisor's instances list + key := vm.UUID + ":" + vm.CurrentHypervisor + if !vmOnHypervisor[key] { + // Check if VM is on a different hypervisor + if actualHypervisor, exists := allVMsOnHypervisors[vm.UUID]; exists { + log.V(2).Info("VM claims to be on one hypervisor but is actually on another", + "vmUUID", vm.UUID, + "claimedHypervisor", vm.CurrentHypervisor, + "actualHypervisor", actualHypervisor) + filteredWrongHypervisor++ + } else { + filteredNotInInstances++ + } + continue + } + result = append(result, vm) + } + + totalFiltered := filteredUnknownHypervisor + filteredNotInInstances + filteredWrongHypervisor + if totalFiltered > 0 { + log.Info("filterVMsOnKnownHypervisors statistics", + "inputVMs", len(vms), + "totalVMsOnHypervisors", totalVMsOnHypervisors, + "filteredUnknownHypervisor", filteredUnknownHypervisor, + "filteredNotInInstances", filteredNotInInstances, + "filteredWrongHypervisor", filteredWrongHypervisor, + "totalFiltered", totalFiltered, + "remainingCount", len(result)) + } + + return result +} + +// warnUnknownVMsOnHypervisors logs a warning for VMs that are on hypervisors but not in the ListVMs (i.e. nova) result. +// This can indicate a data sync issue between the hypervisor operator and the VM datasource. +func warnUnknownVMsOnHypervisors(hypervisors *hv1.HypervisorList, vms []VM) { + // Build a set of VM UUIDs from ListVMs + vmUUIDs := make(map[string]bool, len(vms)) + for _, vm := range vms { + vmUUIDs[vm.UUID] = true + } + + // Build a set of VM UUIDs from hypervisors + hypervisorVMUUIDs := make(map[string]bool) + for _, hv := range hypervisors.Items { + for _, inst := range hv.Status.Instances { + if inst.Active { + hypervisorVMUUIDs[inst.ID] = true + } + } + } + + // Check each hypervisor's instances - VMs on hypervisors but not in ListVMs + vmsOnHypervisorsNotInListVMs := 0 + for _, hv := range hypervisors.Items { + for _, inst := range hv.Status.Instances { + if inst.Active && !vmUUIDs[inst.ID] { + log.Info("WARNING: VM on hypervisor not found in ListVMs - possible data sync issue", + "vmUUID", inst.ID, + "vmName", inst.Name, + "hypervisor", hv.Name) + vmsOnHypervisorsNotInListVMs++ + } + } + } + + // Check VMs in ListVMs but not on any hypervisor + vmsInListVMsNotOnHypervisors := 0 + for _, vm := range vms { + if !hypervisorVMUUIDs[vm.UUID] { + vmsInListVMsNotOnHypervisors++ + } + } + + if vmsOnHypervisorsNotInListVMs > 0 { + log.V(1).Info("VMs on hypervisors not found in ListVMs", + "count", vmsOnHypervisorsNotInListVMs, + "hint", "This may indicate a data sync issue between hypervisor operator and nova servers") + } + + if vmsInListVMsNotOnHypervisors > 0 { + log.V(1).Info("VMs in ListVMs not found on any hypervisor", + "count", vmsInListVMsNotOnHypervisors, + "hint", "This may indicate a data sync issue between nova servers and hypervisor operator") + } +} diff --git a/internal/scheduling/reservations/failover/vm_source_test.go b/internal/scheduling/reservations/failover/vm_source_test.go new file mode 100644 index 000000000..0b30af0e5 --- /dev/null +++ b/internal/scheduling/reservations/failover/vm_source_test.go @@ -0,0 +1,401 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failover + +import ( + "context" + "errors" + "strings" + "testing" + + nova "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/nova" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestDBVMSource_ListVMs(t *testing.T) { + dbError := errors.New("connection refused: database unavailable") + + tests := []struct { + name string + mock *mockNovaReader + wantErr bool + wantErrContains string + wantWrappedErr error + wantVMCount int + wantFirstVMUUID string + }{ + { + name: "GetAllServers error", + mock: &mockNovaReader{ + getAllServersFunc: func(ctx context.Context) ([]nova.Server, error) { + return nil, dbError + }, + }, + wantErr: true, + wantErrContains: "failed to get servers", + wantWrappedErr: dbError, + wantVMCount: 0, + }, + { + name: "GetAllFlavors error", + mock: &mockNovaReader{ + getAllServersFunc: func(ctx context.Context) ([]nova.Server, error) { + return []nova.Server{{ID: "vm-1", FlavorName: "m1.large", OSEXTSRVATTRHost: "host1"}}, nil + }, + getAllFlavorsFunc: func(ctx context.Context) ([]nova.Flavor, error) { + return nil, dbError + }, + }, + wantErr: true, + wantErrContains: "failed to get flavors", + wantWrappedErr: dbError, + wantVMCount: 0, + }, + { + name: "success with multiple VMs", + mock: &mockNovaReader{ + getAllServersFunc: func(ctx context.Context) ([]nova.Server, error) { + return []nova.Server{ + {ID: "vm-1", FlavorName: "m1.large", OSEXTSRVATTRHost: "host1", TenantID: "project-1"}, + {ID: "vm-2", FlavorName: "m1.small", OSEXTSRVATTRHost: "host2", TenantID: "project-2"}, + }, nil + }, + getAllFlavorsFunc: func(ctx context.Context) ([]nova.Flavor, error) { + return []nova.Flavor{ + {Name: "m1.large", VCPUs: 4, RAM: 8192}, + {Name: "m1.small", VCPUs: 2, RAM: 4096}, + }, nil + }, + }, + wantErr: false, + wantVMCount: 2, + wantFirstVMUUID: "vm-1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + source := NewDBVMSource(tt.mock) + vms, err := source.ListVMs(context.Background()) + + if tt.wantErr { + if err == nil { + t.Fatal("expected error, got nil") + } + if tt.wantErrContains != "" && !strings.Contains(err.Error(), tt.wantErrContains) { + t.Errorf("expected error to contain %q, got: %v", tt.wantErrContains, err) + } + if tt.wantWrappedErr != nil && !errors.Is(err, tt.wantWrappedErr) { + t.Errorf("expected error to wrap %v, got: %v", tt.wantWrappedErr, err) + } + if vms != nil { + t.Errorf("expected nil VMs when error occurs, got %v", vms) + } + } else { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(vms) != tt.wantVMCount { + t.Errorf("expected %d VMs, got %d", tt.wantVMCount, len(vms)) + } + if tt.wantFirstVMUUID != "" && len(vms) > 0 && vms[0].UUID != tt.wantFirstVMUUID { + t.Errorf("expected first VM UUID %q, got %q", tt.wantFirstVMUUID, vms[0].UUID) + } + } + }) + } +} + +func TestDBVMSource_GetVM(t *testing.T) { + dbError := errors.New("connection refused: database unavailable") + + tests := []struct { + name string + vmID string + mock *mockNovaReader + wantErr bool + wantErrContains string + wantWrappedErr error + wantNilVM bool + wantVMUUID string + wantFlavorName string + }{ + { + name: "GetServerByID error", + vmID: "vm-1", + mock: &mockNovaReader{ + getServerByIDFunc: func(ctx context.Context, serverID string) (*nova.Server, error) { + return nil, dbError + }, + }, + wantErr: true, + wantErrContains: "failed to get server", + wantWrappedErr: dbError, + wantNilVM: true, + }, + { + name: "GetFlavorByName error", + vmID: "vm-1", + mock: &mockNovaReader{ + getServerByIDFunc: func(ctx context.Context, serverID string) (*nova.Server, error) { + return &nova.Server{ID: "vm-1", FlavorName: "m1.large", OSEXTSRVATTRHost: "host1"}, nil + }, + getFlavorByNameFunc: func(ctx context.Context, flavorName string) (*nova.Flavor, error) { + return nil, dbError + }, + }, + wantErr: true, + wantErrContains: "failed to get flavor", + wantWrappedErr: dbError, + wantNilVM: true, + }, + { + name: "VM not found", + vmID: "non-existent-vm", + mock: &mockNovaReader{ + getServerByIDFunc: func(ctx context.Context, serverID string) (*nova.Server, error) { + return nil, nil // Server not found + }, + }, + wantErr: false, + wantNilVM: true, + }, + { + name: "success", + vmID: "vm-1", + mock: &mockNovaReader{ + getServerByIDFunc: func(ctx context.Context, serverID string) (*nova.Server, error) { + return &nova.Server{ + ID: "vm-1", + FlavorName: "m1.large", + OSEXTSRVATTRHost: "host1", + TenantID: "project-1", + OSEXTAvailabilityZone: "az-1", + }, nil + }, + getFlavorByNameFunc: func(ctx context.Context, flavorName string) (*nova.Flavor, error) { + return &nova.Flavor{Name: "m1.large", VCPUs: 4, RAM: 8192}, nil + }, + }, + wantErr: false, + wantNilVM: false, + wantVMUUID: "vm-1", + wantFlavorName: "m1.large", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + source := NewDBVMSource(tt.mock) + vm, err := source.GetVM(context.Background(), tt.vmID) + + if tt.wantErr { + if err == nil { + t.Fatal("expected error, got nil") + } + if tt.wantErrContains != "" && !strings.Contains(err.Error(), tt.wantErrContains) { + t.Errorf("expected error to contain %q, got: %v", tt.wantErrContains, err) + } + if tt.wantWrappedErr != nil && !errors.Is(err, tt.wantWrappedErr) { + t.Errorf("expected error to wrap %v, got: %v", tt.wantWrappedErr, err) + } + } else if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + switch { + case tt.wantNilVM: + if vm != nil { + t.Errorf("expected nil VM, got %v", vm) + } + case vm == nil: + t.Fatal("expected VM, got nil") + default: + if tt.wantVMUUID != "" && vm.UUID != tt.wantVMUUID { + t.Errorf("expected UUID %q, got %q", tt.wantVMUUID, vm.UUID) + } + if tt.wantFlavorName != "" && vm.FlavorName != tt.wantFlavorName { + t.Errorf("expected FlavorName %q, got %q", tt.wantFlavorName, vm.FlavorName) + } + } + }) + } +} + +func TestDBVMSource_ListVMsOnHypervisors(t *testing.T) { + dbError := errors.New("connection refused: database unavailable") + + tests := []struct { + name string + mock *mockNovaReader + hypervisorList *hv1.HypervisorList + wantErr bool + wantWrappedErr error + }{ + { + name: "propagates error from ListVMs", + mock: &mockNovaReader{ + getAllServersFunc: func(ctx context.Context) ([]nova.Server, error) { + return nil, dbError + }, + }, + hypervisorList: &hv1.HypervisorList{ + Items: []hv1.Hypervisor{ + {ObjectMeta: metav1.ObjectMeta{Name: "host1"}}, + }, + }, + wantErr: true, + wantWrappedErr: dbError, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + source := NewDBVMSource(tt.mock) + vms, err := source.ListVMsOnHypervisors(context.Background(), tt.hypervisorList, false) + + if tt.wantErr { + if err == nil { + t.Fatal("expected error, got nil") + } + if tt.wantWrappedErr != nil && !errors.Is(err, tt.wantWrappedErr) { + t.Errorf("expected error to wrap %v, got: %v", tt.wantWrappedErr, err) + } + if vms != nil { + t.Errorf("expected nil VMs when error occurs, got %v", vms) + } + } + }) + } +} + +func TestBuildVMsFromHypervisors(t *testing.T) { + tests := []struct { + name string + hypervisorList *hv1.HypervisorList + postgresVMs []VM + wantCount int + }{ + { + name: "empty postgres VMs", + hypervisorList: &hv1.HypervisorList{ + Items: []hv1.Hypervisor{ + { + ObjectMeta: metav1.ObjectMeta{Name: "host1"}, + Status: hv1.HypervisorStatus{ + Instances: []hv1.Instance{ + {ID: "vm-1", Name: "vm-1", Active: true}, + }, + }, + }, + }, + }, + postgresVMs: []VM{}, + wantCount: 0, + }, + { + name: "nil hypervisor items", + hypervisorList: &hv1.HypervisorList{Items: nil}, + postgresVMs: []VM{}, + wantCount: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Errorf("buildVMsFromHypervisors panicked: %v", r) + } + }() + + result := buildVMsFromHypervisors(tt.hypervisorList, tt.postgresVMs) + + if len(result) != tt.wantCount { + t.Errorf("expected %d VMs, got %d", tt.wantCount, len(result)) + } + }) + } +} + +func TestFilterVMsOnKnownHypervisors_NilInputs(t *testing.T) { + tests := []struct { + name string + vms []VM + hypervisorList *hv1.HypervisorList + wantCount int + }{ + { + name: "nil hypervisor items does not panic", + vms: []VM{{UUID: "vm-1", CurrentHypervisor: "host1"}}, + hypervisorList: &hv1.HypervisorList{ + Items: nil, + }, + wantCount: 0, + }, + { + name: "nil VMs does not panic", + vms: nil, + hypervisorList: &hv1.HypervisorList{ + Items: []hv1.Hypervisor{ + {ObjectMeta: metav1.ObjectMeta{Name: "host1"}}, + }, + }, + wantCount: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Errorf("filterVMsOnKnownHypervisors panicked: %v", r) + } + }() + + result := filterVMsOnKnownHypervisors(tt.vms, tt.hypervisorList) + + if len(result) != tt.wantCount { + t.Errorf("expected %d VMs, got %d", tt.wantCount, len(result)) + } + }) + } +} + +// mockNovaReader implements NovaReader for testing. +type mockNovaReader struct { + getAllServersFunc func(ctx context.Context) ([]nova.Server, error) + getAllFlavorsFunc func(ctx context.Context) ([]nova.Flavor, error) + getServerByIDFunc func(ctx context.Context, serverID string) (*nova.Server, error) + getFlavorByNameFunc func(ctx context.Context, flavorName string) (*nova.Flavor, error) +} + +func (m *mockNovaReader) GetAllServers(ctx context.Context) ([]nova.Server, error) { + if m.getAllServersFunc != nil { + return m.getAllServersFunc(ctx) + } + return nil, nil +} + +func (m *mockNovaReader) GetAllFlavors(ctx context.Context) ([]nova.Flavor, error) { + if m.getAllFlavorsFunc != nil { + return m.getAllFlavorsFunc(ctx) + } + return nil, nil +} + +func (m *mockNovaReader) GetServerByID(ctx context.Context, serverID string) (*nova.Server, error) { + if m.getServerByIDFunc != nil { + return m.getServerByIDFunc(ctx, serverID) + } + return nil, nil +} + +func (m *mockNovaReader) GetFlavorByName(ctx context.Context, flavorName string) (*nova.Flavor, error) { + if m.getFlavorByNameFunc != nil { + return m.getFlavorByNameFunc(ctx, flavorName) + } + return nil, nil +} diff --git a/internal/scheduling/reservations/scheduler_client.go b/internal/scheduling/reservations/scheduler_client.go new file mode 100644 index 000000000..618cec04a --- /dev/null +++ b/internal/scheduling/reservations/scheduler_client.go @@ -0,0 +1,193 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package reservations + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/go-logr/logr" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +// DefaultHTTPTimeout is the default timeout for HTTP requests to the scheduler API. +const DefaultHTTPTimeout = 30 * time.Second + +var log = logf.Log.WithName("scheduler-client").WithValues("module", "reservations") + +// loggerFromContext returns a logger with greq and req values from the context. +func loggerFromContext(ctx context.Context) logr.Logger { + return log.WithValues( + "greq", GlobalRequestIDFromContext(ctx), + "req", RequestIDFromContext(ctx), + ) +} + +// NOTE+FIXME: we should not send ourselves REST API calls. This needs to be replaced by direct Go call (if possible) or communication via CRDs + +// SchedulerClient is a client for the external scheduler API. +// It can be used by both the ReservationReconciler and FailoverReservationController. +type SchedulerClient struct { + // URL of the external scheduler API. + URL string + // HTTP client to use for requests. + HTTPClient *http.Client +} + +// NewSchedulerClient creates a new SchedulerClient with a default timeout. +func NewSchedulerClient(url string) *SchedulerClient { + return &SchedulerClient{ + URL: url, + HTTPClient: &http.Client{ + Timeout: DefaultHTTPTimeout, + }, + } +} + +// ScheduleReservationRequest contains the parameters for scheduling a reservation. +type ScheduleReservationRequest struct { + // InstanceUUID is the unique identifier for the reservation (usually the reservation name). + InstanceUUID string + // ProjectID is the OpenStack project ID. + ProjectID string + // FlavorName is the name of the flavor. + FlavorName string + // FlavorExtraSpecs are extra specifications for the flavor. + FlavorExtraSpecs map[string]string + // MemoryMB is the memory in MB. + MemoryMB uint64 + // VCPUs is the number of virtual CPUs. + VCPUs uint64 + // EligibleHosts is the list of hosts that can be considered for placement. + EligibleHosts []api.ExternalSchedulerHost + // IgnoreHosts is a list of hosts to ignore during scheduling. + // This is used for failover reservations to avoid placing on the same host as the VMs. + IgnoreHosts []string + // Pipeline is the name of the pipeline to execute. + // If empty, the default pipeline will be used. + Pipeline string + // AvailabilityZone is the availability zone to schedule in. + // This is used by the filter_correct_az filter to ensure hosts are in the correct AZ. + AvailabilityZone string +} + +// ScheduleReservationResponse contains the result of scheduling a reservation. +type ScheduleReservationResponse struct { + // Hosts is the ordered list of hosts that the reservation can be placed on. + // The first host is the best choice. + Hosts []string +} + +// ScheduleReservation calls the external scheduler API to find a host for a reservation. +// The context should contain GlobalRequestID and RequestID for logging (use WithGlobalRequestID/WithRequestID). +func (c *SchedulerClient) ScheduleReservation(ctx context.Context, req ScheduleReservationRequest) (*ScheduleReservationResponse, error) { + logger := loggerFromContext(ctx) + + // Build weights map (all zero for reservations) + weights := make(map[string]float64, len(req.EligibleHosts)) + for _, host := range req.EligibleHosts { + weights[host.ComputeHost] = 0.0 + } + + // Build ignore hosts pointer + var ignoreHosts *[]string + if len(req.IgnoreHosts) > 0 { + ignoreHosts = &req.IgnoreHosts + } + + // Build the context with request IDs + var globalReqID *string + if greq := GlobalRequestIDFromContext(ctx); greq != "" { + globalReqID = &greq + } + + // Build the external scheduler request + externalSchedulerRequest := api.ExternalSchedulerRequest{ + Reservation: true, + Pipeline: req.Pipeline, + Hosts: req.EligibleHosts, + Weights: weights, + Context: api.NovaRequestContext{ + RequestID: RequestIDFromContext(ctx), + GlobalRequestID: globalReqID, + }, + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + InstanceUUID: req.InstanceUUID, + NumInstances: 1, // One for each reservation. + ProjectID: req.ProjectID, + AvailabilityZone: req.AvailabilityZone, + IgnoreHosts: ignoreHosts, + SchedulerHints: make(map[string]any), // Initialize to empty map for consistent behavior + Flavor: api.NovaObject[api.NovaFlavor]{ + Data: api.NovaFlavor{ + Name: req.FlavorName, + ExtraSpecs: req.FlavorExtraSpecs, + MemoryMB: req.MemoryMB, + VCPUs: req.VCPUs, + // Disk is currently not considered. + }, + }, + }, + }, + } + + logger.V(1).Info("sending external scheduler request", + "url", c.URL, + "instanceUUID", req.InstanceUUID, + "projectID", req.ProjectID, + "flavorName", req.FlavorName, + "flavorExtraSpecs", req.FlavorExtraSpecs, + "memoryMB", req.MemoryMB, + "vcpus", req.VCPUs, + "eligibleHostsCount", len(req.EligibleHosts), + "ignoreHosts", req.IgnoreHosts) + + // Marshal the request + reqBody, err := json.Marshal(externalSchedulerRequest) + if err != nil { + logger.Error(err, "failed to marshal external scheduler request") + return nil, fmt.Errorf("failed to marshal external scheduler request: %w", err) + } + + // Create HTTP request with context + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, c.URL, bytes.NewReader(reqBody)) + if err != nil { + logger.Error(err, "failed to create HTTP request") + return nil, fmt.Errorf("failed to create HTTP request: %w", err) + } + httpReq.Header.Set("Content-Type", "application/json") + + // Send the request + response, err := c.HTTPClient.Do(httpReq) + if err != nil { + logger.Error(err, "failed to send external scheduler request") + return nil, fmt.Errorf("failed to send external scheduler request: %w", err) + } + defer response.Body.Close() + + // Check response status + if response.StatusCode != http.StatusOK { + logger.Error(nil, "external scheduler returned non-OK status", "statusCode", response.StatusCode) + return nil, fmt.Errorf("external scheduler returned status %d", response.StatusCode) + } + + // Decode the response + var externalSchedulerResponse api.ExternalSchedulerResponse + if err := json.NewDecoder(response.Body).Decode(&externalSchedulerResponse); err != nil { + logger.Error(err, "failed to decode external scheduler response") + return nil, fmt.Errorf("failed to decode external scheduler response: %w", err) + } + + logger.V(1).Info("received external scheduler response", "hostsCount", len(externalSchedulerResponse.Hosts)) + + return &ScheduleReservationResponse{ + Hosts: externalSchedulerResponse.Hosts, + }, nil +} diff --git a/tools/visualize-reservations/main.go b/tools/visualize-reservations/main.go new file mode 100644 index 000000000..28d4dee23 --- /dev/null +++ b/tools/visualize-reservations/main.go @@ -0,0 +1,1738 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +// Tool to visualize failover reservations and their VM connections +// +// Usage: +// +// go run tools/visualize-reservations/main.go [flags] +// +// Flags: +// +// --sort=vm|vm-host|res-host Sort VMs by UUID, VM host, or reservation host +// --postgres-secret=name Name of the kubernetes secret containing postgres credentials (default: cortex-nova-postgres) +// --namespace=ns Namespace of the postgres secret (default: default) +// --postgres-host=host Override postgres host (useful with port-forward, e.g., localhost) +// --postgres-port=port Override postgres port (useful with port-forward, e.g., 5432) +// --views=view1,view2,... Comma-separated list of views to show (default: all) +// Available views: hypervisors, vms, reservations, summary, +// hypervisor-summary, validation, stale, without-res, not-in-db, by-host +// --hide=view1,view2,... Comma-separated list of views to hide (applied after --views) +// --filter-name=pattern Filter hypervisors by name (substring match) +// --filter-trait=trait Filter hypervisors by trait (e.g., CUSTOM_HANA_EXCLUSIVE_HOST) +// +// To connect to postgres when running locally, use kubectl port-forward: +// +// kubectl port-forward svc/cortex-nova-postgresql 5432:5432 -n +// go run tools/visualize-reservations/main.go --postgres-host=localhost --postgres-port=5432 +package main + +import ( + "context" + "database/sql" + "encoding/json" + "flag" + "fmt" + "os" + "sort" + "strings" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + _ "github.com/lib/pq" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" +) + +var scheme = runtime.NewScheme() + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(v1alpha1.AddToScheme(scheme)) + utilruntime.Must(hv1.AddToScheme(scheme)) +} + +// vmEntry holds VM data for sorting +type vmEntry struct { + UUID string + Host string + Reservations []string // reservation_name@host + // From postgres + InServerTable bool + FlavorName string + VCPUs int + RAMMb int + DiskGb int + // Tracking fields + NotOnHypervisors bool // VM is in reservation but not found on any hypervisor + ReservationSource string // Name of reservation where this VM was found (if not on hypervisor) +} + +// serverInfo holds server data from postgres +type serverInfo struct { + ID string + FlavorName string + HostID string // OS-EXT-SRV-ATTR:host_id (hypervisor ID) + OSEXTSRVATTRHost string // OS-EXT-SRV-ATTR:host (hypervisor hostname) +} + +// flavorInfo holds flavor data from postgres +type flavorInfo struct { + Name string + VCPUs int + RAMMb int + DiskGb int + ExtraSpecs string // JSON string of extra specs +} + +// hypervisorSummary holds aggregated data for a hypervisor +type hypervisorSummary struct { + Name string + NumVMs int + NumReservations int + CapacityCPU int64 + CapacityMemoryGB int64 + UsedByVMsCPU int64 + UsedByVMsMemoryGB int64 + FailoverResCPU int64 + FailoverResMemGB int64 + CommittedResCPU int64 + CommittedResMemGB int64 + FreeCPU int64 + FreeMemoryGB int64 + Traits []string +} + +// viewSet tracks which views should be displayed +type viewSet map[string]bool + +// Available view names +const ( + viewHypervisors = "hypervisors" + viewHypervisorsVMsRes = "hypervisors-vms-res" + viewVMs = "vms" + viewReservations = "reservations" + viewSummary = "summary" + viewHypervisorSummary = "hypervisor-summary" + viewFlavorSummary = "flavor-summary" + viewValidation = "validation" + viewStale = "stale" + viewWithoutRes = "without-res" + viewNotInDB = "not-in-db" + viewByHost = "by-host" + viewAllServers = "all-servers" +) + +var allViews = []string{ + viewHypervisors, + viewHypervisorsVMsRes, + viewVMs, + viewReservations, + viewSummary, + viewHypervisorSummary, + viewFlavorSummary, + viewValidation, + viewStale, + viewWithoutRes, + viewNotInDB, + viewByHost, + viewAllServers, +} + +func parseViews(viewsFlag string) viewSet { + views := make(viewSet) + if viewsFlag == "all" || viewsFlag == "" { + for _, v := range allViews { + views[v] = true + } + return views + } + for _, v := range strings.Split(viewsFlag, ",") { + v = strings.TrimSpace(v) + if v != "" { + views[v] = true + } + } + return views +} + +func (v viewSet) has(view string) bool { + return v[view] +} + +func applyHideViews(views viewSet, hideFlag string) { + if hideFlag == "" { + return + } + for _, v := range strings.Split(hideFlag, ",") { + v = strings.TrimSpace(v) + if v != "" { + delete(views, v) + } + } +} + +func main() { + // Parse command line flags + sortBy := flag.String("sort", "vm", "Sort VMs by: vm (UUID), vm-host (VM's host), res-host (reservation host)") + postgresSecret := flag.String("postgres-secret", "cortex-nova-postgres", "Name of the kubernetes secret containing postgres credentials") + namespace := flag.String("namespace", "", "Namespace of the postgres secret (defaults to 'default')") + postgresHostOverride := flag.String("postgres-host", "", "Override postgres host (useful with port-forward, e.g., localhost)") + postgresPortOverride := flag.String("postgres-port", "", "Override postgres port (useful with port-forward, e.g., 5432)") + viewsFlag := flag.String("views", "all", "Comma-separated list of views to show (all, hypervisors, vms, reservations, summary, hypervisor-summary, validation, stale, without-res, not-in-db, by-host)") + hideFlag := flag.String("hide", "", "Comma-separated list of views to hide (applied after --views)") + filterName := flag.String("filter-name", "", "Filter hypervisors by name (substring match)") + filterTrait := flag.String("filter-trait", "", "Filter hypervisors by trait (e.g., CUSTOM_HANA_EXCLUSIVE_HOST)") + flag.Parse() + + views := parseViews(*viewsFlag) + applyHideViews(views, *hideFlag) + + ctx := context.Background() + + // Create kubernetes client + cfg, err := config.GetConfig() + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting kubeconfig: %v\n", err) + os.Exit(1) + } + + k8sClient, err := client.New(cfg, client.Options{Scheme: scheme}) + if err != nil { + fmt.Fprintf(os.Stderr, "Error creating client: %v\n", err) + os.Exit(1) + } + + // Determine namespace + ns := *namespace + if ns == "" { + ns = "default" // Default fallback + } + + // Try to connect to postgres + var db *sql.DB + var serverMap map[string]serverInfo + var flavorMap map[string]flavorInfo + + db, serverMap, flavorMap = connectToPostgres(ctx, k8sClient, *postgresSecret, ns, *postgresHostOverride, *postgresPortOverride) + if db != nil { + defer db.Close() + } + + // Get all hypervisors to find all VMs + var allHypervisors hv1.HypervisorList + if err := k8sClient.List(ctx, &allHypervisors); err != nil { + fmt.Fprintf(os.Stderr, "Error listing hypervisors: %v\n", err) + return + } + + // Apply filters to hypervisors + var hypervisors hv1.HypervisorList + filteredHosts := make(map[string]bool) // Track which hosts pass the filter + for _, hv := range allHypervisors.Items { + if matchesFilter(hv, *filterName, *filterTrait) { + hypervisors.Items = append(hypervisors.Items, hv) + filteredHosts[hv.Name] = true + } + } + + // Build map of all VMs from hypervisors + allVMs := make(map[string]*vmEntry) // vm_uuid -> vmEntry + for _, hv := range hypervisors.Items { + for _, inst := range hv.Status.Instances { + if inst.Active { + entry := &vmEntry{ + UUID: inst.ID, + Host: hv.Name, + Reservations: []string{}, + } + // Check if VM is in server table and get flavor info + if serverMap != nil { + if server, ok := serverMap[inst.ID]; ok { + entry.InServerTable = true + entry.FlavorName = server.FlavorName + if flavorMap != nil { + if flavor, ok := flavorMap[server.FlavorName]; ok { + entry.VCPUs = flavor.VCPUs + entry.RAMMb = flavor.RAMMb + entry.DiskGb = flavor.DiskGb + } + } + } + } + allVMs[inst.ID] = entry + } + } + } + + // Get all reservations (both failover and committed) + var allReservations v1alpha1.ReservationList + if err := k8sClient.List(ctx, &allReservations); err != nil { + fmt.Fprintf(os.Stderr, "Error listing reservations: %v\n", err) + return + } + + // Filter failover reservations for backward compatibility + var failoverReservations []v1alpha1.Reservation + for _, res := range allReservations.Items { + if res.Spec.Type == v1alpha1.ReservationTypeFailover { + failoverReservations = append(failoverReservations, res) + } + } + + // Filter reservations to only those on filtered hosts + var filteredReservations []v1alpha1.Reservation + var filteredFailoverReservations []v1alpha1.Reservation + hasFilter := *filterName != "" || *filterTrait != "" + if hasFilter { + for _, res := range allReservations.Items { + host := res.Status.Host + if host == "" { + host = res.Spec.TargetHost + } + if filteredHosts[host] { + filteredReservations = append(filteredReservations, res) + if res.Spec.Type == v1alpha1.ReservationTypeFailover { + filteredFailoverReservations = append(filteredFailoverReservations, res) + } + } + } + // Replace the reservation lists with filtered ones + allReservations.Items = filteredReservations + failoverReservations = filteredFailoverReservations + } + + printHeader("Failover Reservations Visualization") + if hasFilter { + fmt.Printf("Filter: name=%q, trait=%q\n", *filterName, *filterTrait) + fmt.Printf("Matched Hypervisors: %d (of %d total)\n", len(hypervisors.Items), len(allHypervisors.Items)) + } else { + fmt.Printf("Total Hypervisors: %d\n", len(hypervisors.Items)) + } + fmt.Printf("Total VMs (from hypervisors): %d\n", len(allVMs)) + fmt.Printf("Total Failover Reservations: %d\n", len(failoverReservations)) + fmt.Printf("Total All Reservations: %d\n", len(allReservations.Items)) + fmt.Printf("Sort by: %s\n", *sortBy) + fmt.Printf("Views: %s\n", *viewsFlag) + if db != nil { + fmt.Printf("Postgres: connected (servers: %d, flavors: %d)\n", len(serverMap), len(flavorMap)) + } else { + fmt.Printf("Postgres: not connected\n") + } + fmt.Println() + + // Print Hypervisor Summary + if views.has(viewHypervisorSummary) { + printHypervisorSummary(hypervisors.Items, allReservations.Items) + } + + // Print Flavor Summary (requires postgres) + if db != nil && views.has(viewFlavorSummary) { + printFlavorSummary(allVMs, flavorMap) + } + + // Print Hypervisors and their VMs + if views.has(viewHypervisors) { + printHeader("Hypervisors and their VMs") + + // Build hypervisor -> VMs map + hypervisorVMs := make(map[string][]string) + for _, hv := range hypervisors.Items { + hypervisorVMs[hv.Name] = []string{} + for _, inst := range hv.Status.Instances { + if inst.Active { + hypervisorVMs[hv.Name] = append(hypervisorVMs[hv.Name], inst.ID) + } + } + } + + // Sort hypervisor names + hypervisorNames := make([]string, 0, len(hypervisorVMs)) + for name := range hypervisorVMs { + hypervisorNames = append(hypervisorNames, name) + } + sort.Strings(hypervisorNames) + + for _, hvName := range hypervisorNames { + vms := hypervisorVMs[hvName] + sort.Strings(vms) + fmt.Printf("🖥️ %s (%d VMs)\n", hvName, len(vms)) + for _, vmUUID := range vms { + vmInfo := vmUUID + if entry, ok := allVMs[vmUUID]; ok && entry.FlavorName != "" { + ramGB := entry.RAMMb / 1024 + vmInfo = fmt.Sprintf("%s [%s, %dvcpu, %dGB]", vmUUID, entry.FlavorName, entry.VCPUs, ramGB) + } + fmt.Printf(" - %s\n", vmInfo) + } + fmt.Println() + } + } + + // Build VM -> Reservations mapping from reservations + vmsWithReservations := make(map[string]bool) + vmsInReservationsNotOnHypervisors := make([]*vmEntry, 0) // Track VMs in reservations but not on hypervisors + + // Build reservation host -> reservations map for the combined view + reservationsByHost := make(map[string][]v1alpha1.Reservation) + for _, res := range failoverReservations { + host := res.Status.Host + if host == "" { + host = res.Spec.TargetHost + } + if host != "" { + reservationsByHost[host] = append(reservationsByHost[host], res) + } + } + + // Build VM -> reservation names mapping (for the combined view) + vmToReservationNames := make(map[string][]string) // vm_uuid -> []reservation_name@host + for _, res := range failoverReservations { + if res.Status.FailoverReservation == nil || res.Status.FailoverReservation.Allocations == nil { + continue + } + resHost := res.Status.Host + if resHost == "" { + resHost = res.Spec.TargetHost + } + for vmUUID := range res.Status.FailoverReservation.Allocations { + vmToReservationNames[vmUUID] = append(vmToReservationNames[vmUUID], fmt.Sprintf("%s@%s", res.Name, resHost)) + } + } + + // Print Hypervisors with their VMs and Reservations (combined view) + if views.has(viewHypervisorsVMsRes) { + printHeader("Hypervisors and their VMs and Reservations") + + // Build hypervisor -> VMs map + hypervisorVMs := make(map[string][]string) + for _, hv := range hypervisors.Items { + hypervisorVMs[hv.Name] = []string{} + for _, inst := range hv.Status.Instances { + if inst.Active { + hypervisorVMs[hv.Name] = append(hypervisorVMs[hv.Name], inst.ID) + } + } + } + + // Sort hypervisor names by BB then node + hypervisorNames := make([]string, 0, len(hypervisorVMs)) + for name := range hypervisorVMs { + hypervisorNames = append(hypervisorNames, name) + } + sort.Slice(hypervisorNames, func(i, j int) bool { + bbI, nodeI := parseHypervisorName(hypervisorNames[i]) + bbJ, nodeJ := parseHypervisorName(hypervisorNames[j]) + if bbI != bbJ { + return bbI < bbJ + } + return nodeI < nodeJ + }) + + for _, hvName := range hypervisorNames { + vms := hypervisorVMs[hvName] + reservations := reservationsByHost[hvName] + sort.Strings(vms) + + fmt.Printf("🖥️ %s (%d VMs, %d Reservations)\n", hvName, len(vms), len(reservations)) + + // Print VMs section + if len(vms) > 0 { + fmt.Println(" 📋 VMs:") + for _, vmUUID := range vms { + vmInfo := vmUUID + if entry, ok := allVMs[vmUUID]; ok && entry.FlavorName != "" { + ramGB := entry.RAMMb / 1024 + vmInfo = fmt.Sprintf("%s [%s, %dvcpu, %dGB]", vmUUID, entry.FlavorName, entry.VCPUs, ramGB) + } + // Add reservation info for this VM + if resNames, ok := vmToReservationNames[vmUUID]; ok && len(resNames) > 0 { + vmInfo += " → " + strings.Join(resNames, ", ") + } else { + vmInfo += " → (no reservations)" + } + fmt.Printf(" - %s\n", vmInfo) + } + } + + // Print Reservations section + if len(reservations) > 0 { + // Sort reservations by name + sort.Slice(reservations, func(i, j int) bool { + return reservations[i].Name < reservations[j].Name + }) + + fmt.Println(" 📦 Reservations (hosted here):") + for _, res := range reservations { + ready := "?" + for _, cond := range res.Status.Conditions { + if cond.Type == "Ready" { + if cond.Status == "True" { + ready = "✅" + } else { + ready = "❌" + } + break + } + } + + // Get reservation size + var sizeStr string + if len(res.Spec.Resources) > 0 { + var parts []string + if vcpus, ok := res.Spec.Resources["vcpus"]; ok { + parts = append(parts, fmt.Sprintf("%dvcpu", vcpus.Value())) + } + if mem, ok := res.Spec.Resources["memory"]; ok { + memGB := mem.Value() / (1024 * 1024 * 1024) + parts = append(parts, fmt.Sprintf("%dGB", memGB)) + } + sizeStr = strings.Join(parts, ", ") + } + + // Get VMs allocated to this reservation + var vmCount int + var vmHosts []string + if res.Status.FailoverReservation != nil && res.Status.FailoverReservation.Allocations != nil { + vmCount = len(res.Status.FailoverReservation.Allocations) + hostSet := make(map[string]bool) + for _, vmHost := range res.Status.FailoverReservation.Allocations { + hostSet[vmHost] = true + } + for h := range hostSet { + vmHosts = append(vmHosts, h) + } + sort.Strings(vmHosts) + } + + resInfo := fmt.Sprintf("%s %s", ready, res.Name) + if sizeStr != "" { + resInfo += fmt.Sprintf(" [%s]", sizeStr) + } + if vmCount > 0 { + resInfo += fmt.Sprintf(" ← %d VMs from %s", vmCount, strings.Join(vmHosts, ", ")) + } + fmt.Printf(" - %s\n", resInfo) + } + } + + fmt.Println() + } + } + + for _, res := range failoverReservations { + if res.Status.FailoverReservation == nil || res.Status.FailoverReservation.Allocations == nil { + continue + } + resHost := res.Status.Host + if resHost == "" { + resHost = res.Spec.TargetHost + } + for vmUUID, vmHost := range res.Status.FailoverReservation.Allocations { + vmsWithReservations[vmUUID] = true + // Update or create VM entry + if allVMs[vmUUID] == nil { + // VM not in hypervisors but in reservation (might be stale/deleted) + entry := &vmEntry{ + UUID: vmUUID, + Host: vmHost, + Reservations: []string{}, + NotOnHypervisors: true, + ReservationSource: res.Name, + } + // Check if VM is in server table + if serverMap != nil { + if server, ok := serverMap[vmUUID]; ok { + entry.InServerTable = true + entry.FlavorName = server.FlavorName + if flavorMap != nil { + if flavor, ok := flavorMap[server.FlavorName]; ok { + entry.VCPUs = flavor.VCPUs + entry.RAMMb = flavor.RAMMb + entry.DiskGb = flavor.DiskGb + } + } + } + } + allVMs[vmUUID] = entry + vmsInReservationsNotOnHypervisors = append(vmsInReservationsNotOnHypervisors, entry) + } + allVMs[vmUUID].Reservations = append(allVMs[vmUUID].Reservations, fmt.Sprintf("%s@%s", res.Name, resHost)) + } + } + + // Print VMs and their Reservations + if views.has(viewVMs) { + printHeader("VMs and their Reservations") + if len(allVMs) == 0 { + fmt.Println("No VMs found.") + } else { + if db != nil { + fmt.Printf("%-40s %-25s %-5s %-20s %-6s %-8s %s\n", + "VM UUID", "VM Host", "InDB", "Flavor", "VCPUs", "RAM(GB)", "Reservations") + fmt.Printf("%-40s %-25s %-5s %-20s %-6s %-8s %s\n", + strings.Repeat("-", 40), strings.Repeat("-", 25), strings.Repeat("-", 5), + strings.Repeat("-", 20), strings.Repeat("-", 6), strings.Repeat("-", 8), + strings.Repeat("-", 30)) + } else { + fmt.Printf("%-40s %-25s %s\n", "VM UUID", "VM Host", "Reservations (name@host)") + fmt.Printf("%-40s %-25s %s\n", strings.Repeat("-", 40), strings.Repeat("-", 25), strings.Repeat("-", 30)) + } + + // Convert map to slice for sorting + vmList := make([]*vmEntry, 0, len(allVMs)) + for _, entry := range allVMs { + vmList = append(vmList, entry) + } + + // Sort based on flag + switch *sortBy { + case "vm-host": + sort.Slice(vmList, func(i, j int) bool { + if vmList[i].Host != vmList[j].Host { + return vmList[i].Host < vmList[j].Host + } + return vmList[i].UUID < vmList[j].UUID + }) + case "res-host": + sort.Slice(vmList, func(i, j int) bool { + // Get first reservation host for sorting (VMs without reservations sort last) + iResHost := "zzz" // Sort VMs without reservations last + jResHost := "zzz" + if len(vmList[i].Reservations) > 0 { + parts := strings.Split(vmList[i].Reservations[0], "@") + if len(parts) == 2 { + iResHost = parts[1] + } + } + if len(vmList[j].Reservations) > 0 { + parts := strings.Split(vmList[j].Reservations[0], "@") + if len(parts) == 2 { + jResHost = parts[1] + } + } + if iResHost != jResHost { + return iResHost < jResHost + } + return vmList[i].UUID < vmList[j].UUID + }) + default: // "vm" - sort by UUID + sort.Slice(vmList, func(i, j int) bool { + return vmList[i].UUID < vmList[j].UUID + }) + } + + for _, entry := range vmList { + reservationsList := "(none)" + if len(entry.Reservations) > 0 { + reservationsList = strings.Join(entry.Reservations, ", ") + } + if db != nil { + inDB := "❌" + if entry.InServerTable { + inDB = "✅" + } + ramGB := entry.RAMMb / 1024 + fmt.Printf("%-40s %-25s %-5s %-20s %-6d %-8d %s\n", + truncate(entry.UUID, 40), truncate(entry.Host, 25), inDB, + truncate(entry.FlavorName, 20), entry.VCPUs, ramGB, + reservationsList) + } else { + fmt.Printf("%-40s %-25s %s\n", truncate(entry.UUID, 40), truncate(entry.Host, 25), reservationsList) + } + } + } + fmt.Println() + } + + // Print VMs not in server table (only if postgres is connected) + if db != nil && views.has(viewNotInDB) { + printHeader("VMs NOT in Server Table (Hypervisor only)") + vmsNotInDB := make([]*vmEntry, 0) + for _, entry := range allVMs { + if !entry.InServerTable { + vmsNotInDB = append(vmsNotInDB, entry) + } + } + sort.Slice(vmsNotInDB, func(i, j int) bool { + if vmsNotInDB[i].Host != vmsNotInDB[j].Host { + return vmsNotInDB[i].Host < vmsNotInDB[j].Host + } + return vmsNotInDB[i].UUID < vmsNotInDB[j].UUID + }) + + if len(vmsNotInDB) == 0 { + fmt.Println(" ✅ All VMs from hypervisors are in the server table") + } else { + fmt.Printf(" ⚠️ %d VMs not in server table:\n\n", len(vmsNotInDB)) + fmt.Printf(" %-40s %s\n", "VM UUID", "VM Host") + fmt.Printf(" %-40s %s\n", strings.Repeat("-", 40), strings.Repeat("-", 25)) + for _, entry := range vmsNotInDB { + fmt.Printf(" %-40s %s\n", truncate(entry.UUID, 40), entry.Host) + } + } + fmt.Println() + } + + // Print VMs without reservations + if views.has(viewWithoutRes) { + printHeader("VMs WITHOUT Reservations") + vmsWithoutRes := make([]*vmEntry, 0) + for _, entry := range allVMs { + if len(entry.Reservations) == 0 { + vmsWithoutRes = append(vmsWithoutRes, entry) + } + } + sort.Slice(vmsWithoutRes, func(i, j int) bool { + if vmsWithoutRes[i].Host != vmsWithoutRes[j].Host { + return vmsWithoutRes[i].Host < vmsWithoutRes[j].Host + } + return vmsWithoutRes[i].UUID < vmsWithoutRes[j].UUID + }) + + // Count VMs without reservations that are not in DB + vmsWithoutResNotInDB := 0 + for _, entry := range vmsWithoutRes { + if !entry.InServerTable { + vmsWithoutResNotInDB++ + } + } + + if len(vmsWithoutRes) == 0 { + fmt.Println(" ✅ All VMs have at least one reservation") + } else { + if db != nil && vmsWithoutResNotInDB > 0 { + fmt.Printf(" ⚠️ %d VMs without any reservations (%d not in DB):\n\n", len(vmsWithoutRes), vmsWithoutResNotInDB) + } else { + fmt.Printf(" ⚠️ %d VMs without any reservations:\n\n", len(vmsWithoutRes)) + } + if db != nil { + fmt.Printf(" %-40s %-25s %-5s %-20s %-6s %-8s\n", "VM UUID", "VM Host", "InDB", "Flavor", "VCPUs", "RAM(GB)") + fmt.Printf(" %-40s %-25s %-5s %-20s %-6s %-8s\n", + strings.Repeat("-", 40), strings.Repeat("-", 25), strings.Repeat("-", 5), + strings.Repeat("-", 20), strings.Repeat("-", 6), strings.Repeat("-", 8)) + for _, entry := range vmsWithoutRes { + inDB := "❌" + if entry.InServerTable { + inDB = "✅" + } + ramGB := entry.RAMMb / 1024 + fmt.Printf(" %-40s %-25s %-5s %-20s %-6d %-8d\n", + truncate(entry.UUID, 40), entry.Host, inDB, truncate(entry.FlavorName, 20), + entry.VCPUs, ramGB) + } + } else { + fmt.Printf(" %-40s %s\n", "VM UUID", "VM Host") + fmt.Printf(" %-40s %s\n", strings.Repeat("-", 40), strings.Repeat("-", 25)) + for _, entry := range vmsWithoutRes { + fmt.Printf(" %-40s %s\n", truncate(entry.UUID, 40), entry.Host) + } + } + } + fmt.Println() + } + + // Print VMs in reservations but NOT on any hypervisor (stale/deleted VMs) + if views.has(viewStale) { + printHeader("VMs in Reservations but NOT on Hypervisors (STALE)") + if len(vmsInReservationsNotOnHypervisors) == 0 { + fmt.Println(" ✅ All VMs in reservations are found on hypervisors") + } else { + fmt.Printf(" ⚠️ %d VMs in reservations but not on any hypervisor:\n\n", len(vmsInReservationsNotOnHypervisors)) + + // Sort by reservation source + sort.Slice(vmsInReservationsNotOnHypervisors, func(i, j int) bool { + if vmsInReservationsNotOnHypervisors[i].ReservationSource != vmsInReservationsNotOnHypervisors[j].ReservationSource { + return vmsInReservationsNotOnHypervisors[i].ReservationSource < vmsInReservationsNotOnHypervisors[j].ReservationSource + } + return vmsInReservationsNotOnHypervisors[i].UUID < vmsInReservationsNotOnHypervisors[j].UUID + }) + + if db != nil { + fmt.Printf(" %-40s %-25s %-5s %-25s %s\n", "VM UUID", "Claimed Host", "InDB", "Found in Reservation", "All Reservations") + fmt.Printf(" %-40s %-25s %-5s %-25s %s\n", + strings.Repeat("-", 40), strings.Repeat("-", 25), strings.Repeat("-", 5), + strings.Repeat("-", 25), strings.Repeat("-", 30)) + for _, entry := range vmsInReservationsNotOnHypervisors { + inDB := "❌" + if entry.InServerTable { + inDB = "✅" + } + fmt.Printf(" %-40s %-25s %-5s %-25s %s\n", + truncate(entry.UUID, 40), truncate(entry.Host, 25), inDB, + truncate(entry.ReservationSource, 25), strings.Join(entry.Reservations, ", ")) + } + } else { + fmt.Printf(" %-40s %-25s %-25s %s\n", "VM UUID", "Claimed Host", "Found in Reservation", "All Reservations") + fmt.Printf(" %-40s %-25s %-25s %s\n", + strings.Repeat("-", 40), strings.Repeat("-", 25), strings.Repeat("-", 25), strings.Repeat("-", 30)) + for _, entry := range vmsInReservationsNotOnHypervisors { + fmt.Printf(" %-40s %-25s %-25s %s\n", + truncate(entry.UUID, 40), truncate(entry.Host, 25), + truncate(entry.ReservationSource, 25), strings.Join(entry.Reservations, ", ")) + } + } + fmt.Println() + fmt.Println(" Note: These VMs may have been deleted or moved. The failover controller should clean them up.") + } + fmt.Println() + } + + // Print Reservations and their VMs (multiline format) + if views.has(viewReservations) && len(failoverReservations) > 0 { + printHeader("Reservations and their VMs") + + // Sort reservations by name + sort.Slice(failoverReservations, func(i, j int) bool { + return failoverReservations[i].Name < failoverReservations[j].Name + }) + + for _, res := range failoverReservations { + host := res.Status.Host + if host == "" { + host = res.Spec.TargetHost + } + if host == "" { + host = "N/A" + } + + ready := "Unknown" + for _, cond := range res.Status.Conditions { + if cond.Type == "Ready" { + ready = string(cond.Status) + break + } + } + + fmt.Printf("📦 %s\n", res.Name) + fmt.Printf(" Host: %s\n", host) + fmt.Printf(" Ready: %s\n", ready) + + // Print reservation size from Spec.Resources if available + if len(res.Spec.Resources) > 0 { + var sizeInfo []string + for name, qty := range res.Spec.Resources { + // Convert to appropriate units + switch name { + case "memory": + // Memory is in bytes (resource.Quantity.Value() returns bytes) + // 4080Mi = 4080 * 1024 * 1024 bytes + memBytes := qty.Value() + memGB := memBytes / (1024 * 1024 * 1024) + if memGB > 0 { + sizeInfo = append(sizeInfo, fmt.Sprintf("%s: %dGB", name, memGB)) + } else { + // Less than 1GB, show in MB + memMB := memBytes / (1024 * 1024) + sizeInfo = append(sizeInfo, fmt.Sprintf("%s: %dMB", name, memMB)) + } + case "cpu": + sizeInfo = append(sizeInfo, fmt.Sprintf("%s: %d", name, qty.Value())) + default: + sizeInfo = append(sizeInfo, fmt.Sprintf("%s: %s", name, qty.String())) + } + } + sort.Strings(sizeInfo) + fmt.Printf(" Size: %s\n", strings.Join(sizeInfo, ", ")) + } else if db != nil && res.Status.FailoverReservation != nil && res.Status.FailoverReservation.Allocations != nil { + // Calculate size from the largest VM in the reservation + var maxVCPUs, maxRAMGB int + for vmUUID := range res.Status.FailoverReservation.Allocations { + if entry, ok := allVMs[vmUUID]; ok { + if entry.VCPUs > maxVCPUs { + maxVCPUs = entry.VCPUs + } + ramGB := entry.RAMMb / 1024 + if ramGB > maxRAMGB { + maxRAMGB = ramGB + } + } + } + if maxVCPUs > 0 || maxRAMGB > 0 { + fmt.Printf(" Size: cpu: %d, memory: %dGB (from largest VM)\n", maxVCPUs, maxRAMGB) + } + } + + if res.Status.FailoverReservation != nil && res.Status.FailoverReservation.Allocations != nil { + vmList := make([]string, 0, len(res.Status.FailoverReservation.Allocations)) + for vmUUID, vmHost := range res.Status.FailoverReservation.Allocations { + vmInfo := fmt.Sprintf("%s @ %s", vmUUID, vmHost) + // Add flavor info if available + if entry, ok := allVMs[vmUUID]; ok && entry.FlavorName != "" { + ramGB := entry.RAMMb / 1024 + vmInfo += fmt.Sprintf(" [%s, %dvcpu, %dGB]", entry.FlavorName, entry.VCPUs, ramGB) + } + vmList = append(vmList, vmInfo) + } + sort.Strings(vmList) + fmt.Printf(" VMs (%d):\n", len(vmList)) + for _, vm := range vmList { + fmt.Printf(" - %s\n", vm) + } + } else { + fmt.Println(" VMs: (none)") + } + fmt.Println() + } + } + + // Validation: Check for VMs with reservations on their own host + if views.has(viewValidation) { + printHeader("Validation: VMs with reservations on same host (ERRORS)") + errorsFound := 0 + for _, res := range failoverReservations { + resHost := res.Status.Host + if resHost == "" { + resHost = res.Spec.TargetHost + } + if res.Status.FailoverReservation != nil && res.Status.FailoverReservation.Allocations != nil { + for vmUUID, vmHost := range res.Status.FailoverReservation.Allocations { + if vmHost == resHost { + fmt.Printf(" ❌ ERROR: VM %s is on host %s, but has reservation %s on SAME host %s\n", + vmUUID, vmHost, res.Name, resHost) + errorsFound++ + } + } + } + } + if errorsFound == 0 { + fmt.Println(" ✅ No errors found - all VMs have reservations on different hosts") + } else { + fmt.Printf("\n Total errors: %d\n", errorsFound) + } + fmt.Println() + } + + // Reservations by Host + if views.has(viewByHost) { + printHeader("Reservations by Host") + hostCounts := make(map[string]int) + for _, res := range failoverReservations { + host := res.Status.Host + if host == "" { + host = res.Spec.TargetHost + } + if host == "" { + host = "N/A" + } + hostCounts[host]++ + } + + // Sort hosts by count (descending) + type hostCount struct { + host string + count int + } + hostCountList := make([]hostCount, 0, len(hostCounts)) + for host, count := range hostCounts { + hostCountList = append(hostCountList, hostCount{host, count}) + } + sort.Slice(hostCountList, func(i, j int) bool { + return hostCountList[i].count > hostCountList[j].count + }) + + for _, hc := range hostCountList { + fmt.Printf(" %s: %d reservation(s)\n", hc.host, hc.count) + } + fmt.Println() + } + + // Summary Statistics + if views.has(viewSummary) { + printHeader("Summary Statistics") + + // Database connection status + if db != nil { + fmt.Printf("Database: ✅ connected (servers: %d, flavors: %d)\n", len(serverMap), len(flavorMap)) + } else { + fmt.Printf("Database: ❌ not connected\n") + } + fmt.Println() + + readyCount := 0 + notReadyCount := 0 + unknownCount := 0 + for _, res := range failoverReservations { + found := false + for _, cond := range res.Status.Conditions { + if cond.Type == "Ready" { + found = true + if cond.Status == "True" { + readyCount++ + } else { + notReadyCount++ + } + break + } + } + if !found { + unknownCount++ + } + } + + fmt.Println("Reservations by Status:") + fmt.Printf(" Ready: %d\n", readyCount) + fmt.Printf(" Not Ready: %d\n", notReadyCount) + fmt.Printf(" Unknown: %d\n", unknownCount) + fmt.Println() + + // Count VMs per reservation + totalVMsInReservations := 0 + for _, res := range failoverReservations { + if res.Status.FailoverReservation != nil && res.Status.FailoverReservation.Allocations != nil { + totalVMsInReservations += len(res.Status.FailoverReservation.Allocations) + } + } + + // Count VMs without reservations + vmsWithoutRes := 0 + vmsWithoutResNotInDB := 0 + for _, entry := range allVMs { + if len(entry.Reservations) == 0 { + vmsWithoutRes++ + if !entry.InServerTable { + vmsWithoutResNotInDB++ + } + } + } + + fmt.Printf("Total Hypervisors: %d\n", len(hypervisors.Items)) + fmt.Printf("Total VMs (from hypervisors): %d\n", len(allVMs)) + if db != nil { + vmsInDB := 0 + for _, entry := range allVMs { + if entry.InServerTable { + vmsInDB++ + } + } + fmt.Printf("VMs in server table: %d\n", vmsInDB) + fmt.Printf("VMs NOT in server table: %d\n", len(allVMs)-vmsInDB) + } + fmt.Printf("VMs with reservations: %d\n", len(vmsWithReservations)) + if vmsWithoutRes > 0 { + if db != nil && vmsWithoutResNotInDB > 0 { + fmt.Printf("VMs without reservations: %d (%d not in DB) ⚠️\n", vmsWithoutRes, vmsWithoutResNotInDB) + } else { + fmt.Printf("VMs without reservations: %d ⚠️\n", vmsWithoutRes) + } + } else { + fmt.Printf("VMs without reservations: %d\n", vmsWithoutRes) + } + fmt.Printf("Total Reservations: %d\n", len(failoverReservations)) + fmt.Printf("Total VM allocations across all reservations: %d\n", totalVMsInReservations) + if len(failoverReservations) > 0 { + fmt.Printf("Average VMs per reservation: %.2f\n", float64(totalVMsInReservations)/float64(len(failoverReservations))) + } + + // Count unique hosts with reservations + uniqueHosts := make(map[string]bool) + for _, res := range failoverReservations { + host := res.Status.Host + if host == "" { + host = res.Spec.TargetHost + } + if host != "" { + uniqueHosts[host] = true + } + } + fmt.Printf("Unique hosts with reservations: %d\n", len(uniqueHosts)) + fmt.Println() + + // Resource usage summary (VMs vs Reservations) + var totalVMCPU int64 + var totalVMRAMGB int64 + var totalResCPU int64 + var totalResRAMGB int64 + + // Calculate total VM resources + for _, entry := range allVMs { + if !entry.NotOnHypervisors { + totalVMCPU += int64(entry.VCPUs) + totalVMRAMGB += int64(entry.RAMMb / 1024) + } + } + + // Calculate total reservation resources + for _, res := range failoverReservations { + if len(res.Spec.Resources) > 0 { + if vcpus, ok := res.Spec.Resources["vcpus"]; ok { + totalResCPU += vcpus.Value() + } + if mem, ok := res.Spec.Resources["memory"]; ok { + totalResRAMGB += mem.Value() / (1024 * 1024 * 1024) + } + } + } + + fmt.Println("Resource Usage (VMs vs Failover Reservations):") + fmt.Printf(" VMs Total: %4d vCPUs, %6d GB RAM\n", totalVMCPU, totalVMRAMGB) + fmt.Printf(" Reservations Total: %4d vCPUs, %6d GB RAM\n", totalResCPU, totalResRAMGB) + + // Calculate and display ratios + if totalVMCPU > 0 { + cpuRatio := float64(totalResCPU) / float64(totalVMCPU) + fmt.Printf(" CPU Ratio (Res/VM): %.2f (%.0f%% of VM capacity reserved for failover)\n", cpuRatio, cpuRatio*100) + } + if totalVMRAMGB > 0 { + ramRatio := float64(totalResRAMGB) / float64(totalVMRAMGB) + fmt.Printf(" RAM Ratio (Res/VM): %.2f (%.0f%% of VM capacity reserved for failover)\n", ramRatio, ramRatio*100) + } + fmt.Println() + } + + // Print all servers from postgres (for debugging data sync issues) + if db != nil && views.has(viewAllServers) { + printAllServers(serverMap, flavorMap, allVMs, filteredHosts) + } +} + +func printHypervisorSummary(hypervisors []hv1.Hypervisor, reservations []v1alpha1.Reservation) { + printHeader("Hypervisor Capacity Summary") + + // Build reservation resources per host + failoverResPerHost := make(map[string]map[string]int64) // host -> resource -> value + committedResPerHost := make(map[string]map[string]int64) // host -> resource -> value + reservationCountPerHost := make(map[string]int) // host -> count + + for _, res := range reservations { + host := res.Status.Host + if host == "" { + host = res.Spec.TargetHost + } + if host == "" { + continue + } + + reservationCountPerHost[host]++ + + // Get resources from spec + if len(res.Spec.Resources) == 0 { + continue + } + + var targetMap map[string]map[string]int64 + switch res.Spec.Type { + case v1alpha1.ReservationTypeFailover: + targetMap = failoverResPerHost + case v1alpha1.ReservationTypeCommittedResource: + targetMap = committedResPerHost + default: + continue + } + + if targetMap[host] == nil { + targetMap[host] = make(map[string]int64) + } + + for name, qty := range res.Spec.Resources { + targetMap[host][string(name)] += qty.Value() + } + } + + // Build summary for each hypervisor + summaries := make([]hypervisorSummary, 0, len(hypervisors)) + + for _, hv := range hypervisors { + summary := hypervisorSummary{ + Name: hv.Name, + NumVMs: hv.Status.NumInstances, + NumReservations: reservationCountPerHost[hv.Name], + Traits: hv.Status.Traits, + } + + // Get capacity from hypervisor status + if cpu, ok := hv.Status.Capacity["cpu"]; ok { + summary.CapacityCPU = cpu.Value() + } + if mem, ok := hv.Status.Capacity["memory"]; ok { + // Memory is in bytes, convert to GB + summary.CapacityMemoryGB = mem.Value() / (1024 * 1024 * 1024) + } + + // Get allocation (used by VMs) from hypervisor status + if cpu, ok := hv.Status.Allocation["cpu"]; ok { + summary.UsedByVMsCPU = cpu.Value() + } + if mem, ok := hv.Status.Allocation["memory"]; ok { + // Memory is in bytes, convert to GB + summary.UsedByVMsMemoryGB = mem.Value() / (1024 * 1024 * 1024) + } + + // Get failover reservation resources + // Note: Reservations use "vcpus" key, not "cpu" + if failoverRes, ok := failoverResPerHost[hv.Name]; ok { + if cpu, ok := failoverRes["vcpus"]; ok { + summary.FailoverResCPU = cpu + } + if mem, ok := failoverRes["memory"]; ok { + // Memory from reservations is in bytes + summary.FailoverResMemGB = mem / (1024 * 1024 * 1024) + } + } + + // Get committed reservation resources + // Note: Reservations use "vcpus" key, not "cpu" + if committedRes, ok := committedResPerHost[hv.Name]; ok { + if cpu, ok := committedRes["vcpus"]; ok { + summary.CommittedResCPU = cpu + } + if mem, ok := committedRes["memory"]; ok { + // Memory from reservations is in bytes + summary.CommittedResMemGB = mem / (1024 * 1024 * 1024) + } + } + + // Calculate free resources + summary.FreeCPU = summary.CapacityCPU - summary.UsedByVMsCPU - summary.FailoverResCPU - summary.CommittedResCPU + summary.FreeMemoryGB = summary.CapacityMemoryGB - summary.UsedByVMsMemoryGB - summary.FailoverResMemGB - summary.CommittedResMemGB + + summaries = append(summaries, summary) + } + + // Sort by BB (blade block) then by node number + // Hypervisor names follow the pattern: nodeXXX-bbYYY (e.g., node001-bb086) + sort.Slice(summaries, func(i, j int) bool { + bbI, nodeI := parseHypervisorName(summaries[i].Name) + bbJ, nodeJ := parseHypervisorName(summaries[j].Name) + if bbI != bbJ { + return bbI < bbJ + } + return nodeI < nodeJ + }) + + // Print table header + fmt.Printf("%-30s %5s %5s %15s %15s %15s %15s %15s %s\n", + "Hypervisor", "#VMs", "#Res", "Capacity", "Used by VMs", "Failover Res", "Committed Res", "Free", "Traits") + fmt.Printf("%-30s %5s %5s %15s %15s %15s %15s %15s %s\n", + "", "", "", "(CPU / RAM)", "(CPU / RAM)", "(CPU / RAM)", "(CPU / RAM)", "(CPU / RAM)", "") + fmt.Printf("%-30s %5s %5s %15s %15s %15s %15s %15s %s\n", + strings.Repeat("-", 30), strings.Repeat("-", 5), strings.Repeat("-", 5), + strings.Repeat("-", 15), strings.Repeat("-", 15), strings.Repeat("-", 15), + strings.Repeat("-", 15), strings.Repeat("-", 15), strings.Repeat("-", 40)) + + // Print each hypervisor + for _, s := range summaries { + capacityStr := fmt.Sprintf("%d / %dGB", s.CapacityCPU, s.CapacityMemoryGB) + usedByVMsStr := fmt.Sprintf("%d / %dGB", s.UsedByVMsCPU, s.UsedByVMsMemoryGB) + failoverStr := fmt.Sprintf("%d / %dGB", s.FailoverResCPU, s.FailoverResMemGB) + committedStr := fmt.Sprintf("%d / %dGB", s.CommittedResCPU, s.CommittedResMemGB) + freeStr := fmt.Sprintf("%d / %dGB", s.FreeCPU, s.FreeMemoryGB) + traitsStr := formatTraits(s.Traits) + + fmt.Printf("%-30s %5d %5d %15s %15s %15s %15s %15s %s\n", + truncate(s.Name, 30), s.NumVMs, s.NumReservations, + capacityStr, usedByVMsStr, failoverStr, committedStr, freeStr, traitsStr) + } + + // Print totals + var totalCapCPU, totalCapMem, totalUsedCPU, totalUsedMem int64 + var totalFailoverCPU, totalFailoverMem, totalCommittedCPU, totalCommittedMem int64 + var totalFreeCPU, totalFreeMem int64 + var totalVMs, totalRes int + + for _, s := range summaries { + totalVMs += s.NumVMs + totalRes += s.NumReservations + totalCapCPU += s.CapacityCPU + totalCapMem += s.CapacityMemoryGB + totalUsedCPU += s.UsedByVMsCPU + totalUsedMem += s.UsedByVMsMemoryGB + totalFailoverCPU += s.FailoverResCPU + totalFailoverMem += s.FailoverResMemGB + totalCommittedCPU += s.CommittedResCPU + totalCommittedMem += s.CommittedResMemGB + totalFreeCPU += s.FreeCPU + totalFreeMem += s.FreeMemoryGB + } + + fmt.Printf("%-30s %5s %5s %15s %15s %15s %15s %15s\n", + strings.Repeat("-", 30), strings.Repeat("-", 5), strings.Repeat("-", 5), + strings.Repeat("-", 15), strings.Repeat("-", 15), strings.Repeat("-", 15), + strings.Repeat("-", 15), strings.Repeat("-", 15)) + fmt.Printf("%-30s %5d %5d %15s %15s %15s %15s %15s\n", + "TOTAL", totalVMs, totalRes, + fmt.Sprintf("%d / %dGB", totalCapCPU, totalCapMem), + fmt.Sprintf("%d / %dGB", totalUsedCPU, totalUsedMem), + fmt.Sprintf("%d / %dGB", totalFailoverCPU, totalFailoverMem), + fmt.Sprintf("%d / %dGB", totalCommittedCPU, totalCommittedMem), + fmt.Sprintf("%d / %dGB", totalFreeCPU, totalFreeMem)) + + fmt.Println() +} + +func connectToPostgres(ctx context.Context, k8sClient client.Client, secretName, namespace, hostOverride, portOverride string) (db *sql.DB, serverMap map[string]serverInfo, flavorMap map[string]flavorInfo) { + // Get the postgres secret + secret := &corev1.Secret{} + if err := k8sClient.Get(ctx, client.ObjectKey{ + Namespace: namespace, + Name: secretName, + }, secret); err != nil { + fmt.Fprintf(os.Stderr, "Warning: Could not get postgres secret '%s' in namespace '%s': %v\n", secretName, namespace, err) + fmt.Fprintf(os.Stderr, " Postgres features will be disabled.\n") + fmt.Fprintf(os.Stderr, " Use --postgres-secret and --namespace flags to specify the secret.\n\n") + return nil, nil, nil + } + + // Extract connection details + host := string(secret.Data["host"]) + port := string(secret.Data["port"]) + user := string(secret.Data["user"]) + password := string(secret.Data["password"]) + database := string(secret.Data["database"]) + + if user == "" || password == "" || database == "" { + fmt.Fprintf(os.Stderr, "Warning: Postgres secret is missing required fields (user, password, database)\n") + return nil, nil, nil + } + + if port == "" { + port = "5432" + } + + // Strip newlines from values + strip := func(s string) string { return strings.ReplaceAll(strings.TrimSpace(s), "\n", "") } + host = strip(host) + port = strip(port) + user = strip(user) + password = strip(password) + database = strip(database) + + // Apply overrides if provided + if hostOverride != "" { + host = hostOverride + } + if portOverride != "" { + port = portOverride + } + + if host == "" { + fmt.Fprintf(os.Stderr, "Warning: Postgres host is empty. Use --postgres-host to specify.\n") + return nil, nil, nil + } + + // Connect to postgres + connStr := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", + host, port, user, password, database) + + db, err := sql.Open("postgres", connStr) + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: Could not connect to postgres: %v\n", err) + return nil, nil, nil + } + + // Test connection + if err := db.PingContext(ctx); err != nil { + fmt.Fprintf(os.Stderr, "Warning: Could not ping postgres at %s:%s: %v\n", host, port, err) + fmt.Fprintf(os.Stderr, " If running locally, use kubectl port-forward:\n") + fmt.Fprintf(os.Stderr, " kubectl port-forward svc/%s %s:%s -n %s\n", host, port, port, namespace) + fmt.Fprintf(os.Stderr, " ./visualize-reservations --postgres-host=localhost --postgres-port=%s\n\n", port) + db.Close() + return nil, nil, nil + } + + // Query servers with host information + serverMap = make(map[string]serverInfo) + rows, err := db.QueryContext(ctx, "SELECT id, flavor_name, COALESCE(host_id, ''), COALESCE(os_ext_srv_attr_host, '') FROM openstack_servers") + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: Could not query openstack_servers: %v\n", err) + } else { + defer rows.Close() + for rows.Next() { + var s serverInfo + if err := rows.Scan(&s.ID, &s.FlavorName, &s.HostID, &s.OSEXTSRVATTRHost); err != nil { + fmt.Fprintf(os.Stderr, "Warning: Could not scan server row: %v\n", err) + continue + } + serverMap[s.ID] = s + } + if err := rows.Err(); err != nil { + fmt.Fprintf(os.Stderr, "Warning: Error iterating server rows: %v\n", err) + } + } + + // Query flavors (including extra_specs for traits) + flavorMap = make(map[string]flavorInfo) + rows2, err := db.QueryContext(ctx, "SELECT name, vcpus, ram, disk, COALESCE(extra_specs, '') FROM openstack_flavors_v2") + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: Could not query openstack_flavors_v2: %v\n", err) + } else { + defer rows2.Close() + for rows2.Next() { + var f flavorInfo + if err := rows2.Scan(&f.Name, &f.VCPUs, &f.RAMMb, &f.DiskGb, &f.ExtraSpecs); err != nil { + fmt.Fprintf(os.Stderr, "Warning: Could not scan flavor row: %v\n", err) + continue + } + flavorMap[f.Name] = f + } + if err := rows2.Err(); err != nil { + fmt.Fprintf(os.Stderr, "Warning: Error iterating flavor rows: %v\n", err) + } + } + + return db, serverMap, flavorMap +} + +func printHeader(title string) { + fmt.Println("==============================================") + fmt.Printf(" %s\n", title) + fmt.Println("==============================================") + fmt.Println() +} + +func truncate(s string, maxLen int) string { + if len(s) <= maxLen { + return s + } + return s[:maxLen-3] + "..." +} + +// parseHypervisorName extracts the BB (blade block) and node parts from a hypervisor name. +// Hypervisor names follow the pattern: nodeXXX-bbYYY (e.g., node001-bb086) +// Returns (bb, node) where bb is the blade block suffix and node is the node prefix. +// If the name doesn't match the expected pattern, returns the full name as both values. +func parseHypervisorName(name string) (bb, node string) { + // Look for the pattern: anything-bbXXX + parts := strings.Split(name, "-") + if len(parts) >= 2 { + // Find the bb part (could be at any position) + for i, part := range parts { + if strings.HasPrefix(part, "bb") { + bb = part + // Node is everything before the bb part + node = strings.Join(parts[:i], "-") + return bb, node + } + } + } + // Fallback: return the full name for both (will sort alphabetically) + return name, name +} + +// formatTraits formats the traits slice for display. +// It filters out common/standard traits and shows only custom or interesting ones. +func formatTraits(traits []string) string { + if len(traits) == 0 { + return "" + } + + // Filter out common standard traits that are not interesting + // Keep custom traits and important hardware traits + var interesting []string + for _, trait := range traits { + // Skip common standard traits + if strings.HasPrefix(trait, "COMPUTE_") || + strings.HasPrefix(trait, "HW_CPU_X86_") || + strings.HasPrefix(trait, "MISC_SHARES_") || + trait == "COMPUTE_NET_ATTACH_INTERFACE" || + trait == "COMPUTE_VOLUME_ATTACH" || + trait == "COMPUTE_VOLUME_EXTEND" || + trait == "COMPUTE_VOLUME_MULTI_ATTACH" { + continue + } + interesting = append(interesting, trait) + } + + if len(interesting) == 0 { + return "" + } + + // Sort for consistent output + sort.Strings(interesting) + + // Join with commas + return strings.Join(interesting, ", ") +} + +// printFlavorSummary prints a summary of flavors with VM counts and traits +func printFlavorSummary(allVMs map[string]*vmEntry, flavorMap map[string]flavorInfo) { + printHeader("Flavor Summary") + + // Count VMs per flavor + flavorVMCount := make(map[string]int) + for _, vm := range allVMs { + if vm.FlavorName != "" { + flavorVMCount[vm.FlavorName]++ + } + } + + // Build summary list + type flavorSummary struct { + Name string + VMCount int + VCPUs int + RAMGb int + DiskGb int + Traits string + ExtraSpecs string + } + + summaries := make([]flavorSummary, 0) + for flavorName, count := range flavorVMCount { + summary := flavorSummary{ + Name: flavorName, + VMCount: count, + } + if flavor, ok := flavorMap[flavorName]; ok { + summary.VCPUs = flavor.VCPUs + summary.RAMGb = flavor.RAMMb / 1024 + summary.DiskGb = flavor.DiskGb + summary.ExtraSpecs = flavor.ExtraSpecs + summary.Traits = extractTraitsFromExtraSpecs(flavor.ExtraSpecs) + } + summaries = append(summaries, summary) + } + + // Sort by VM count (descending), then by name + sort.Slice(summaries, func(i, j int) bool { + if summaries[i].VMCount != summaries[j].VMCount { + return summaries[i].VMCount > summaries[j].VMCount + } + return summaries[i].Name < summaries[j].Name + }) + + // Print table + fmt.Printf("%-40s %6s %15s %s\n", "Flavor", "#VMs", "Size", "Traits") + fmt.Printf("%-40s %6s %15s %s\n", "", "", "(CPU/RAM/Disk)", "") + fmt.Printf("%-40s %6s %15s %s\n", + strings.Repeat("-", 40), strings.Repeat("-", 6), strings.Repeat("-", 15), strings.Repeat("-", 50)) + + for _, s := range summaries { + sizeStr := fmt.Sprintf("%d / %dGB / %dGB", s.VCPUs, s.RAMGb, s.DiskGb) + fmt.Printf("%-40s %6d %15s %s\n", + truncate(s.Name, 40), s.VMCount, sizeStr, s.Traits) + } + + // Print total + totalVMs := 0 + for _, s := range summaries { + totalVMs += s.VMCount + } + fmt.Printf("%-40s %6s %15s\n", + strings.Repeat("-", 40), strings.Repeat("-", 6), strings.Repeat("-", 15)) + fmt.Printf("%-40s %6d\n", "TOTAL", totalVMs) + fmt.Println() +} + +// extractTraitsFromExtraSpecs extracts trait requirements from flavor extra_specs JSON +// Shows both required and forbidden traits with indicators +func extractTraitsFromExtraSpecs(extraSpecsJSON string) string { + if extraSpecsJSON == "" { + return "" + } + + // Parse JSON + var extraSpecs map[string]string + if err := json.Unmarshal([]byte(extraSpecsJSON), &extraSpecs); err != nil { + return "" + } + + // Extract traits (keys starting with "trait:") + var requiredTraits []string + var forbiddenTraits []string + for key, value := range extraSpecs { + if strings.HasPrefix(key, "trait:") { + trait := strings.TrimPrefix(key, "trait:") + // Skip common traits + if strings.HasPrefix(trait, "COMPUTE_") || + strings.HasPrefix(trait, "HW_CPU_X86_") { + continue + } + switch value { + case "required": + requiredTraits = append(requiredTraits, trait) + case "forbidden": + forbiddenTraits = append(forbiddenTraits, trait) + } + } + } + + if len(requiredTraits) == 0 && len(forbiddenTraits) == 0 { + return "" + } + + // Sort for consistent output + sort.Strings(requiredTraits) + sort.Strings(forbiddenTraits) + + // Build output with indicators + var parts []string + for _, t := range requiredTraits { + parts = append(parts, "+"+t) // + for required + } + for _, t := range forbiddenTraits { + parts = append(parts, "-"+t) // - for forbidden + } + + return strings.Join(parts, ", ") +} + +// matchesFilter checks if a hypervisor matches the given name and trait filters. +// If both filters are empty, all hypervisors match. +// Name filter uses substring matching (case-insensitive). +// Trait filter checks if the hypervisor has the specified trait. +func matchesFilter(hv hv1.Hypervisor, nameFilter, traitFilter string) bool { + // If no filters, match all + if nameFilter == "" && traitFilter == "" { + return true + } + + // Check name filter (substring match, case-insensitive) + if nameFilter != "" { + if !strings.Contains(strings.ToLower(hv.Name), strings.ToLower(nameFilter)) { + return false + } + } + + // Check trait filter + if traitFilter != "" { + hasTrait := false + for _, trait := range hv.Status.Traits { + if strings.EqualFold(trait, traitFilter) { + hasTrait = true + break + } + } + if !hasTrait { + return false + } + } + + return true +} + +// printAllServers prints servers from postgres that are on hypervisors +// This is useful for debugging data sync issues between nova and hypervisor operator +func printAllServers(serverMap map[string]serverInfo, _ map[string]flavorInfo, allVMs map[string]*vmEntry, _ map[string]bool) { + printHeader("Servers on Hypervisors (data sync debugging)") + + if len(allVMs) == 0 { + fmt.Println(" No VMs found on hypervisors") + fmt.Println() + return + } + + // Build list of VMs on hypervisors with their postgres info + type vmWithPGInfo struct { + UUID string + ActualHost string // From hypervisor CRD + PGHost string // From postgres (OSEXTSRVATTRHost) + FlavorName string + InPostgres bool + HostMatch bool + Status string + } + + vms := make([]vmWithPGInfo, 0, len(allVMs)) + for uuid, vm := range allVMs { + if vm.NotOnHypervisors { + continue // Skip VMs that are only in reservations but not on hypervisors + } + + info := vmWithPGInfo{ + UUID: uuid, + ActualHost: vm.Host, + FlavorName: vm.FlavorName, + } + + // Check if VM is in postgres + if server, ok := serverMap[uuid]; ok { + info.InPostgres = true + info.PGHost = server.OSEXTSRVATTRHost + if info.FlavorName == "" { + info.FlavorName = server.FlavorName + } + + // Check host match + switch server.OSEXTSRVATTRHost { + case "": + info.Status = "PG_NO_HOST" + case vm.Host: + info.HostMatch = true + info.Status = "OK" + default: + info.Status = "WRONG_HOST" + } + } else { + info.Status = "NOT_IN_PG" + } + + vms = append(vms, info) + } + + // Sort by status (errors first), then by host, then by ID + statusOrder := map[string]int{ + "WRONG_HOST": 0, + "NOT_IN_PG": 1, + "PG_NO_HOST": 2, + "OK": 3, + } + sort.Slice(vms, func(i, j int) bool { + if statusOrder[vms[i].Status] != statusOrder[vms[j].Status] { + return statusOrder[vms[i].Status] < statusOrder[vms[j].Status] + } + if vms[i].ActualHost != vms[j].ActualHost { + return vms[i].ActualHost < vms[j].ActualHost + } + return vms[i].UUID < vms[j].UUID + }) + + // Count statistics + var wrongHost, notInPG, pgNoHost, ok int + for _, v := range vms { + switch v.Status { + case "WRONG_HOST": + wrongHost++ + case "NOT_IN_PG": + notInPG++ + case "PG_NO_HOST": + pgNoHost++ + case "OK": + ok++ + } + } + + fmt.Printf(" Total VMs on hypervisors: %d\n", len(vms)) + fmt.Printf(" - OK (host matches): %d\n", ok) + fmt.Printf(" - WRONG_HOST (postgres != hypervisor): %d ⚠️\n", wrongHost) + fmt.Printf(" - NOT_IN_PG (not in postgres): %d\n", notInPG) + fmt.Printf(" - PG_NO_HOST (postgres has empty host): %d\n", pgNoHost) + fmt.Println() + + // Print table header + fmt.Printf(" %-40s %-25s %-25s %-12s %-20s\n", + "Server ID", "Actual Host (HV)", "OSEXTSRVATTRHost (PG)", "Status", "Flavor") + fmt.Printf(" %-40s %-25s %-25s %-12s %-20s\n", + strings.Repeat("-", 40), strings.Repeat("-", 25), strings.Repeat("-", 25), + strings.Repeat("-", 12), strings.Repeat("-", 20)) + + for _, v := range vms { + statusIcon := "" + switch v.Status { + case "OK": + statusIcon = "✅ OK" + case "WRONG_HOST": + statusIcon = "❌ WRONG" + case "NOT_IN_PG": + statusIcon = "❓ NO_PG" + case "PG_NO_HOST": + statusIcon = "❓ NO_HOST" + } + + fmt.Printf(" %-40s %-25s %-25s %-12s %-20s\n", + truncate(v.UUID, 40), + truncate(v.ActualHost, 25), + truncate(v.PGHost, 25), + statusIcon, + truncate(v.FlavorName, 20)) + } + fmt.Println() +} + +// Ensure resource.Quantity is used (for compile check) +var _ = resource.Quantity{}