From d7196605e2b5e6b123620d2278cb7d65dcb2c636 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Mon, 1 Sep 2025 22:12:52 -0400 Subject: [PATCH 01/50] Add terragrunt unit to create VMs for a MAAS environment Signed-off-by: Felipe Reyes --- testing/.gitignore | 4 + testing/units/maas/main.tf | 0 testing/units/maas/terragrunt.hcl | 20 ++ testing/units/virtualnodes/main.tf | 223 ++++++++++++++++++ .../templates/maas_controller.cloudinit.cfg | 81 +++++++ .../templates/maas_controller.netplan.yaml | 15 ++ testing/units/virtualnodes/terragrunt.hcl | 13 + testing/units/virtualnodes/variables.tf | 83 +++++++ 8 files changed, 439 insertions(+) create mode 100644 testing/.gitignore create mode 100644 testing/units/maas/main.tf create mode 100644 testing/units/maas/terragrunt.hcl create mode 100644 testing/units/virtualnodes/main.tf create mode 100644 testing/units/virtualnodes/templates/maas_controller.cloudinit.cfg create mode 100644 testing/units/virtualnodes/templates/maas_controller.netplan.yaml create mode 100644 testing/units/virtualnodes/terragrunt.hcl create mode 100644 testing/units/virtualnodes/variables.tf diff --git a/testing/.gitignore b/testing/.gitignore new file mode 100644 index 000000000..a8bb00dc2 --- /dev/null +++ b/testing/.gitignore @@ -0,0 +1,4 @@ +*.tfstate +*.tfstate.backup +*.lock.hcl +.terraform/ diff --git a/testing/units/maas/main.tf b/testing/units/maas/main.tf new file mode 100644 index 000000000..e69de29bb diff --git a/testing/units/maas/terragrunt.hcl b/testing/units/maas/terragrunt.hcl new file mode 100644 index 000000000..e4f86fdcb --- /dev/null +++ b/testing/units/maas/terragrunt.hcl @@ -0,0 +1,20 @@ +include "stack" { + path = find_in_parent_folders("stack.hcl") + expose = true +} + +terraform { + source = "." +} + +dependency "vpc" { + config_path = "../virtualnodes" +} + +inputs = merge( + include.stack.locals.stack_config, + { + maas_controller_ip_address = dependency.virtualnodes.outputs.maas_controller_ip_address + nodes = dependency.virtualnodes.outputs.nodes + } +) diff --git a/testing/units/virtualnodes/main.tf b/testing/units/virtualnodes/main.tf new file mode 100644 index 000000000..12a8f6d12 --- /dev/null +++ b/testing/units/virtualnodes/main.tf @@ -0,0 +1,223 @@ +terraform { + required_version = ">= 0.14.0" + required_providers { + libvirt = { + source = "dmacvicar/libvirt" + version = "0.8.3" + } + external = { + source = "hashicorp/external" + version = "2.3.5" + } + } +} + +provider "libvirt" { + uri = var.libvirt_uri +} + +#### Locals +locals { + maas_controller_ip_addr = "172.16.1.2" + generic_net_addresses = ["172.16.1.0/24"] + external_net_addresses = ["172.16.2.0/24"] +} + +#### Networks + +resource "libvirt_network" "generic_net" { + name = "generic_net" + mode = "nat" + + domain = var.generic_net_domain + addresses = local.generic_net_addresses + + dns { + enabled = false + } +} + +resource "libvirt_network" "external_net" { + name = "external_net" + mode = "nat" + + domain = var.external_net_domain + addresses = local.external_net_addresses + + dns { + enabled = false + } +} + +#### Volumes + +resource "libvirt_volume" "node_vol" { + name = "node_${count.index}.qcow2" + count = var.nodes_count + size = var.node_rootfs_size +} + +resource "libvirt_volume" "node_vol_secondary" { + name = "node_${count.index}_secondary.qcow2" + count = var.nodes_count + size = var.node_secondary_disk_size +} + + +resource "libvirt_volume" "ubuntu_noble" { + name = "ubuntu-noble.qcow2" + source = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" +} + +resource "libvirt_volume" "maas_controller_vol" { + name = "maas-controller-vol" + base_volume_id = libvirt_volume.ubuntu_noble.id + size = var.maas_controller_rootfs_size +} + +#### Virtual machines (domains) + +resource "libvirt_cloudinit_disk" "maas_controller_cloudinit" { + name = "maas_controller_cloudinit.iso" + user_data = templatefile( + "${path.module}/templates/maas_controller.cloudinit.cfg", + { + address = local.maas_controller_ip_addr + dns_server = var.upstream_dns_server + networks = "generic:172.16.1.0/24" + ssh_public_key = file(var.ssh_public_key_path) + }) + network_config = templatefile( + "${path.module}/templates/maas_controller.netplan.yaml", + { + dns_server = var.upstream_dns_server + ip_address = local.maas_controller_ip_addr + mac_address = var.maas_controller_mac_address + }) +} + +resource "libvirt_domain" "maas_controller" { + name = "maas-controller" + memory = var.maas_controller_mem + vcpu = var.maas_controller_vcpu + disk { + volume_id = libvirt_volume.maas_controller_vol.id + scsi = "true" + } + cloudinit = libvirt_cloudinit_disk.maas_controller_cloudinit.id + boot_device { + dev = [ "hd"] + } + network_interface { + network_id = libvirt_network.generic_net.id + hostname = "maas_controller" + addresses = [local.maas_controller_ip_addr] + mac = var.maas_controller_mac_address + } + console { + type = "pty" + target_type = "serial" + target_port = "0" + } + + console { + type = "pty" + target_type = "virtio" + target_port = "1" + } + graphics { + type = "spice" + listen_type = "address" + autoport = true + } + + connection { + type = "ssh" + user = "ubuntu" + private_key = file(var.ssh_private_key_path) + host = local.maas_controller_ip_addr + } + + provisioner "remote-exec" { + inline = [ + "until test -f /tmp/.i_am_done; do sleep 10;done", + ] + } +} + +resource "libvirt_domain" "node" { + depends_on = [ + libvirt_domain.maas_controller, + ] + count = var.nodes_count + name = "node-${count.index}" + memory = var.node_mem + vcpu = var.node_vcpu + running = false + disk { + volume_id = libvirt_volume.node_vol[count.index].id + scsi = "true" + } + disk { + volume_id = libvirt_volume.node_vol_secondary[count.index].id + scsi = "true" + } + boot_device { + dev = [ "network"] + } + network_interface { + network_id = libvirt_network.generic_net.id + hostname = "node-${count.index}" + mac = format("AA:BB:CC:11:22:%02d", count.index + 10) + } + network_interface { + network_id = libvirt_network.external_net.id + mac = format("AA:BB:CC:33:44:%02d", count.index + 10) + } + console { + type = "pty" + target_type = "serial" + target_port = "0" + } + + console { + type = "pty" + target_type = "virtio" + target_port = "1" + } + graphics { + type = "spice" + listen_type = "address" + autoport = true + } +} + +data "external" "remote_command" { + depends_on = [ + libvirt_domain.maas_controller + ] + program = ["bash", "-c", <<-EOF + ssh -i ${var.ssh_private_key_path} ubuntu@${local.maas_controller_ip_addr} 'cat api.key' | jq -R '{apikey: .}' + EOF + ] +} + +output "maas_controller_ip_address" { + value = local.maas_controller_ip_addr + description = "MAAS Controller IP Address." +} + +output "nodes" { + description = "List of (virtual) nodes" + value = [ + for node in libvirt_domain.node : { + name = node.name + mac_address = node.network_interface[0].mac + } + ] +} + +output "maas_api_key" { + description = "MAAS Admin API Key" + value = data.external.remote_command.result.apikey +} diff --git a/testing/units/virtualnodes/templates/maas_controller.cloudinit.cfg b/testing/units/virtualnodes/templates/maas_controller.cloudinit.cfg new file mode 100644 index 000000000..dbc00ff80 --- /dev/null +++ b/testing/units/virtualnodes/templates/maas_controller.cloudinit.cfg @@ -0,0 +1,81 @@ +#cloud-config +ssh_pwauth: True +ssh_authorized_keys: + - ${ssh_public_key} +chpasswd: + list: | + ubuntu:linux + expire: False + +hostname: maas-controller +create_hostname_file: true +locale: C.UTF-8 +package_update: true +package_upgrade: true +packages: + - openssh-server + - snapd + - jq + - git + - python3-pip + - python3-venv + - moreutils +write_files: + - content: | + #!/bin/bash + set -x + sudo snap install maas --channel 3.6/edge + sudo snap install maas-test-db --channel 3.6/edge + + sudo -E maas init region+rack --database-uri maas-test-db:/// --maas-url http://${address}:5240/MAAS + sleep 25 + sudo maas createadmin --username admin --password admin --email admin + sudo maas apikey --username admin > /home/ubuntu/api.key + profile="admin" + maas login $profile http://${address}:5240/MAAS $(cat /home/ubuntu/api.key) + # Configure MAAS DNS forwarding to OpenStack network DNS server + maas $profile maas set-config name=upstream_dns value="${dns_server}" + # TODO: Find how to properly configure DNSSEC + maas $profile maas set-config name=dnssec_validation value=no + # Download noble image + # maas $profile boot-source-selections create 1 os="ubuntu" release=noble arches=amd64 subarches="*" labels="*" + # To make 'openstack console log show' work + maas $profile maas set-config name=kernel_opts value="console=ttyS0 console=tty0" + # Skip setup screen + maas admin maas set-config name=completed_intro value=true + + PRIMARY_RACK=$(maas $profile rack-controllers read | jq -r ".[] | .system_id") + + networks=(${networks}) + for network in "$${networks[@]}" ; do + SUBNET=$${network#*:} + SPACE=$${network%%:*} + space=space-$SPACE + FABRIC_ID=$(maas admin subnet read "$SUBNET" | jq -r ".vlan.fabric_id") + VLAN_TAG=$(maas admin subnet read "$SUBNET" | jq -r ".vlan.vid") + maas $profile spaces create name=$space + maas $profile fabric update $FABRIC_ID name=fabric-$SPACE + if [ "$SPACE" != "pxe-boot" ]; then + maas $profile vlan update $FABRIC_ID $VLAN_TAG space=$space primary_rack="$PRIMARY_RACK" + continue + fi + DHCP_START=$(python3 -c "import ipaddress as ip;print(list(ip.ip_network('$SUBNET').hosts())[-10])") + DHCP_END=$(python3 -c "import ipaddress as ip;print(list(ip.ip_network('$SUBNET').hosts())[-1])") + maas $profile ipranges create type=dynamic start_ip="$DHCP_START" end_ip="$DHCP_END" + maas $profile vlan update $FABRIC_ID $VLAN_TAG space=$space dhcp_on=True primary_rack="$PRIMARY_RACK" + done + + # Wait for boot os to be downloaded + while [ $(maas $profile boot-resources is-importing | cat) == "true" ]; + do + echo "waiting for boot resources to import" + sleep 15 + done + # Temporary workaround to get MAAS DHCP service started + sudo snap restart maas + touch /tmp/.i_am_done + path: /usr/local/share/init.sh + owner: root:root + permissions: "0555" +runcmd: + - sudo -i -u ubuntu /usr/local/share/init.sh diff --git a/testing/units/virtualnodes/templates/maas_controller.netplan.yaml b/testing/units/virtualnodes/templates/maas_controller.netplan.yaml new file mode 100644 index 000000000..eb6a43b07 --- /dev/null +++ b/testing/units/virtualnodes/templates/maas_controller.netplan.yaml @@ -0,0 +1,15 @@ +network: + version: 2 + renderer: networkd + ethernets: + virtio-nic: + match: + macaddress: ${mac_address} + addresses: + - ${ip_address}/24 + routes: + - to: 0.0.0.0/0 + via: 172.16.1.1 + nameservers: + addresses: + - ${dns_server} diff --git a/testing/units/virtualnodes/terragrunt.hcl b/testing/units/virtualnodes/terragrunt.hcl new file mode 100644 index 000000000..4b93efd2b --- /dev/null +++ b/testing/units/virtualnodes/terragrunt.hcl @@ -0,0 +1,13 @@ +include "stack" { + path = find_in_parent_folders("stack.hcl") + + # NOTE(freyes): expose shouldn't be needed to access `locals` in `stack.hcl`, + # although without it `stack` is `null`. + expose = true +} + +terraform { + source = "./" +} + +inputs = include.stack.locals.stack_config diff --git a/testing/units/virtualnodes/variables.tf b/testing/units/virtualnodes/variables.tf new file mode 100644 index 000000000..930998095 --- /dev/null +++ b/testing/units/virtualnodes/variables.tf @@ -0,0 +1,83 @@ +variable "libvirt_uri" { + type = string + default = "qemu:///system" +} + +variable "nodes_count" { + type = number + default = 6 +} + +variable "node_mem" { + type = string + default = "2048" +} + +variable "node_vcpu" { + type = number + default = 2 +} + +variable "maas_controller_mem" { + type = string + default = "4096" # 4GiB +} + +variable "maas_controller_vcpu" { + type = number + default = 2 +} + +variable "node_rootfs_size" { + description = "Node rootfs disk size (in bytes)" + type = number + default = 21474836480 # 20 GiB +} + +variable "node_secondary_disk_size" { + description = "Node secondary disk size (in bytes)" + type = number + default = 21474836480 # 20 GiB +} + +variable "generic_net_domain" { + description = "" + type = string + default = "generic.maas" +} + +variable "external_net_domain" { + description = "" + type = string + default = "external.maas" +} + +variable "ssh_public_key_path" { + description = "Path to the SSH public key to use in deployed nodes" + type = string + default = "~/.ssh/id_ecdsa.pub" +} + +variable "ssh_private_key_path" { + description = "Path to the SSH private key to use in deployed nodes" + type = string + default = "~/.ssh/id_ecdsa" +} + +variable "upstream_dns_server" { + description = "upstream dns server to use in MAAS" + type = string + default = "8.8.8.8" +} + +variable "maas_controller_mac_address" { + description = "MAC address to assign to the maas controller nic in the management network" + type = string + default = "AA:BB:CC:11:11:01" +} + +variable "maas_controller_rootfs_size" { + description = "MAAS Controller rootfs disk size (in bytes)" + type = number + default = 21474836480 # 20 GiB +} From a8d186384257ac52cb77f1d775a005eba735e3ee Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 10 Sep 2025 11:25:44 -0300 Subject: [PATCH 02/50] Add private directory. This directory is meant to be used to store sensitive information that shouldn't be committed to the git repository, but it's worth to not be written to /tmp since it's helpful when debugging, for example API keys, ssh keys, etc. Signed-off-by: Felipe Reyes --- testing/private/.keep | 0 testing/private/README.md | 1 + 2 files changed, 1 insertion(+) create mode 100644 testing/private/.keep create mode 100644 testing/private/README.md diff --git a/testing/private/.keep b/testing/private/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/testing/private/README.md b/testing/private/README.md new file mode 100644 index 000000000..c161760b9 --- /dev/null +++ b/testing/private/README.md @@ -0,0 +1 @@ +Directory to write temporary files that are private in nature, for example generated password-less ssh keys. From ce16536d1055d9485d8a21007521b863d614c419 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 10 Sep 2025 11:27:06 -0300 Subject: [PATCH 03/50] gitignore: add .terragrunt-cache Signed-off-by: Felipe Reyes --- testing/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/.gitignore b/testing/.gitignore index a8bb00dc2..a8fd9076e 100644 --- a/testing/.gitignore +++ b/testing/.gitignore @@ -2,3 +2,4 @@ *.tfstate.backup *.lock.hcl .terraform/ +.terragrunt-cache/ From d4bd1f8ebd36b4e6611bf5ad79b9039ded2a7614 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 10 Sep 2025 11:27:59 -0300 Subject: [PATCH 04/50] Generate maas provider configuration. The maas provider needs information that comes out of the virtualnodes unit, this change introduces autogeneration of the maas provider block based on it. To allow `plan` and `validate` subcommands keep working, the necessary data is mocked. Signed-off-by: Felipe Reyes --- testing/templates/maas-provider.tf.tpl | 43 ++++++++++++++++++++++++++ testing/units/maas/terragrunt.hcl | 24 +++++++++++++- 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 testing/templates/maas-provider.tf.tpl diff --git a/testing/templates/maas-provider.tf.tpl b/testing/templates/maas-provider.tf.tpl new file mode 100644 index 000000000..9d0294f7f --- /dev/null +++ b/testing/templates/maas-provider.tf.tpl @@ -0,0 +1,43 @@ +terraform { + required_version = ">= 0.14.0" + required_providers { + local = { + source = "hashicorp/local" + version = "2.5.3" + } + maas = { + source = "canonical/maas" + version = "2.6.0" + } + netparse = { + source = "gmeligio/netparse" + version = "0.0.4" + } + null = { + source = "hashicorp/null" + version = "3.2.4" + } + tls = { + source = "hashicorp/tls" + version = "4.1.0" + } + } +} + +provider "local" { +} + +provider "maas" { + api_version = "2.0" + api_key = "${api_key}" + api_url = "http://${controller_ip_address}:5240/MAAS" +} + +provider "null" { +} + +provider "netparse" { +} + +provider "tls" { +} diff --git a/testing/units/maas/terragrunt.hcl b/testing/units/maas/terragrunt.hcl index e4f86fdcb..1fc75b6ff 100644 --- a/testing/units/maas/terragrunt.hcl +++ b/testing/units/maas/terragrunt.hcl @@ -7,14 +7,36 @@ terraform { source = "." } -dependency "vpc" { +dependency "virtualnodes" { config_path = "../virtualnodes" + + mock_outputs = { + maas_api_key = "abc123-apikey" + maas_controller_ip_address = "1.2.3.4" + nodes = [] + } + mock_outputs_allowed_terraform_commands = ["plan", "validate"] +} + +# Generate provider configuration using dependency outputs +generate "provider" { + path = "provider.tf" + if_exists = "overwrite" + contents = templatefile("${get_parent_terragrunt_dir()}/templates/maas-provider.tf.tpl", { + api_key = dependency.virtualnodes.outputs.maas_api_key + controller_ip_address = dependency.virtualnodes.outputs.maas_controller_ip_address + }) } inputs = merge( include.stack.locals.stack_config, { + maas_api_key = dependency.virtualnodes.outputs.maas_api_key maas_controller_ip_address = dependency.virtualnodes.outputs.maas_controller_ip_address nodes = dependency.virtualnodes.outputs.nodes + # TODO(freyes): make `172.16.1.1` dynamic, it's the gateway of the generic + # network created by libvirt, so this should be an output of the + # virtualnodes' unit. + libvirt_uri = format("qemu+ssh://%s@%s/system", get_env("USER"), "172.16.1.1") } ) From 55790bbc58967adb2340e2c41578b430a00875b3 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 10 Sep 2025 11:44:41 -0300 Subject: [PATCH 05/50] virtualnodes: move output definitions to their own file Signed-off-by: Felipe Reyes --- testing/units/virtualnodes/main.tf | 20 -------------------- testing/units/virtualnodes/outputs.tf | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+), 20 deletions(-) create mode 100644 testing/units/virtualnodes/outputs.tf diff --git a/testing/units/virtualnodes/main.tf b/testing/units/virtualnodes/main.tf index 12a8f6d12..7fbc5e45e 100644 --- a/testing/units/virtualnodes/main.tf +++ b/testing/units/virtualnodes/main.tf @@ -201,23 +201,3 @@ data "external" "remote_command" { EOF ] } - -output "maas_controller_ip_address" { - value = local.maas_controller_ip_addr - description = "MAAS Controller IP Address." -} - -output "nodes" { - description = "List of (virtual) nodes" - value = [ - for node in libvirt_domain.node : { - name = node.name - mac_address = node.network_interface[0].mac - } - ] -} - -output "maas_api_key" { - description = "MAAS Admin API Key" - value = data.external.remote_command.result.apikey -} diff --git a/testing/units/virtualnodes/outputs.tf b/testing/units/virtualnodes/outputs.tf new file mode 100644 index 000000000..1a96b370c --- /dev/null +++ b/testing/units/virtualnodes/outputs.tf @@ -0,0 +1,23 @@ +output "maas_controller_ip_address" { + description = "MAAS Controller IP Address." + value = local.maas_controller_ip_addr + depends_on = [libvirt_domain.maas_controller] +} + +output "nodes" { + description = "List of (virtual) nodes" + value = [ + for node in libvirt_domain.node : { + name = node.name + mac_address = node.network_interface[0].mac + } + ] + depends_on = [libvirt_domain.node] +} + +output "maas_api_key" { + description = "MAAS Admin API Key" + value = data.external.remote_command.result.apikey + sensitive = true + depends_on = [libvirt_domain.maas_controller] +} From 4180db00492a84be7f40bda6e867bd4555a03851 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 10 Sep 2025 11:45:11 -0300 Subject: [PATCH 06/50] virtualnodes: make network definitions to autostart Signed-off-by: Felipe Reyes --- testing/units/virtualnodes/main.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testing/units/virtualnodes/main.tf b/testing/units/virtualnodes/main.tf index 7fbc5e45e..ac7c2fc10 100644 --- a/testing/units/virtualnodes/main.tf +++ b/testing/units/virtualnodes/main.tf @@ -28,6 +28,7 @@ locals { resource "libvirt_network" "generic_net" { name = "generic_net" mode = "nat" + autostart = true domain = var.generic_net_domain addresses = local.generic_net_addresses @@ -40,6 +41,7 @@ resource "libvirt_network" "generic_net" { resource "libvirt_network" "external_net" { name = "external_net" mode = "nat" + autostart = true domain = var.external_net_domain addresses = local.external_net_addresses From 532cb5bc0f44dad1611c19f25e90265bfe2a84c9 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 10 Sep 2025 11:46:13 -0300 Subject: [PATCH 07/50] virtualnodes: move maas controller hostname to a variable This allows to reuse the value in cloud-init configuration and any other place where the hostname needs to be referenced, e.g. when reading data from MAAS API Signed-off-by: Felipe Reyes --- testing/units/virtualnodes/main.tf | 3 ++- .../virtualnodes/templates/maas_controller.cloudinit.cfg | 2 +- testing/units/virtualnodes/variables.tf | 6 ++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/testing/units/virtualnodes/main.tf b/testing/units/virtualnodes/main.tf index ac7c2fc10..a814e5fe2 100644 --- a/testing/units/virtualnodes/main.tf +++ b/testing/units/virtualnodes/main.tf @@ -86,6 +86,7 @@ resource "libvirt_cloudinit_disk" "maas_controller_cloudinit" { { address = local.maas_controller_ip_addr dns_server = var.upstream_dns_server + maas_hostname = var.maas_hostname networks = "generic:172.16.1.0/24" ssh_public_key = file(var.ssh_public_key_path) }) @@ -112,7 +113,7 @@ resource "libvirt_domain" "maas_controller" { } network_interface { network_id = libvirt_network.generic_net.id - hostname = "maas_controller" + hostname = var.maas_hostname addresses = [local.maas_controller_ip_addr] mac = var.maas_controller_mac_address } diff --git a/testing/units/virtualnodes/templates/maas_controller.cloudinit.cfg b/testing/units/virtualnodes/templates/maas_controller.cloudinit.cfg index dbc00ff80..7ac1975f8 100644 --- a/testing/units/virtualnodes/templates/maas_controller.cloudinit.cfg +++ b/testing/units/virtualnodes/templates/maas_controller.cloudinit.cfg @@ -7,7 +7,7 @@ chpasswd: ubuntu:linux expire: False -hostname: maas-controller +hostname: ${maas_hostname} create_hostname_file: true locale: C.UTF-8 package_update: true diff --git a/testing/units/virtualnodes/variables.tf b/testing/units/virtualnodes/variables.tf index 930998095..5cbcac528 100644 --- a/testing/units/virtualnodes/variables.tf +++ b/testing/units/virtualnodes/variables.tf @@ -28,6 +28,12 @@ variable "maas_controller_vcpu" { default = 2 } +variable "maas_hostname" { + description = "MAAS Controller hostname" + type = string + default = "maas-controller" +} + variable "node_rootfs_size" { description = "Node rootfs disk size (in bytes)" type = number From de312273326c5b6e590f58b226005e552ba4ccf3 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 10 Sep 2025 11:47:16 -0300 Subject: [PATCH 08/50] virtualnodes: add description to libvirt_uri variable Signed-off-by: Felipe Reyes --- testing/units/virtualnodes/variables.tf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/testing/units/virtualnodes/variables.tf b/testing/units/virtualnodes/variables.tf index 5cbcac528..d79cf0c3e 100644 --- a/testing/units/virtualnodes/variables.tf +++ b/testing/units/virtualnodes/variables.tf @@ -1,6 +1,7 @@ variable "libvirt_uri" { - type = string - default = "qemu:///system" + description = "Libvirt connection URI" + type = string + default = "qemu:///system" } variable "nodes_count" { From 4fedc199197afbc1f23d8c09a419b3757ab1820d Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 10 Sep 2025 11:48:30 -0300 Subject: [PATCH 09/50] virtualnodes: install maas from 3.6/stable instead of edge Signed-off-by: Felipe Reyes --- .../virtualnodes/templates/maas_controller.cloudinit.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/units/virtualnodes/templates/maas_controller.cloudinit.cfg b/testing/units/virtualnodes/templates/maas_controller.cloudinit.cfg index 7ac1975f8..51e749cac 100644 --- a/testing/units/virtualnodes/templates/maas_controller.cloudinit.cfg +++ b/testing/units/virtualnodes/templates/maas_controller.cloudinit.cfg @@ -24,8 +24,8 @@ write_files: - content: | #!/bin/bash set -x - sudo snap install maas --channel 3.6/edge - sudo snap install maas-test-db --channel 3.6/edge + sudo snap install maas --channel 3.6/stable + sudo snap install maas-test-db --channel 3.6/stable sudo -E maas init region+rack --database-uri maas-test-db:/// --maas-url http://${address}:5240/MAAS sleep 25 From 2974a8bad89364ef01b84ef6e13d96ea76b2dfc3 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 10 Sep 2025 11:48:50 -0300 Subject: [PATCH 10/50] virtualnodes: comment out maas configuration in cloud-init the maas setup needs to happen in the maas unit, not in a cloud-init driven config script. Signed-off-by: Felipe Reyes --- .../templates/maas_controller.cloudinit.cfg | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/testing/units/virtualnodes/templates/maas_controller.cloudinit.cfg b/testing/units/virtualnodes/templates/maas_controller.cloudinit.cfg index 51e749cac..e175e1a15 100644 --- a/testing/units/virtualnodes/templates/maas_controller.cloudinit.cfg +++ b/testing/units/virtualnodes/templates/maas_controller.cloudinit.cfg @@ -33,46 +33,46 @@ write_files: sudo maas apikey --username admin > /home/ubuntu/api.key profile="admin" maas login $profile http://${address}:5240/MAAS $(cat /home/ubuntu/api.key) - # Configure MAAS DNS forwarding to OpenStack network DNS server - maas $profile maas set-config name=upstream_dns value="${dns_server}" - # TODO: Find how to properly configure DNSSEC - maas $profile maas set-config name=dnssec_validation value=no - # Download noble image - # maas $profile boot-source-selections create 1 os="ubuntu" release=noble arches=amd64 subarches="*" labels="*" - # To make 'openstack console log show' work - maas $profile maas set-config name=kernel_opts value="console=ttyS0 console=tty0" - # Skip setup screen - maas admin maas set-config name=completed_intro value=true + # # Configure MAAS DNS forwarding to OpenStack network DNS server + # maas $profile maas set-config name=upstream_dns value="${dns_server}" + # # TODO: Find how to properly configure DNSSEC + # maas $profile maas set-config name=dnssec_validation value=no + # # Download noble image + # # maas $profile boot-source-selections create 1 os="ubuntu" release=noble arches=amd64 subarches="*" labels="*" + # # To make 'openstack console log show' work + # maas $profile maas set-config name=kernel_opts value="console=ttyS0 console=tty0" + # # Skip setup screen + # maas admin maas set-config name=completed_intro value=true - PRIMARY_RACK=$(maas $profile rack-controllers read | jq -r ".[] | .system_id") + # PRIMARY_RACK=$(maas $profile rack-controllers read | jq -r ".[] | .system_id") - networks=(${networks}) - for network in "$${networks[@]}" ; do - SUBNET=$${network#*:} - SPACE=$${network%%:*} - space=space-$SPACE - FABRIC_ID=$(maas admin subnet read "$SUBNET" | jq -r ".vlan.fabric_id") - VLAN_TAG=$(maas admin subnet read "$SUBNET" | jq -r ".vlan.vid") - maas $profile spaces create name=$space - maas $profile fabric update $FABRIC_ID name=fabric-$SPACE - if [ "$SPACE" != "pxe-boot" ]; then - maas $profile vlan update $FABRIC_ID $VLAN_TAG space=$space primary_rack="$PRIMARY_RACK" - continue - fi - DHCP_START=$(python3 -c "import ipaddress as ip;print(list(ip.ip_network('$SUBNET').hosts())[-10])") - DHCP_END=$(python3 -c "import ipaddress as ip;print(list(ip.ip_network('$SUBNET').hosts())[-1])") - maas $profile ipranges create type=dynamic start_ip="$DHCP_START" end_ip="$DHCP_END" - maas $profile vlan update $FABRIC_ID $VLAN_TAG space=$space dhcp_on=True primary_rack="$PRIMARY_RACK" - done + # networks=(${networks}) + # for network in "$${networks[@]}" ; do + # SUBNET=$${network#*:} + # SPACE=$${network%%:*} + # space=space-$SPACE + # FABRIC_ID=$(maas admin subnet read "$SUBNET" | jq -r ".vlan.fabric_id") + # VLAN_TAG=$(maas admin subnet read "$SUBNET" | jq -r ".vlan.vid") + # maas $profile spaces create name=$space + # maas $profile fabric update $FABRIC_ID name=fabric-$SPACE + # if [ "$SPACE" != "pxe-boot" ]; then + # maas $profile vlan update $FABRIC_ID $VLAN_TAG space=$space primary_rack="$PRIMARY_RACK" + # continue + # fi + # DHCP_START=$(python3 -c "import ipaddress as ip;print(list(ip.ip_network('$SUBNET').hosts())[-10])") + # DHCP_END=$(python3 -c "import ipaddress as ip;print(list(ip.ip_network('$SUBNET').hosts())[-1])") + # maas $profile ipranges create type=dynamic start_ip="$DHCP_START" end_ip="$DHCP_END" + # maas $profile vlan update $FABRIC_ID $VLAN_TAG space=$space dhcp_on=True primary_rack="$PRIMARY_RACK" + # done # Wait for boot os to be downloaded - while [ $(maas $profile boot-resources is-importing | cat) == "true" ]; + while [ "x$(maas $profile boot-resources is-importing | cat)" != "xfalse" ]; do - echo "waiting for boot resources to import" + echo "$(date) - waiting for boot resources to import" sleep 15 done - # Temporary workaround to get MAAS DHCP service started - sudo snap restart maas + # # Temporary workaround to get MAAS DHCP service started + # sudo snap restart maas touch /tmp/.i_am_done path: /usr/local/share/init.sh owner: root:root From 4b935f545b9bc2b25110ba2ae4f50e28361b08e9 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 10 Sep 2025 11:57:14 -0300 Subject: [PATCH 11/50] maas: setup maas - Set kernel global options - Disable dnssec - Configure password-less ssh access to the kvm host - Register virtual nodes - Configure networking Signed-off-by: Felipe Reyes --- testing/units/maas/main.tf | 163 ++++++++++++++++++++++++++++++++ testing/units/maas/variables.tf | 40 ++++++++ 2 files changed, 203 insertions(+) create mode 100644 testing/units/maas/variables.tf diff --git a/testing/units/maas/main.tf b/testing/units/maas/main.tf index e69de29bb..d4c7b822d 100644 --- a/testing/units/maas/main.tf +++ b/testing/units/maas/main.tf @@ -0,0 +1,163 @@ +locals { + generic_net_addresses = ["172.16.1.0/24"] + external_net_addresses = ["172.16.2.0/24"] +} + +resource "maas_configuration" "kernel_opts" { + key = "kernel_opts" + value = "console=ttyS0 console=tty0" +} + +resource "maas_configuration" "dnssec_disable" { + key = "dnssec_validation" + value = "no" +} + +resource "maas_configuration" "completed_intro" { + key = "completed_intro" + value = "true" +} + +resource "maas_configuration" "upstream_dns" { + key = "upstream_dns" + value = var.upstream_dns_server +} + +# NOTE(freyes): this selection is made automatically by MAAS when installed, +# running this block raises the following error: +# +# Error: error creating ubuntu noble: ServerError: 400 Bad Request ({"__all__": +# ["Boot source selection with this Boot source, Os and Release already +# exists."]}) +# +# It's possible to use the `import` block, although there seems to not be a need +# of making this a managed resource at the moment. +# +# data "maas_boot_source" "boot_source" {} +# +# resource "maas_boot_source_selection" "amd64" { +# boot_source = data.maas_boot_source.boot_source.id +# os = "ubuntu" +# release = "noble" +# arches = ["amd64"] +# } + + +# Generate SSH key pair +resource "tls_private_key" "ssh_key" { + algorithm = "RSA" + rsa_bits = 4096 +} + +# Save private key to local file +resource "local_file" "private_key" { + content = tls_private_key.ssh_key.private_key_pem + filename = "${path.module}/../../private/id_rsa" + file_permission = "0600" +} + +# Read existing file content +data "local_file" "existing_keys" { + filename = pathexpand("~/.ssh/authorized_keys") +} + +locals { + existing_content = try(data.local_file.existing_keys.content, "") + new_key = tls_private_key.ssh_key.public_key_openssh + all_keys = "${local.existing_content}${local.new_key}" + + kvm_host_addr = provider::netparse::parse_url(var.libvirt_uri).host +} + +resource "local_file" "updated_keys" { + content = local.all_keys + filename = pathexpand("~/.ssh/authorized_keys") +} + +resource "null_resource" "maas_controller_null" { + + connection { + type = "ssh" + user = "ubuntu" + private_key = file(var.ssh_private_key_path) + host = var.maas_controller_ip_address + } + + provisioner "remote-exec" { + inline = [ + "#!/bin/bash", + "sudo mkdir -p /var/snap/maas/current/root/.ssh", + "echo '${tls_private_key.ssh_key.private_key_openssh}' | sudo tee /var/snap/maas/current/root/.ssh/id_rsa", + "sudo chmod 700 /var/snap/maas/current/root/.ssh", + "sudo chmod 600 /var/snap/maas/current/root/.ssh/id_rsa", + "ssh-keyscan -H ${local.kvm_host_addr} | sudo tee -a /var/snap/maas/current/root/.ssh/known_hosts", + "sudo chmod 600 /var/snap/maas/current/root/.ssh/known_hosts", + ] + } +} + + +data "maas_rack_controller" "rack_controller" { + hostname = var.maas_hostname +} + +data "maas_subnet" "generic_subnet" { + cidr = local.generic_net_addresses[0] +} + +resource "maas_space" "space_external" { + name = "space-external" +} + +resource "maas_space" "space_generic" { + name = "space-generic" +} + +# Fabric - generic +data "maas_fabric" "generic_fabric" { + name = "fabric-0" +} + +import { + to = maas_fabric.generic_fabric + id = "${data.maas_fabric.generic_fabric.id}" +} + +resource "maas_fabric" "generic_fabric" { + name = "generic-fabric" +} + +# VLAN +data "maas_vlan" "generic_vlan" { + fabric = resource.maas_fabric.generic_fabric.id + vlan = 0 +} + +import { + to = maas_vlan.generic_vlan + id = "${data.maas_vlan.generic_vlan.fabric}:${data.maas_vlan.generic_vlan.id}" +} + +resource "maas_vlan" "generic_vlan" { + fabric = maas_fabric.generic_fabric.id + vid = 0 + name = "Default VLAN" + space = maas_space.space_generic +} + +resource "maas_subnet" "generic_subnet" { + cidr = local.generic_net_addresses[0] + fabric = maas_fabric.generic_fabric.id + vlan = maas_vlan.generic_vlan.id +} + +resource "maas_machine" "node" { + count = length(var.nodes) + hostname = var.nodes[count.index].name + power_type = "virsh" + power_parameters = jsonencode({ + power_address = var.libvirt_uri + power_id = var.nodes[count.index].name + }) + pxe_mac_address = var.nodes[count.index].mac_address +} diff --git a/testing/units/maas/variables.tf b/testing/units/maas/variables.tf new file mode 100644 index 000000000..a86117a69 --- /dev/null +++ b/testing/units/maas/variables.tf @@ -0,0 +1,40 @@ +variable "libvirt_uri" { + description = "Libvirt connection URI" + type = string + default = "qemu:///system" +} + +variable "maas_api_key" { + description = "MAAS Admin API Key" + type = string +} + +variable "maas_controller_ip_address" { + description = "MAAS Controller IP Address" + type = string +} + +variable "maas_hostname" { + description = "MAAS controller hostname" + type = string +} + +variable "nodes" { + description = "List of (virtual) nodes" + type = list(object({ + name = string + mac_address = string + })) +} + +variable "ssh_private_key_path" { + description = "Path to the SSH private key to use in deployed nodes" + type = string + default = "~/.ssh/id_ecdsa" +} + +variable "upstream_dns_server" { + description = "upstream dns server to use in MAAS" + type = string + default = "8.8.8.8" +} From 4107772eb44168d63386fc7b51a5c83f6c3e5d65 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 10 Sep 2025 11:59:20 -0300 Subject: [PATCH 12/50] Add stack.hcl to put together the units in a single stack Signed-off-by: Felipe Reyes --- testing/stack.hcl | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 testing/stack.hcl diff --git a/testing/stack.hcl b/testing/stack.hcl new file mode 100644 index 000000000..7b3af0be5 --- /dev/null +++ b/testing/stack.hcl @@ -0,0 +1,20 @@ +locals { + units = { + virtualnodes = { + source = "./virtualnodes" + } + + maas = { + source = "./maas" + dependencies = ["virtualnodes"] + } + } + + # Stack-wide variables + stack_config = { + ssh_private_key_path = "~/.ssh/passwordless" + ssh_public_key_path = "~/.ssh/passwordless.pub" + libvirt_uri = get_env("LIBVIRT_DEFAULT_URI", "qemu:///system") + maas_hostname = "maas-controller" + } +} From 5ec091fae09c0651bd7e1c3723f721124f34053f Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 10 Sep 2025 16:13:49 -0300 Subject: [PATCH 13/50] maas: mock the MAAS API Key in a compatible format Signed-off-by: Felipe Reyes --- testing/units/maas/terragrunt.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/units/maas/terragrunt.hcl b/testing/units/maas/terragrunt.hcl index 1fc75b6ff..02cfcf9d5 100644 --- a/testing/units/maas/terragrunt.hcl +++ b/testing/units/maas/terragrunt.hcl @@ -11,7 +11,7 @@ dependency "virtualnodes" { config_path = "../virtualnodes" mock_outputs = { - maas_api_key = "abc123-apikey" + maas_api_key = "ConsumerSecret:TokenKey:TokenSecret" maas_controller_ip_address = "1.2.3.4" nodes = [] } From c34f894e57419ac2d34ffd8fc7d7baed4b21e6f4 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 10 Sep 2025 16:14:31 -0300 Subject: [PATCH 14/50] maas: import fabric, vlan and subnet for the generic network Signed-off-by: Felipe Reyes --- testing/units/maas/main.tf | 50 ++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/testing/units/maas/main.tf b/testing/units/maas/main.tf index d4c7b822d..5f053b46e 100644 --- a/testing/units/maas/main.tf +++ b/testing/units/maas/main.tf @@ -101,10 +101,6 @@ data "maas_rack_controller" "rack_controller" { hostname = var.maas_hostname } -data "maas_subnet" "generic_subnet" { - cidr = local.generic_net_addresses[0] -} - resource "maas_space" "space_external" { name = "space-external" } @@ -124,10 +120,10 @@ import { } resource "maas_fabric" "generic_fabric" { - name = "generic-fabric" + name = "fabric-0" } -# VLAN +# VLAN - Generic data "maas_vlan" "generic_vlan" { fabric = resource.maas_fabric.generic_fabric.id vlan = 0 @@ -135,29 +131,41 @@ data "maas_vlan" "generic_vlan" { import { to = maas_vlan.generic_vlan - id = "${data.maas_vlan.generic_vlan.fabric}:${data.maas_vlan.generic_vlan.id}" + id = "${data.maas_fabric.generic_fabric.name}:0" } resource "maas_vlan" "generic_vlan" { fabric = maas_fabric.generic_fabric.id vid = 0 - name = "Default VLAN" - space = maas_space.space_generic + name = "untagged" + space = maas_space.space_generic.name } -resource "maas_subnet" "generic_subnet" { +# Subent - generic + +data "maas_subnet" "generic_subnet" { cidr = local.generic_net_addresses[0] - fabric = maas_fabric.generic_fabric.id - vlan = maas_vlan.generic_vlan.id } -resource "maas_machine" "node" { - count = length(var.nodes) - hostname = var.nodes[count.index].name - power_type = "virsh" - power_parameters = jsonencode({ - power_address = var.libvirt_uri - power_id = var.nodes[count.index].name - }) - pxe_mac_address = var.nodes[count.index].mac_address +import { + to = maas_subnet.generic_subnet + id = "${data.maas_subnet.generic_subnet.cidr}" } + +resource "maas_subnet" "generic_subnet" { + name = local.generic_net_addresses[0] + cidr = local.generic_net_addresses[0] + fabric = maas_fabric.generic_fabric.id + vlan = maas_vlan.generic_vlan.vid +} + +# resource "maas_machine" "node" { +# count = length(var.nodes) +# hostname = var.nodes[count.index].name +# power_type = "virsh" +# power_parameters = jsonencode({ +# power_address = var.libvirt_uri +# power_id = var.nodes[count.index].name +# }) +# pxe_mac_address = var.nodes[count.index].mac_address +# } From 7729c134e51ecf28e7ad014e7ca2330eb669cde0 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Thu, 11 Sep 2025 09:22:27 -0300 Subject: [PATCH 15/50] maas: Configure DHCP in generic subnet Signed-off-by: Felipe Reyes --- testing/units/maas/main.tf | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/testing/units/maas/main.tf b/testing/units/maas/main.tf index 5f053b46e..d916f3aea 100644 --- a/testing/units/maas/main.tf +++ b/testing/units/maas/main.tf @@ -1,6 +1,8 @@ locals { generic_net_addresses = ["172.16.1.0/24"] external_net_addresses = ["172.16.2.0/24"] + generic_dhcp_start = cidrhost(local.generic_net_addresses[0], 200) + generic_dhcp_end = cidrhost(local.generic_net_addresses[0], 254) } resource "maas_configuration" "kernel_opts" { @@ -97,7 +99,7 @@ resource "null_resource" "maas_controller_null" { } -data "maas_rack_controller" "rack_controller" { +data "maas_rack_controller" "primary" { hostname = var.maas_hostname } @@ -159,6 +161,20 @@ resource "maas_subnet" "generic_subnet" { vlan = maas_vlan.generic_vlan.vid } +resource "maas_subnet_ip_range" "generic_subnet_dhcp_range" { + subnet = maas_subnet.generic_subnet.id + start_ip = local.generic_dhcp_start + end_ip = local.generic_dhcp_end + type = "dynamic" +} + +resource "maas_vlan_dhcp" "generic_vlan_dhcp" { + fabric = maas_fabric.generic_fabric.id + vlan = maas_vlan.generic_vlan.vid + primary_rack_controller = data.maas_rack_controller.primary.id + ip_ranges = [maas_subnet_ip_range.generic_subnet_dhcp_range.id] +} + # resource "maas_machine" "node" { # count = length(var.nodes) # hostname = var.nodes[count.index].name From 1fa8ad15a326863575410ca406d80533f0a9757d Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Thu, 11 Sep 2025 12:21:05 -0300 Subject: [PATCH 16/50] Add helper scripts - deploy_deps.sh prepare the host to allow it to run VMs and terragrunt stacks. - deploy.sh is wrapper to kick off the terragrunt stack deployment Signed-off-by: Felipe Reyes --- testing/deploy.sh | 4 ++++ testing/install_deps.sh | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100755 testing/deploy.sh create mode 100755 testing/install_deps.sh diff --git a/testing/deploy.sh b/testing/deploy.sh new file mode 100755 index 000000000..9ec53ce5d --- /dev/null +++ b/testing/deploy.sh @@ -0,0 +1,4 @@ +#!/bin/bash -x + + +terragrunt --non-interactive run-all apply diff --git a/testing/install_deps.sh b/testing/install_deps.sh new file mode 100755 index 000000000..db048fb36 --- /dev/null +++ b/testing/install_deps.sh @@ -0,0 +1,34 @@ +#!/bin/bash -x + +if [ "x$(which terragrunt)" != "x0" ]; then + sudo wget -O /usr/local/bin/terragrunt https://github.com/gruntwork-io/terragrunt/releases/download/v0.87.1/terragrunt_linux_amd64 + chmod +x /usr/local/bin/terragrunt +fi + +if [ "x$(which tofu)" != "x0" ]; then + sudo apt-get update + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg + sudo install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://get.opentofu.org/opentofu.gpg | sudo tee /etc/apt/keyrings/opentofu.gpg >/dev/null + curl -fsSL https://packages.opentofu.org/opentofu/tofu/gpgkey | sudo gpg --no-tty --batch --dearmor -o /etc/apt/keyrings/opentofu-repo.gpg >/dev/null + sudo chmod a+r /etc/apt/keyrings/opentofu.gpg /etc/apt/keyrings/opentofu-repo.gpg +echo \ + "deb [signed-by=/etc/apt/keyrings/opentofu.gpg,/etc/apt/keyrings/opentofu-repo.gpg] https://packages.opentofu.org/opentofu/tofu/any/ any main +deb-src [signed-by=/etc/apt/keyrings/opentofu.gpg,/etc/apt/keyrings/opentofu-repo.gpg] https://packages.opentofu.org/opentofu/tofu/any/ any main" | \ + sudo tee /etc/apt/sources.list.d/opentofu.list > /dev/null + + sudo chmod a+r /etc/apt/sources.list.d/opentofu.list + sudo apt-get update + sudo apt-get install -y -qq tofu +fi + +if [ "x$(which virsh)" != "x0" ]; then + sudo apt-get install -y -qq \ + libvirt-daemon \ + libvirt-daemon-driver-qemu \ + libvirt-daemon-system \ + libvirt-clients + sudo sed '/^security_driver/d' /etc/libvirt/qemu.conf + echo 'security_driver = "none"' | sudo tee -a /etc/libvirt/qemu.conf + sudo systemctl restart libvirtd +fi From 180cb5b56b0d30e5ebfa7d99284ca5ab38763c2f Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Thu, 11 Sep 2025 12:26:46 -0300 Subject: [PATCH 17/50] Move snap testing script to a standalone file Signed-off-by: Felipe Reyes --- .github/workflows/build-snap.yml | 87 +------------------------------- testing/test-standalone.sh | 86 +++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 86 deletions(-) create mode 100755 testing/test-standalone.sh diff --git a/.github/workflows/build-snap.yml b/.github/workflows/build-snap.yml index 0ecde6f08..cc3a17a32 100644 --- a/.github/workflows/build-snap.yml +++ b/.github/workflows/build-snap.yml @@ -44,92 +44,7 @@ jobs: name: local-${{ needs.build.outputs.snap }} - name: test run: | - set -x - export COLUMNS=256 - - # Check docker, containerd and remove them if exists - sudo apt remove --purge docker.io containerd runc -y - sudo rm -rf /run/containerd - - # Allow lxd controller to reach to k8s controller on loadbalancer ip - # sudo nft insert rule ip filter FORWARD tcp dport 17070 accept - # sudo nft insert rule ip filter FORWARD tcp sport 17070 accept - # With above rules, got the following error: - # api.charmhub.io on 10.152.183.182:53: server misbehaving - # Accept all packets filtered for forward - sudo nft chain ip filter FORWARD '{policy accept;}' - - sudo snap remove --purge lxd - sudo snap install --channel 3.6 juju - - sudo snap install ${{ needs.build.outputs.snap }} --dangerous - sudo snap connect openstack:juju-bin juju:juju-bin - openstack.sunbeam prepare-node-script --bootstrap | bash -x - sudo snap connect openstack:dot-local-share-juju - sudo snap connect openstack:dot-config-openstack - sudo snap connect openstack:dot-local-share-openstack - - # Even though `--topology single --database single` is not used in the - # single-node tutorial, explicitly speficy it here to force the single - # mysql mode. - # The tutorial assumes ~16 GiB of memory where Sunbeam selects the singe - # mysql single mysql mode automatically. And self-hosted runners may - # have more than 32 GiB of memory where Sunbeam selects the multi mysql - # mode instead. So we have to override the Sunbeam's decision to be - # closer to the tutorial scenario. - sg snap_daemon "openstack.sunbeam cluster bootstrap --manifest .github/assets/testing/edge.yml --accept-defaults --topology single --database single" - sg snap_daemon "openstack.sunbeam cluster list" - # Note: Moving configure before enabling caas just to ensure caas images are not downloaded - # To download caas image, require ports to open on firewall to access fedora images. - sg snap_daemon "openstack.sunbeam configure --accept-defaults --openrc demo-openrc" - sg snap_daemon "openstack.sunbeam launch --name test" - # The cloud-init process inside the VM takes ~2 minutes to bring up the - # SSH service after the VM gets ACTIVE in OpenStack - sleep 300 - source demo-openrc - openstack console log show --lines 200 test - demo_floating_ip="$(openstack floating ip list -c 'Floating IP Address' -f value | head -n1)" - ssh -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -i ~/snap/openstack/current/sunbeam "ubuntu@${demo_floating_ip}" true - - sg snap_daemon "openstack.sunbeam enable orchestration" - sg snap_daemon "openstack.sunbeam enable loadbalancer" - sg snap_daemon "openstack.sunbeam enable dns testing.github." - # Disabled until https://github.com/canonical/mysql-router-k8s-operator/issues/452 - # or corresponding juju bug is fixed - # sg snap_daemon "openstack.sunbeam disable dns" - # sg snap_daemon "openstack.sunbeam disable loadbalancer" - # sg snap_daemon "openstack.sunbeam disable orchestration" - - # Vault has storage requirements > 15G - # Commenting as CI servers might not have enough disk space - # sg snap_daemon "openstack.sunbeam enable vault --dev-mode" - # sg snap_daemon "openstack.sunbeam enable secrets" - # sg snap_daemon "openstack.sunbeam disable secrets" - # sg snap_daemon "openstack.sunbeam disable vault" - - # Disable caas temporarily while MySQL memory gets adjusted - # sg snap_daemon "openstack.sunbeam enable caas" - # sg snap_daemon "openstack.sunbeam enable validation" - # If smoke tests fails, logs should be collected via sunbeam command in "Collect logs" - # sg snap_daemon "openstack.sunbeam validation run smoke" - # sg snap_daemon "openstack.sunbeam validation run --output tempest_validation.log" - # sg snap_daemon "openstack.sunbeam disable caas" - # sg snap_daemon "openstack.sunbeam disable validation" - - sg snap_daemon "openstack.sunbeam enable telemetry" - # Commenting observability as storage requirements ~6G - # sg snap_daemon "openstack.sunbeam enable observability embedded" - # Commented disabling observability due to LP#1998282 - # sg snap_daemon "openstack.sunbeam disable observability embedded" - # sg snap_daemon "openstack.sunbeam disable telemetry" - - # Commenting features as storage is full in CI machines - # sg snap_daemon "openstack.sunbeam enable resource-optimization" - # sg snap_daemon "openstack.sunbeam enable instance-recovery" - # Disable IR as the consul pods are stuck in getting terminated - # sg snap_daemon "openstack.sunbeam disable instance-recovery" - # sg snap_daemon "openstack.sunbeam disable resource-optimization" - + ./testing/test-standalone.sh - name: Collect logs if: always() run: | diff --git a/testing/test-standalone.sh b/testing/test-standalone.sh new file mode 100755 index 000000000..e0267b6fd --- /dev/null +++ b/testing/test-standalone.sh @@ -0,0 +1,86 @@ +#!/bin/bash +set -x +export COLUMNS=256 + +# Check docker, containerd and remove them if exists +sudo apt remove --purge docker.io containerd runc -y +sudo rm -rf /run/containerd + +# Allow lxd controller to reach to k8s controller on loadbalancer ip +# sudo nft insert rule ip filter FORWARD tcp dport 17070 accept +# sudo nft insert rule ip filter FORWARD tcp sport 17070 accept +# With above rules, got the following error: +# api.charmhub.io on 10.152.183.182:53: server misbehaving +# Accept all packets filtered for forward +sudo nft chain ip filter FORWARD '{policy accept;}' + +sudo snap remove --purge lxd +sudo snap install --channel 3.6 juju + +sudo snap install ${{ needs.build.outputs.snap }} --dangerous +sudo snap connect openstack:juju-bin juju:juju-bin +openstack.sunbeam prepare-node-script --bootstrap | bash -x +sudo snap connect openstack:dot-local-share-juju +sudo snap connect openstack:dot-config-openstack +sudo snap connect openstack:dot-local-share-openstack + +# Even though `--topology single --database single` is not used in the +# single-node tutorial, explicitly speficy it here to force the single +# mysql mode. +# The tutorial assumes ~16 GiB of memory where Sunbeam selects the singe +# mysql single mysql mode automatically. And self-hosted runners may +# have more than 32 GiB of memory where Sunbeam selects the multi mysql +# mode instead. So we have to override the Sunbeam's decision to be +# closer to the tutorial scenario. +sg snap_daemon "openstack.sunbeam cluster bootstrap --manifest .github/assets/testing/edge.yml --accept-defaults --topology single --database single" +sg snap_daemon "openstack.sunbeam cluster list" +# Note: Moving configure before enabling caas just to ensure caas images are not downloaded +# To download caas image, require ports to open on firewall to access fedora images. +sg snap_daemon "openstack.sunbeam configure --accept-defaults --openrc demo-openrc" +sg snap_daemon "openstack.sunbeam launch --name test" +# The cloud-init process inside the VM takes ~2 minutes to bring up the +# SSH service after the VM gets ACTIVE in OpenStack +sleep 300 +source demo-openrc +openstack console log show --lines 200 test +demo_floating_ip="$(openstack floating ip list -c 'Floating IP Address' -f value | head -n1)" +ssh -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -i ~/snap/openstack/current/sunbeam "ubuntu@${demo_floating_ip}" true + +sg snap_daemon "openstack.sunbeam enable orchestration" +sg snap_daemon "openstack.sunbeam enable loadbalancer" +sg snap_daemon "openstack.sunbeam enable dns testing.github." +# Disabled until https://github.com/canonical/mysql-router-k8s-operator/issues/452 +# or corresponding juju bug is fixed +# sg snap_daemon "openstack.sunbeam disable dns" +# sg snap_daemon "openstack.sunbeam disable loadbalancer" +# sg snap_daemon "openstack.sunbeam disable orchestration" + +# Vault has storage requirements > 15G +# Commenting as CI servers might not have enough disk space +# sg snap_daemon "openstack.sunbeam enable vault --dev-mode" +# sg snap_daemon "openstack.sunbeam enable secrets" +# sg snap_daemon "openstack.sunbeam disable secrets" +# sg snap_daemon "openstack.sunbeam disable vault" + +# Disable caas temporarily while MySQL memory gets adjusted +# sg snap_daemon "openstack.sunbeam enable caas" +# sg snap_daemon "openstack.sunbeam enable validation" +# If smoke tests fails, logs should be collected via sunbeam command in "Collect logs" +# sg snap_daemon "openstack.sunbeam validation run smoke" +# sg snap_daemon "openstack.sunbeam validation run --output tempest_validation.log" +# sg snap_daemon "openstack.sunbeam disable caas" +# sg snap_daemon "openstack.sunbeam disable validation" + +sg snap_daemon "openstack.sunbeam enable telemetry" +# Commenting observability as storage requirements ~6G +# sg snap_daemon "openstack.sunbeam enable observability embedded" +# Commented disabling observability due to LP#1998282 +# sg snap_daemon "openstack.sunbeam disable observability embedded" +# sg snap_daemon "openstack.sunbeam disable telemetry" + +# Commenting features as storage is full in CI machines +# sg snap_daemon "openstack.sunbeam enable resource-optimization" +# sg snap_daemon "openstack.sunbeam enable instance-recovery" +# Disable IR as the consul pods are stuck in getting terminated +# sg snap_daemon "openstack.sunbeam disable instance-recovery" +# sg snap_daemon "openstack.sunbeam disable resource-optimization" From 483e7479e335b41d31f0a344adc6013cc690290a Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Thu, 11 Sep 2025 12:27:17 -0300 Subject: [PATCH 18/50] Add functional-test-maas to build-snap workflow Signed-off-by: Felipe Reyes --- .github/workflows/build-snap.yml | 68 ++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/.github/workflows/build-snap.yml b/.github/workflows/build-snap.yml index cc3a17a32..9e791be1f 100644 --- a/.github/workflows/build-snap.yml +++ b/.github/workflows/build-snap.yml @@ -82,3 +82,71 @@ jobs: - name: Setup tmate session if: ${{ failure() && runner.debug }} uses: canonical/action-tmate@main + functional-test-maas: + needs: build + name: Functional test on MAAS + runs-on: [self-hosted, testflinger] + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + path: repository + - name: Download snap artifact + id: download + uses: actions/download-artifact@v5 + with: + name: local-${{ needs.build.outputs.snap }} + path: repository + - name: Pack the repository + run: | + tar acf repository.tar.gz repository/ + - name: Submit job + uses: canonical/testflinger/.github/actions/submit@main + with: + poll: true + job: | + job_queue: openstack + provision_data: + distro: noble + global_timeout: 14400 # 4 hours + output_timeout: 5400 # 90 min + test_data: + attachments: + - local: repository.tar.gz + test_cmds: | + set -ex + scp ./attachments/test/repository.tar.gz "ubuntu@${DEVICE_IP}:" + if ssh "ubuntu@${DEVICE_IP}" ' + set -ex + # LP: #2093303 + sudo mv -v /etc/apt/sources.list{,.bak} + sudo apt-get update + # include ~/.local/bin in PATH + source ~/.profile + set -o pipefail + # LP: #2097451 + # LP: #2102175 + tar xf repository.tar.gz + cd repository/testing/ + ./install_deps.sh + ./deploy.sh + cd ../ + ./testing/test.sh + '; then + scp -r "ubuntu@${DEVICE_IP}:repository/artifacts/" artifacts/ || true + find artifacts/ + else + scp -r "ubuntu@${DEVICE_IP}:repository/artifacts/" artifacts/ || true + find artifacts/ + exit 1 + fi + - name: Upload logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: sunbeam_logs + path: logs + retention-days: 30 + - name: Setup tmate session + if: ${{ failure() && runner.debug }} + uses: canonical/action-tmate@main From 34d7a1e5fd052c8dad8dfbe107f67428bd1fac01 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 3 Oct 2025 13:28:33 -0300 Subject: [PATCH 19/50] testing: Validate number of arguments passed to test-standalone.sh Signed-off-by: Felipe Reyes --- testing/test-standalone.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/testing/test-standalone.sh b/testing/test-standalone.sh index e0267b6fd..98fa43423 100755 --- a/testing/test-standalone.sh +++ b/testing/test-standalone.sh @@ -2,6 +2,19 @@ set -x export COLUMNS=256 +if [$# -ne 1]; then + echo "Invalid number of arguments" >&2 + echo "Usage:" + echo " $0 " +fi + +TEST_SNAP_OPENSTACK=${1} + +if [[ ! -f "${TEST_SNAP_OPENSTACK}" ]]; then + echo "${TEST_SNAP_OPENSTACK}: No such file or directory" >&2 + exit 1 +fi + # Check docker, containerd and remove them if exists sudo apt remove --purge docker.io containerd runc -y sudo rm -rf /run/containerd @@ -17,7 +30,7 @@ sudo nft chain ip filter FORWARD '{policy accept;}' sudo snap remove --purge lxd sudo snap install --channel 3.6 juju -sudo snap install ${{ needs.build.outputs.snap }} --dangerous +sudo snap install --dangerous ${TEST_SNAP_OPENSTACK} sudo snap connect openstack:juju-bin juju:juju-bin openstack.sunbeam prepare-node-script --bootstrap | bash -x sudo snap connect openstack:dot-local-share-juju From c355baba45158835fe522af75c6a38fb69c8d14f Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 3 Oct 2025 13:29:12 -0300 Subject: [PATCH 20/50] testing: Add test-multinode-maas.sh Signed-off-by: Felipe Reyes --- .github/workflows/build-snap.yml | 6 +- testing/test-multinode-maas.sh | 98 ++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 3 deletions(-) create mode 100755 testing/test-multinode-maas.sh diff --git a/.github/workflows/build-snap.yml b/.github/workflows/build-snap.yml index 9e791be1f..e3672fa98 100644 --- a/.github/workflows/build-snap.yml +++ b/.github/workflows/build-snap.yml @@ -44,7 +44,7 @@ jobs: name: local-${{ needs.build.outputs.snap }} - name: test run: | - ./testing/test-standalone.sh + ./testing/test-standalone.sh local-${{ needs.build.outputs.snap }} - name: Collect logs if: always() run: | @@ -85,7 +85,7 @@ jobs: functional-test-maas: needs: build name: Functional test on MAAS - runs-on: [self-hosted, testflinger] + runs-on: [self-hosted, self-hosted-linux-amd64-noble-private-endpoint-medium] steps: - name: Checkout uses: actions/checkout@v5 @@ -131,7 +131,7 @@ jobs: ./install_deps.sh ./deploy.sh cd ../ - ./testing/test.sh + ./testing/test-multinode-maas.sh local-${{ needs.build.outputs.snap }} '; then scp -r "ubuntu@${DEVICE_IP}:repository/artifacts/" artifacts/ || true find artifacts/ diff --git a/testing/test-multinode-maas.sh b/testing/test-multinode-maas.sh new file mode 100755 index 000000000..72cef0b13 --- /dev/null +++ b/testing/test-multinode-maas.sh @@ -0,0 +1,98 @@ +#!/bin/bash +set -x +export COLUMNS=256 + +if [$# -ne 1]; then + echo "Invalid number of arguments" >&2 + echo "Usage:" + echo " $0 " +fi + +TEST_SNAP_OPENSTACK=${1} +TEST_JUJU_CHANNEL=${TEST_JUJU_CHANNEL:-3.6} + +if [[ ! -f "${TEST_SNAP_OPENSTACK}" ]]; then + echo "${TEST_SNAP_OPENSTACK}: No such file or directory" >&2 + exit 1 +fi + +if [[ -z "$TEST_MAAS_API_KEY" ]];then + echo "Error: Please define the TEST_MAAS_API_KEY environment variable" >&1 + exit 1 +fi + +if [[ -z "$TEST_MAAS_URL" ]];then + echo "Error: Please define the TEST_MAAS_URL environment variable" >&1 + exit 1 +fi + +if [[ ! -f "$HOME/.ssh/passwordless" ]]; then + ssh-keygen -b 2048 -t rsa -f $HOME/.ssh/passwordless -q -N "" +fi + +function run_snap_daemon { + sg snap_daemon "$@" +} + +sudo snap install --channel ${TEST_JUJU_CHANNEL} juju +sudo snap install --dangerous ${TEST_SNAP_OPENSTACK} +sudo snap connect openstack:juju-bin juju:juju-bin +openstack.sunbeam prepare-node-script --bootstrap | bash -x +sudo snap connect openstack:dot-local-share-juju +sudo snap connect openstack:dot-config-openstack +sudo snap connect openstack:dot-local-share-openstack +sudo snap alias openstack.sunbeam sunbeam + +run_snap_daemon sunbeam deployment add maas \ + --name my_maas \ + --url ${TEST_MAAS_URL} \ + --token ${TEST_MAAS_API_TOKEN} \ + + +run_snap_daemon sunbeam deployment space map myspace data +run_snap_daemon sunbeam deployment space map myspace internal +run_snap_daemon sunbeam deployment space map myspace management +run_snap_daemon sunbeam deployment space map myspace public +run_snap_daemon sunbeam deployment space map myspace storage +run_snap_daemon sunbeam deployment space map myspace storage-cluster + +run_snap_daemon sunbeam deployment validate + +run_snap_daemon sunbeam cluster bootstrap + +run_snap_daemon sunbeam cluster list + +run_snap_daemon sunbeam configure --accept-defaults --openrc demo-openrc +run_snap_daemon sunbeam launch --name test +# The cloud-init process inside the VM takes ~2 minutes to bring up the +# SSH service after the VM gets ACTIVE in OpenStack +sleep 300 +source demo-openrc +openstack console log show --lines 200 test +demo_floating_ip="$(openstack floating ip list -c 'Floating IP Address' -f value | head -n1)" +ssh -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -i ~/snap/openstack/current/sunbeam "ubuntu@${demo_floating_ip}" true + +run_snap_daemon sunbeam enable orchestration +run_snap_daemon sunbeam enable loadbalancer +run_snap_daemon sunbeam enable dns testing.github. + +run_snap_daemon sunbeam enable vault --dev-mode +run_snap_daemon sunbeam enable secrets +run_snap_daemon sunbeam disable secrets +run_snap_daemon sunbeam disable vault + +sg snap_daemon "openstack.sunbeam enable validation" +# sg snap_daemon "openstack.sunbeam validation run smoke" +# sg snap_daemon "openstack.sunbeam validation run --output tempest_validation.log" + +sg snap_daemon "openstack.sunbeam enable telemetry" +sg snap_daemon "openstack.sunbeam enable observability embedded" +sg snap_daemon "openstack.sunbeam disable observability embedded" +sg snap_daemon "openstack.sunbeam disable telemetry" + +# Commenting features as storage is full in CI machines +# sg snap_daemon "openstack.sunbeam enable resource-optimization" +# sg snap_daemon "openstack.sunbeam enable instance-recovery" +# Disable IR as the consul pods are stuck in getting terminated +# sg snap_daemon "openstack.sunbeam disable instance-recovery" +# sg snap_daemon "openstack.sunbeam disable resource-optimization" From 362a864fb7feb7bcd9eec32c6ac73cc6e21de8b7 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 3 Oct 2025 16:18:51 -0300 Subject: [PATCH 21/50] Move testlinger job definition to its own file Signed-off-by: Felipe Reyes --- .github/workflows/build-snap.yml | 47 +++++----------------- .github/workflows/testflinger/job.yaml.tpl | 36 +++++++++++++++++ .gitignore | 2 + 3 files changed, 49 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/testflinger/job.yaml.tpl diff --git a/.github/workflows/build-snap.yml b/.github/workflows/build-snap.yml index e3672fa98..f4449febc 100644 --- a/.github/workflows/build-snap.yml +++ b/.github/workflows/build-snap.yml @@ -86,6 +86,8 @@ jobs: needs: build name: Functional test on MAAS runs-on: [self-hosted, self-hosted-linux-amd64-noble-private-endpoint-medium] + env: + TESTFLINGER_DIR: .github/workflows/testflinger steps: - name: Checkout uses: actions/checkout@v5 @@ -100,46 +102,19 @@ jobs: - name: Pack the repository run: | tar acf repository.tar.gz repository/ + - name: Create Testflinger job + env: + OPENSTACK_SNAP_PATH: local-${{ needs.build.outputs.snap }} + run: | + # Prepare job + envsubst '$OPENSTACK_SNAP_PATH' \ + < $TESTFLINGER_DIR/job.yaml.tpl \ + > $TESTFLINGER_DIR/job.yaml - name: Submit job uses: canonical/testflinger/.github/actions/submit@main with: poll: true - job: | - job_queue: openstack - provision_data: - distro: noble - global_timeout: 14400 # 4 hours - output_timeout: 5400 # 90 min - test_data: - attachments: - - local: repository.tar.gz - test_cmds: | - set -ex - scp ./attachments/test/repository.tar.gz "ubuntu@${DEVICE_IP}:" - if ssh "ubuntu@${DEVICE_IP}" ' - set -ex - # LP: #2093303 - sudo mv -v /etc/apt/sources.list{,.bak} - sudo apt-get update - # include ~/.local/bin in PATH - source ~/.profile - set -o pipefail - # LP: #2097451 - # LP: #2102175 - tar xf repository.tar.gz - cd repository/testing/ - ./install_deps.sh - ./deploy.sh - cd ../ - ./testing/test-multinode-maas.sh local-${{ needs.build.outputs.snap }} - '; then - scp -r "ubuntu@${DEVICE_IP}:repository/artifacts/" artifacts/ || true - find artifacts/ - else - scp -r "ubuntu@${DEVICE_IP}:repository/artifacts/" artifacts/ || true - find artifacts/ - exit 1 - fi + job-path: ${{ env.TESTFLINGER_DIR }}/job.yaml - name: Upload logs if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/testflinger/job.yaml.tpl b/.github/workflows/testflinger/job.yaml.tpl new file mode 100644 index 000000000..b44bc211e --- /dev/null +++ b/.github/workflows/testflinger/job.yaml.tpl @@ -0,0 +1,36 @@ +# -*- mode: yaml -*- +job_queue: openstack +provision_data: + distro: noble +global_timeout: 14400 # 4 hours +output_timeout: 5400 # 90 min +test_data: + attachments: + - local: repository.tar.gz + test_cmds: | + set -ex + scp ./attachments/test/repository.tar.gz "ubuntu@${DEVICE_IP}:" + if ssh "ubuntu@${DEVICE_IP}" ' + set -ex + # LP: #2093303 + sudo mv -v /etc/apt/sources.list{,.bak} + sudo apt-get update + # include ~/.local/bin in PATH + source ~/.profile + set -o pipefail + # LP: #2097451 + # LP: #2102175 + tar xf repository.tar.gz + cd repository/testing/ + ./install_deps.sh + ./deploy.sh + cd ../ + ./testing/test-multinode-maas.sh ${OPENSTACK_SNAP_PATH} + '; then + scp -r "ubuntu@${DEVICE_IP}:repository/artifacts/" artifacts/ || true + find artifacts/ + else + scp -r "ubuntu@${DEVICE_IP}:repository/artifacts/" artifacts/ || true + find artifacts/ + exit 1 + fi diff --git a/.gitignore b/.gitignore index 084cc5996..3e557f4b4 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,5 @@ cython_debug/ .vscode/ .stestr/ + +.github/workflows/testflinger/*.yaml From 254a9e3868516e70d65de8888b1a7311c19b2290 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 3 Oct 2025 17:30:18 -0300 Subject: [PATCH 22/50] install_deps.sh: fix terragrunt permissions Signed-off-by: Felipe Reyes --- testing/install_deps.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/install_deps.sh b/testing/install_deps.sh index db048fb36..0dd2a7de1 100755 --- a/testing/install_deps.sh +++ b/testing/install_deps.sh @@ -2,7 +2,7 @@ if [ "x$(which terragrunt)" != "x0" ]; then sudo wget -O /usr/local/bin/terragrunt https://github.com/gruntwork-io/terragrunt/releases/download/v0.87.1/terragrunt_linux_amd64 - chmod +x /usr/local/bin/terragrunt + sudo chmod +x /usr/local/bin/terragrunt fi if [ "x$(which tofu)" != "x0" ]; then @@ -28,7 +28,7 @@ if [ "x$(which virsh)" != "x0" ]; then libvirt-daemon-driver-qemu \ libvirt-daemon-system \ libvirt-clients - sudo sed '/^security_driver/d' /etc/libvirt/qemu.conf - echo 'security_driver = "none"' | sudo tee -a /etc/libvirt/qemu.conf + sudo sed -i '/^security_driver/d' /etc/libvirt/qemu.conf + echo 'security_driver = "none"' | sudo tee -a /etc/libvirt/qemu.conf >/dev/null sudo systemctl restart libvirtd fi From f16999dc5a83e944f669e515c637906c7f6c68ff Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 3 Oct 2025 17:35:05 -0300 Subject: [PATCH 23/50] Make decompression verbose Signed-off-by: Felipe Reyes --- .github/workflows/testflinger/job.yaml.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testflinger/job.yaml.tpl b/.github/workflows/testflinger/job.yaml.tpl index b44bc211e..5850f5428 100644 --- a/.github/workflows/testflinger/job.yaml.tpl +++ b/.github/workflows/testflinger/job.yaml.tpl @@ -20,7 +20,7 @@ test_data: set -o pipefail # LP: #2097451 # LP: #2102175 - tar xf repository.tar.gz + tar xzvf repository.tar.gz cd repository/testing/ ./install_deps.sh ./deploy.sh From 2a2f98be5107ec048f1473eebbce04dc3a0977ef Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 17 Oct 2025 12:02:43 -0300 Subject: [PATCH 24/50] Add README.md for testing/ Signed-off-by: Felipe Reyes --- testing/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 testing/README.md diff --git a/testing/README.md b/testing/README.md new file mode 100644 index 000000000..9dda21a14 --- /dev/null +++ b/testing/README.md @@ -0,0 +1,12 @@ +# Testflinger Testing + +## TODO + +* [ ] Expose a knob to turn on/off the log level of terragrunt/terraform. + + +## Known Issues + +* When a libvirt instance does PXE boot, there could be situations where it + doesn't boot and it just times out, making the whole deployment timeout or + fail when terraform's apply times out. From 1c504963dec53bed566ffbe9ee6a095edab4f6be Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 17 Oct 2025 12:09:04 -0300 Subject: [PATCH 25/50] testing: change the CWD to where the deploy.sh is. This allows calling the deploy.sh from anywhere and any relative paths will still work. Signed-off-by: Felipe Reyes --- testing/deploy.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/testing/deploy.sh b/testing/deploy.sh index 9ec53ce5d..7bf6a3d20 100755 --- a/testing/deploy.sh +++ b/testing/deploy.sh @@ -1,4 +1,9 @@ -#!/bin/bash -x +#!/bin/bash -exu +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +pushd $SCRIPT_DIR + +# export TERRAGRUNT_LOG_LEVEL=trace +# export TF_LOG=TRACE terragrunt --non-interactive run-all apply From 6982cdacc55d11b184cc2efce2e3d426fcc29047 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 17 Oct 2025 12:10:16 -0300 Subject: [PATCH 26/50] testing: Install terraform Signed-off-by: Felipe Reyes --- testing/install_deps.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/testing/install_deps.sh b/testing/install_deps.sh index 0dd2a7de1..65df9a25e 100755 --- a/testing/install_deps.sh +++ b/testing/install_deps.sh @@ -5,6 +5,7 @@ if [ "x$(which terragrunt)" != "x0" ]; then sudo chmod +x /usr/local/bin/terragrunt fi +# install opentofu if [ "x$(which tofu)" != "x0" ]; then sudo apt-get update sudo apt-get install -y apt-transport-https ca-certificates curl gnupg @@ -22,6 +23,19 @@ deb-src [signed-by=/etc/apt/keyrings/opentofu.gpg,/etc/apt/keyrings/opentofu-rep sudo apt-get install -y -qq tofu fi +# install terraform +if [ "x$(which tofu)" != "x0" ]; then + sudo apt-get update -q + sudo apt-get install -yq gnupg software-properties-common + wget -O- https://apt.releases.hashicorp.com/gpg | \ + gpg --dearmor | \ + sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg > /dev/null + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(grep -oP '(?<=UBUNTU_CODENAME=).*' /etc/os-release || lsb_release -cs) main" | \ + sudo tee /etc/apt/sources.list.d/hashicorp.list + sudo apt-get update -q + sudo apt-get install -yq terraform +fi + if [ "x$(which virsh)" != "x0" ]; then sudo apt-get install -y -qq \ libvirt-daemon \ From 0965ccf9328886427b8523c2354cdbee6978892c Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 17 Oct 2025 12:11:19 -0300 Subject: [PATCH 27/50] Run sosreport to collect logs The script collect-logs.sh calls sosreport on the maas-controller node and the hypervisor. Signed-off-by: Felipe Reyes --- testing/collect-logs.sh | 9 +++++++++ testing/install_deps.sh | 1 + 2 files changed, 10 insertions(+) create mode 100755 testing/collect-logs.sh diff --git a/testing/collect-logs.sh b/testing/collect-logs.sh new file mode 100755 index 000000000..70fa0fb30 --- /dev/null +++ b/testing/collect-logs.sh @@ -0,0 +1,9 @@ +#!/bin/bash -ux + +## Collect relevant files (if possible) +sudo mkdir /tmp/sosreport/ +sudo sosreport -a --batch --label hypervisor --all-logs --tmp-dir=/tmp/sosreport/ +sudo mv /tmp/sosreport/* $ARTIFACTS_DIR +ssh -i ~/.ssh/passwordless ubuntu@172.16.1.2 "sudo mkdir /tmp/sosreport; sudo sosreport -a --batch --label maas-controller --all-logs --tmp-dir=/tmp/sosreport/; sudo chmod +r /tmp/sosreport/" +scp -i ~/.ssh/passwordless ubuntu@172.16.1.2:"/tmp/sosreport/*" $ARTIFACTS_DIR +sudo chmod +r -R $ARTIFACTS_DIR diff --git a/testing/install_deps.sh b/testing/install_deps.sh index 65df9a25e..e1489c112 100755 --- a/testing/install_deps.sh +++ b/testing/install_deps.sh @@ -38,6 +38,7 @@ fi if [ "x$(which virsh)" != "x0" ]; then sudo apt-get install -y -qq \ + sosreport \ libvirt-daemon \ libvirt-daemon-driver-qemu \ libvirt-daemon-system \ From 0b46aa45bbfaec3dc999f838b2b9381ec3e95359 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 17 Oct 2025 12:29:18 -0300 Subject: [PATCH 28/50] Retry `apt-get update` There are scenarios where the apt repo index is presenting invalid checksums, likely during re-sync of the mirrors, so adding a simple retry mechanism to make this step more robust. Signed-off-by: Felipe Reyes --- .github/workflows/testflinger/job.yaml.tpl | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testflinger/job.yaml.tpl b/.github/workflows/testflinger/job.yaml.tpl index 5850f5428..2c4c3385d 100644 --- a/.github/workflows/testflinger/job.yaml.tpl +++ b/.github/workflows/testflinger/job.yaml.tpl @@ -12,9 +12,27 @@ test_data: scp ./attachments/test/repository.tar.gz "ubuntu@${DEVICE_IP}:" if ssh "ubuntu@${DEVICE_IP}" ' set -ex - # LP: #2093303 + ssh-import-id lp:freyes + timeout_loop () { + local TIMEOUT=90 + while [ "$TIMEOUT" -gt 0 ]; do + if "$@" > /dev/null 2>&1; then + echo "OK" + return 0 + fi + TIMEOUT=$((TIMEOUT - 1)) + sleep 1 + done + echo "ERROR: $* FAILED" + ret=1 + return 1 + } + # http://pad.lv/2093303 sudo mv -v /etc/apt/sources.list{,.bak} - sudo apt-get update + # Workaround for: + # E: Failed to fetch http://... Hash Sum mismatch + timeout_loop sudo apt-get update -q + # include ~/.local/bin in PATH source ~/.profile set -o pipefail From 4d991f4c48afa4e110f9897db724b3799a0eb0ee Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 17 Oct 2025 12:30:45 -0300 Subject: [PATCH 29/50] Generate a password-less ssh key This key is then used by terragrunt/terraform to provision the maas-controller and access it, but more importantly to configure the hypervisor access and allow MAAS to manage the libvirt instances via the virsh power driver --- .github/workflows/testflinger/job.yaml.tpl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/testflinger/job.yaml.tpl b/.github/workflows/testflinger/job.yaml.tpl index 2c4c3385d..fe16e5e16 100644 --- a/.github/workflows/testflinger/job.yaml.tpl +++ b/.github/workflows/testflinger/job.yaml.tpl @@ -40,6 +40,16 @@ test_data: # LP: #2102175 tar xzvf repository.tar.gz cd repository/testing/ + + # generate passwordless key if needed + test -f ~/.ssh/passwordless || ssh-keygen -b 2048 -t rsa -f ~/.ssh/passwordless -q -N "" + + # Allow ssh connections to the virtual nodes without having host fingerprint issues. + echo "Host 172.16.1.* 172.16.2.*" >> ~/.ssh/config + echo " UserKnownHostsFile /dev/null" >> ~/.ssh/config + echo " StrictHostKeyChecking no" >> ~/.ssh/config + + # Install depependencies in the hypervisor. ./install_deps.sh ./deploy.sh cd ../ From 26b6f29c02510d51273531da34e811f3ea86e093 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 17 Oct 2025 12:43:45 -0300 Subject: [PATCH 30/50] Run deploy.sh from a new session Rely on `sudo su - ubuntu -c foo` to enforce the execution of the deploy.sh script in a new shell session so the new group memberships become effective Signed-off-by: Felipe Reyes --- .github/workflows/testflinger/job.yaml.tpl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testflinger/job.yaml.tpl b/.github/workflows/testflinger/job.yaml.tpl index fe16e5e16..42fc89aed 100644 --- a/.github/workflows/testflinger/job.yaml.tpl +++ b/.github/workflows/testflinger/job.yaml.tpl @@ -51,7 +51,10 @@ test_data: # Install depependencies in the hypervisor. ./install_deps.sh - ./deploy.sh + + # Prepare the testing bed running terragrunt + # make the libvirt group effective in this shell, so terraform can talk to the libvirt unix socket + sudo su - ubuntu -c $(realpath ./deploy.sh) cd ../ ./testing/test-multinode-maas.sh ${OPENSTACK_SNAP_PATH} '; then From d03a67bef3d7ca397a14a285572ce38d2b2cf194 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 17 Oct 2025 12:45:05 -0300 Subject: [PATCH 31/50] Set TEST_* env variables before kicking off the testing script Signed-off-by: Felipe Reyes --- .github/workflows/testflinger/job.yaml.tpl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/testflinger/job.yaml.tpl b/.github/workflows/testflinger/job.yaml.tpl index 42fc89aed..83f650de0 100644 --- a/.github/workflows/testflinger/job.yaml.tpl +++ b/.github/workflows/testflinger/job.yaml.tpl @@ -56,6 +56,11 @@ test_data: # make the libvirt group effective in this shell, so terraform can talk to the libvirt unix socket sudo su - ubuntu -c $(realpath ./deploy.sh) cd ../ + + # Start the testing using the previously prepare test bed. + export TEST_SNAP_OPENSTACK=${OPENSTACK_SNAP_PATH} + export TEST_MAAS_API_KEY="$(cat /tmp/maas-api.key)" + export TEST_MAAS_URL="http://172.16.1.2:5240/MAAS" ./testing/test-multinode-maas.sh ${OPENSTACK_SNAP_PATH} '; then scp -r "ubuntu@${DEVICE_IP}:repository/artifacts/" artifacts/ || true From 21b9aaa3620a40ed0ecf55d8072efc396b4e387d Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 17 Oct 2025 12:45:33 -0300 Subject: [PATCH 32/50] Call the collect-logs.sh script on failure Signed-off-by: Felipe Reyes --- .github/workflows/testflinger/job.yaml.tpl | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/testflinger/job.yaml.tpl b/.github/workflows/testflinger/job.yaml.tpl index 83f650de0..36a91a5b7 100644 --- a/.github/workflows/testflinger/job.yaml.tpl +++ b/.github/workflows/testflinger/job.yaml.tpl @@ -66,6 +66,7 @@ test_data: scp -r "ubuntu@${DEVICE_IP}:repository/artifacts/" artifacts/ || true find artifacts/ else + ssh ubuntu@${DEVICE_IP} /home/ubuntu/repository/testing/collect-logs.sh scp -r "ubuntu@${DEVICE_IP}:repository/artifacts/" artifacts/ || true find artifacts/ exit 1 From 0b539f4107afcb03da120029fb2ccd724d225f47 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 17 Oct 2025 12:45:54 -0300 Subject: [PATCH 33/50] debug: keep the job in a loop on failure This allows the user to ssh into the node and debug interatively, the job will be held on until it times out or the file /tmp/.continue appears. --- .github/workflows/testflinger/job.yaml.tpl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/testflinger/job.yaml.tpl b/.github/workflows/testflinger/job.yaml.tpl index 36a91a5b7..7f9773cb0 100644 --- a/.github/workflows/testflinger/job.yaml.tpl +++ b/.github/workflows/testflinger/job.yaml.tpl @@ -69,5 +69,8 @@ test_data: ssh ubuntu@${DEVICE_IP} /home/ubuntu/repository/testing/collect-logs.sh scp -r "ubuntu@${DEVICE_IP}:repository/artifacts/" artifacts/ || true find artifacts/ + echo "blocking until file /tmp/.continue shows up in ${DEVICE_IP}" + echo ssh ubuntu@${DEVICE_IP} + ssh ubuntu@${DEVICE_IP} "until test -f /tmp/.continue; do sleep 10;done" exit 1 fi From f29031ab6839ef5d050e33297f2b15592e6a775f Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 17 Oct 2025 12:47:14 -0300 Subject: [PATCH 34/50] install_deps.sh: install genisoimage The genisoimage package is needed to build a iso image that's used to be passed to cloud-init, this image contains all the cloud-init configuration produced by the virtualnodes stack unit Signed-off-by: Felipe Reyes --- testing/install_deps.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testing/install_deps.sh b/testing/install_deps.sh index e1489c112..8f335da4f 100755 --- a/testing/install_deps.sh +++ b/testing/install_deps.sh @@ -42,7 +42,10 @@ if [ "x$(which virsh)" != "x0" ]; then libvirt-daemon \ libvirt-daemon-driver-qemu \ libvirt-daemon-system \ - libvirt-clients + libvirt-clients \ + genisoimage + + # allow a non-root user to use libvirt/virsh easily with no permission issues sudo sed -i '/^security_driver/d' /etc/libvirt/qemu.conf echo 'security_driver = "none"' | sudo tee -a /etc/libvirt/qemu.conf >/dev/null sudo systemctl restart libvirtd From ff389ab8a47dd87bccc40e53fc0fda1f4fb1004f Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 17 Oct 2025 12:48:24 -0300 Subject: [PATCH 35/50] Register virtualnodes in MAAS --- testing/units/maas/main.tf | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/testing/units/maas/main.tf b/testing/units/maas/main.tf index d916f3aea..b087cea86 100644 --- a/testing/units/maas/main.tf +++ b/testing/units/maas/main.tf @@ -175,13 +175,13 @@ resource "maas_vlan_dhcp" "generic_vlan_dhcp" { ip_ranges = [maas_subnet_ip_range.generic_subnet_dhcp_range.id] } -# resource "maas_machine" "node" { -# count = length(var.nodes) -# hostname = var.nodes[count.index].name -# power_type = "virsh" -# power_parameters = jsonencode({ -# power_address = var.libvirt_uri -# power_id = var.nodes[count.index].name -# }) -# pxe_mac_address = var.nodes[count.index].mac_address -# } +resource "maas_machine" "node" { + count = length(var.nodes) + hostname = var.nodes[count.index].name + power_type = "virsh" + power_parameters = jsonencode({ + power_address = var.libvirt_uri + power_id = var.nodes[count.index].name + }) + pxe_mac_address = var.nodes[count.index].mac_address +} From 07f04606e5c3a35ccf530b439234b400359c42df Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 17 Oct 2025 12:48:48 -0300 Subject: [PATCH 36/50] Add local-testflinger.sh This script allows the execution of the testing machinery directly interacting with testflinger. Usage example: ``` ./testing/local-testflinger.sh ``` The script depends on finding a snap file at the top of the git repository, this can be a locally built snap or a snap that was fetched via `snap download ...` Signed-off-by: Felipe Reyes --- testing/local-testflinger.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100755 testing/local-testflinger.sh diff --git a/testing/local-testflinger.sh b/testing/local-testflinger.sh new file mode 100755 index 000000000..17360a25f --- /dev/null +++ b/testing/local-testflinger.sh @@ -0,0 +1,21 @@ +#!/bin/bash -ex + +TMP_DIR=$(mktemp -d) +trap "rm -rf $TMP_DIR; echo 'Cleaned up temporary file.'" EXIT +cp -rf ../snap-openstack/ $TMP_DIR/repository +pushd $TMP_DIR +tar --exclude=repository/.tox --exclude=repository/.github/workflows/testflinger/repository.tar.gz --exclude=repository/.git -acf repository.tar.gz repository/ +ls -lh repository.tar.gz +popd +export TESTFLINGER_DIR=$(pwd)/.github/workflows/testflinger/ +cp $TMP_DIR/repository.tar.gz $TESTFLINGER_DIR +export OPENSTACK_SNAP_PATH=$(ls openstack_*.snap) +JOB_FILE=$TESTFLINGER_DIR/job.yaml +envsubst '$OPENSTACK_SNAP_PATH' \ + < $TESTFLINGER_DIR/job.yaml.tpl \ + > $JOB_FILE + +test -f $JOB_FILE +cd $TESTFLINGER_DIR +testflinger-cli -d submit --poll $JOB_FILE +rm -rf $TMP_DIR From 87d552f21e1fbfdeb07d5ec4204d802f07e79c46 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 17 Oct 2025 12:52:51 -0300 Subject: [PATCH 37/50] virtualnodes: Use a dedicated libvirt pool Create a libvirt pool called "sunbeam" to store all the volumes used by the libvirt instances, this allows a cleaner setup on non-dedicated hypervisors Signed-off-by: Felipe Reyes --- testing/units/virtualnodes/main.tf | 13 +++++++++++++ testing/units/virtualnodes/terragrunt.hcl | 19 ++++++++++++++++++- testing/units/virtualnodes/variables.tf | 5 +++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/testing/units/virtualnodes/main.tf b/testing/units/virtualnodes/main.tf index a814e5fe2..b999aeee0 100644 --- a/testing/units/virtualnodes/main.tf +++ b/testing/units/virtualnodes/main.tf @@ -51,17 +51,27 @@ resource "libvirt_network" "external_net" { } } +resource "libvirt_pool" "sunbeam" { + name = "sunbeam" + type = "dir" + target { + path = var.storage_pool_path + } +} + #### Volumes resource "libvirt_volume" "node_vol" { name = "node_${count.index}.qcow2" count = var.nodes_count + pool = libvirt_pool.sunbeam.name size = var.node_rootfs_size } resource "libvirt_volume" "node_vol_secondary" { name = "node_${count.index}_secondary.qcow2" count = var.nodes_count + pool = libvirt_pool.sunbeam.name size = var.node_secondary_disk_size } @@ -69,11 +79,13 @@ resource "libvirt_volume" "node_vol_secondary" { resource "libvirt_volume" "ubuntu_noble" { name = "ubuntu-noble.qcow2" source = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" + pool = libvirt_pool.sunbeam.name } resource "libvirt_volume" "maas_controller_vol" { name = "maas-controller-vol" base_volume_id = libvirt_volume.ubuntu_noble.id + pool = libvirt_pool.sunbeam.name size = var.maas_controller_rootfs_size } @@ -81,6 +93,7 @@ resource "libvirt_volume" "maas_controller_vol" { resource "libvirt_cloudinit_disk" "maas_controller_cloudinit" { name = "maas_controller_cloudinit.iso" + pool = libvirt_pool.sunbeam.name user_data = templatefile( "${path.module}/templates/maas_controller.cloudinit.cfg", { diff --git a/testing/units/virtualnodes/terragrunt.hcl b/testing/units/virtualnodes/terragrunt.hcl index 4b93efd2b..230ab3ebb 100644 --- a/testing/units/virtualnodes/terragrunt.hcl +++ b/testing/units/virtualnodes/terragrunt.hcl @@ -8,6 +8,23 @@ include "stack" { terraform { source = "./" + before_hook "create_directories" { + commands = ["apply", "plan"] + execute = ["bash", "-c", <<-EOT + chmod 755 $HOME + mkdir -p $HOME/sunbeam_storage + chmod 775 $HOME/sunbeam_storage + chgrp libvirt $HOME/sunbeam_storage + echo "Directory $HOME/sunbeam_storage created" + ls -ld $HOME/sunbeam_storage + EOT + ] + } } -inputs = include.stack.locals.stack_config +inputs = merge( + include.stack.locals.stack_config, + { + storage_pool_path = format("%s/sunbeam_storage", get_env("HOME")) + } +) diff --git a/testing/units/virtualnodes/variables.tf b/testing/units/virtualnodes/variables.tf index d79cf0c3e..374c1bd01 100644 --- a/testing/units/virtualnodes/variables.tf +++ b/testing/units/virtualnodes/variables.tf @@ -71,6 +71,11 @@ variable "ssh_private_key_path" { default = "~/.ssh/id_ecdsa" } +variable "storage_pool_path" { + description = "Path to the storage pool used by the virtual nodes" + type = string +} + variable "upstream_dns_server" { description = "upstream dns server to use in MAAS" type = string From 4408281e06b75e95c5f4291c02d7db34e0413ab2 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 17 Oct 2025 12:54:33 -0300 Subject: [PATCH 38/50] virtualnodes: block until the maas api key is available Add a busy loop until the MAAS admin API key is written to disk, this prevents race conditions found when testing on physical hardware --- testing/units/virtualnodes/main.tf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testing/units/virtualnodes/main.tf b/testing/units/virtualnodes/main.tf index b999aeee0..00e28f55d 100644 --- a/testing/units/virtualnodes/main.tf +++ b/testing/units/virtualnodes/main.tf @@ -213,7 +213,10 @@ data "external" "remote_command" { libvirt_domain.maas_controller ] program = ["bash", "-c", <<-EOF - ssh -i ${var.ssh_private_key_path} ubuntu@${local.maas_controller_ip_addr} 'cat api.key' | jq -R '{apikey: .}' + # Block until the api.key file shows up + API_KEY_FILE=/tmp/maas-api.key + ssh -i ${var.ssh_private_key_path} ubuntu@${local.maas_controller_ip_addr} 'touch /home/ubuntu/api.key; until [ -s /home/ubuntu/api.key ]; do sleep 5;done; cat /home/ubuntu/api.key' > $API_KEY_FILE + cat $API_KEY_FILE 2>&1 | jq -R '{apikey: .}' 2>&1 EOF ] } From edd4f64360ad3cd1897f17a04acb96d3de70fa88 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Fri, 17 Oct 2025 12:55:34 -0300 Subject: [PATCH 39/50] testing: Fix space mapping --- testing/test-multinode-maas.sh | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/testing/test-multinode-maas.sh b/testing/test-multinode-maas.sh index 72cef0b13..a00ba055c 100755 --- a/testing/test-multinode-maas.sh +++ b/testing/test-multinode-maas.sh @@ -1,8 +1,8 @@ -#!/bin/bash -set -x +#!/bin/bash -eux + export COLUMNS=256 -if [$# -ne 1]; then +if [ $# -ne 1 ]; then echo "Invalid number of arguments" >&2 echo "Usage:" echo " $0 " @@ -31,30 +31,27 @@ if [[ ! -f "$HOME/.ssh/passwordless" ]]; then fi function run_snap_daemon { - sg snap_daemon "$@" + sg snap_daemon -c "$*" } sudo snap install --channel ${TEST_JUJU_CHANNEL} juju sudo snap install --dangerous ${TEST_SNAP_OPENSTACK} sudo snap connect openstack:juju-bin juju:juju-bin openstack.sunbeam prepare-node-script --bootstrap | bash -x + +# connect plugs manually since the snap is installed from a locally built one. sudo snap connect openstack:dot-local-share-juju sudo snap connect openstack:dot-config-openstack sudo snap connect openstack:dot-local-share-openstack sudo snap alias openstack.sunbeam sunbeam -run_snap_daemon sunbeam deployment add maas \ - --name my_maas \ - --url ${TEST_MAAS_URL} \ - --token ${TEST_MAAS_API_TOKEN} \ - - -run_snap_daemon sunbeam deployment space map myspace data -run_snap_daemon sunbeam deployment space map myspace internal -run_snap_daemon sunbeam deployment space map myspace management -run_snap_daemon sunbeam deployment space map myspace public -run_snap_daemon sunbeam deployment space map myspace storage -run_snap_daemon sunbeam deployment space map myspace storage-cluster +run_snap_daemon sunbeam deployment add maas mymaas ${TEST_MAAS_API_KEY} ${TEST_MAAS_URL} +run_snap_daemon sunbeam deployment space map space-generic:data +run_snap_daemon sunbeam deployment space map space-generic:internal +run_snap_daemon sunbeam deployment space map space-generic:management +run_snap_daemon sunbeam deployment space map space-generic:storage +run_snap_daemon sunbeam deployment space map space-generic:storage-cluster +run_snap_daemon sunbeam deployment space map space-external:public run_snap_daemon sunbeam deployment validate From e0fa1a86911bd3c392f802f57d27f8379674ad6a Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Mon, 20 Oct 2025 11:07:54 -0300 Subject: [PATCH 40/50] Set parallelism=1 for the maas unit This has the intention of putting less pressure on the maas controller and reduce the chances that the libvirt instance fails to pxe boot. --- testing/units/maas/terragrunt.hcl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/testing/units/maas/terragrunt.hcl b/testing/units/maas/terragrunt.hcl index 02cfcf9d5..3f8887ab4 100644 --- a/testing/units/maas/terragrunt.hcl +++ b/testing/units/maas/terragrunt.hcl @@ -5,6 +5,10 @@ include "stack" { terraform { source = "." + extra_arguments "parallelism" { + commands = ["apply", "plan", "destroy"] + arguments = ["-parallelism=1"] + } } dependency "virtualnodes" { From 747ab90f4b8e12d74471a1955360af1db135d76a Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Mon, 20 Oct 2025 11:14:19 -0300 Subject: [PATCH 41/50] Update README.md Include a section to document how to run the testing from a local development environment. Signed-off-by: Felipe Reyes --- testing/README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/testing/README.md b/testing/README.md index 9dda21a14..283379915 100644 --- a/testing/README.md +++ b/testing/README.md @@ -1,5 +1,25 @@ # Testflinger Testing +## Local Development/Testing + +To run the testing, it's possible to use the `./testing/local-testflinger.sh` script. + +Usage example: + +1. Install the testflinger-cli snap: `sudo snap install testflinger-cli`. +2. Make sure there is a copy of the openstack snap at the toplevel of the git + repo. Use `snap download` or `snapcraft pack`. + +``` sh +snap download --channel 2024.1/edge openstack +``` + +``` sh +snapcraft pack --use-lxd +``` + +3. Run `./testing/local-testflinger.sh`. + ## TODO * [ ] Expose a knob to turn on/off the log level of terragrunt/terraform. From ba3e6f226724dc6d2d20ad0bfb91c713560a8135 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 17 Dec 2025 16:42:37 -0300 Subject: [PATCH 42/50] Update virtualnodes default hardware specs Signed-off-by: Felipe Reyes --- testing/units/virtualnodes/variables.tf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/testing/units/virtualnodes/variables.tf b/testing/units/virtualnodes/variables.tf index 374c1bd01..4d8638474 100644 --- a/testing/units/virtualnodes/variables.tf +++ b/testing/units/virtualnodes/variables.tf @@ -11,22 +11,22 @@ variable "nodes_count" { variable "node_mem" { type = string - default = "2048" + default = "16777216" # 16GiB } variable "node_vcpu" { type = number - default = 2 + default = 4 } variable "maas_controller_mem" { type = string - default = "4096" # 4GiB + default = "8388608" # 8GiB } variable "maas_controller_vcpu" { type = number - default = 2 + default = 4 } variable "maas_hostname" { @@ -85,7 +85,7 @@ variable "upstream_dns_server" { variable "maas_controller_mac_address" { description = "MAC address to assign to the maas controller nic in the management network" type = string - default = "AA:BB:CC:11:11:01" + default = "52:54:00:11:11:01" } variable "maas_controller_rootfs_size" { From 03c04c31709b7453e8d5c6a4875f6977d8d40b73 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 17 Dec 2025 16:43:47 -0300 Subject: [PATCH 43/50] Drop -parallelism=1 Signed-off-by: Felipe Reyes --- testing/units/maas/terragrunt.hcl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/testing/units/maas/terragrunt.hcl b/testing/units/maas/terragrunt.hcl index 3f8887ab4..02cfcf9d5 100644 --- a/testing/units/maas/terragrunt.hcl +++ b/testing/units/maas/terragrunt.hcl @@ -5,10 +5,6 @@ include "stack" { terraform { source = "." - extra_arguments "parallelism" { - commands = ["apply", "plan", "destroy"] - arguments = ["-parallelism=1"] - } } dependency "virtualnodes" { From e12eef5d65911dc5803c7ced57f9c1fe162ec7b4 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 17 Dec 2025 16:44:51 -0300 Subject: [PATCH 44/50] testing: Add reserved range to maas Signed-off-by: Felipe Reyes --- testing/units/maas/main.tf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/testing/units/maas/main.tf b/testing/units/maas/main.tf index b087cea86..326454427 100644 --- a/testing/units/maas/main.tf +++ b/testing/units/maas/main.tf @@ -3,6 +3,8 @@ locals { external_net_addresses = ["172.16.2.0/24"] generic_dhcp_start = cidrhost(local.generic_net_addresses[0], 200) generic_dhcp_end = cidrhost(local.generic_net_addresses[0], 254) + generic_reserved_start = cidrhost(local.generic_net_addresses[0], 1) + generic_reserved_end = cidrhost(local.generic_net_addresses[0], 5) } resource "maas_configuration" "kernel_opts" { @@ -168,6 +170,13 @@ resource "maas_subnet_ip_range" "generic_subnet_dhcp_range" { type = "dynamic" } +resource "maas_subnet_ip_range" "generic_subnet_reserved_range" { + subnet = maas_subnet.generic_subnet.id + start_ip = local.generic_reserved_start + end_ip = local.generic_reserved_end + type = "reserved" +} + resource "maas_vlan_dhcp" "generic_vlan_dhcp" { fabric = maas_fabric.generic_fabric.id vlan = maas_vlan.generic_vlan.vid From 6e544620380a37faade27f6ad0f93b8d9c658678 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 17 Dec 2025 16:45:10 -0300 Subject: [PATCH 45/50] virtualnodes: rewrote to make it compatible with latest provider The terraform libvirt provider changed their schema definition in 0.9.0 Signed-off-by: Felipe Reyes --- testing/units/virtualnodes/main.tf | 306 ++++++++++++++++++-------- testing/units/virtualnodes/outputs.tf | 2 +- 2 files changed, 213 insertions(+), 95 deletions(-) diff --git a/testing/units/virtualnodes/main.tf b/testing/units/virtualnodes/main.tf index 00e28f55d..797ea409a 100644 --- a/testing/units/virtualnodes/main.tf +++ b/testing/units/virtualnodes/main.tf @@ -3,7 +3,7 @@ terraform { required_providers { libvirt = { source = "dmacvicar/libvirt" - version = "0.8.3" + version = "0.9.1" } external = { source = "hashicorp/external" @@ -19,42 +19,76 @@ provider "libvirt" { #### Locals locals { maas_controller_ip_addr = "172.16.1.2" - generic_net_addresses = ["172.16.1.0/24"] - external_net_addresses = ["172.16.2.0/24"] + generic_net_addresses = { + start = "172.16.1.2", + end = "172.16.1.254", + cidr = "172.16.1.0/24", + gateway = "172.16.1.1" + } + external_net_addresses = { + start = "172.16.2.2", + end = "172.16.2.254", + cidr = "172.16.2.0/24", + gateway = "172.16.2.1" + } } #### Networks resource "libvirt_network" "generic_net" { - name = "generic_net" - mode = "nat" + name = "generic_net" autostart = true - domain = var.generic_net_domain - addresses = local.generic_net_addresses - - dns { - enabled = false + domain = { + name = var.generic_net_domain } + forward = { + nat = { + ports = [ + { + start = 1024 + end = 65535 + } + ] + } + } + ips = [ + { + address = local.generic_net_addresses.gateway + prefix = 24 + } + ] } resource "libvirt_network" "external_net" { - name = "external_net" - mode = "nat" + name = "external_net" autostart = true - domain = var.external_net_domain - addresses = local.external_net_addresses - - dns { - enabled = false + domain = { + name = var.external_net_domain } + forward = { + nat = { + ports = [ + { + start = 1024 + end = 65535 + } + ] + } + } + ips = [ + { + address = local.external_net_addresses.gateway + prefix = 24 + } + ] } resource "libvirt_pool" "sunbeam" { name = "sunbeam" type = "dir" - target { + target = { path = var.storage_pool_path } } @@ -62,38 +96,52 @@ resource "libvirt_pool" "sunbeam" { #### Volumes resource "libvirt_volume" "node_vol" { - name = "node_${count.index}.qcow2" - count = var.nodes_count - pool = libvirt_pool.sunbeam.name - size = var.node_rootfs_size + name = "node_${count.index}.qcow2" + count = var.nodes_count + pool = libvirt_pool.sunbeam.name + capacity = var.node_rootfs_size + target = { format = { type = "qcow2" } } } resource "libvirt_volume" "node_vol_secondary" { - name = "node_${count.index}_secondary.qcow2" - count = var.nodes_count - pool = libvirt_pool.sunbeam.name - size = var.node_secondary_disk_size + name = "node_${count.index}_secondary.qcow2" + count = var.nodes_count + pool = libvirt_pool.sunbeam.name + capacity = var.node_secondary_disk_size + target = { format = { type = "qcow2" } } } resource "libvirt_volume" "ubuntu_noble" { name = "ubuntu-noble.qcow2" - source = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" - pool = libvirt_pool.sunbeam.name + pool = libvirt_pool.sunbeam.name + target = { format = { type = "qcow2" } } + create = { + content = { + url = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" + } + } } resource "libvirt_volume" "maas_controller_vol" { - name = "maas-controller-vol" - base_volume_id = libvirt_volume.ubuntu_noble.id - pool = libvirt_pool.sunbeam.name - size = var.maas_controller_rootfs_size + name = "maas-controller-vol" + pool = libvirt_pool.sunbeam.name + capacity = var.maas_controller_rootfs_size + target = { format = { type = "qcow2" } } + backing_store = { + path = libvirt_volume.ubuntu_noble.path + format = { type = "qcow2" } + } } #### Virtual machines (domains) resource "libvirt_cloudinit_disk" "maas_controller_cloudinit" { name = "maas_controller_cloudinit.iso" - pool = libvirt_pool.sunbeam.name + meta_data = yamlencode({ + instance-id = "maas-controller" + local-hostname = "maas-controller" + }) user_data = templatefile( "${path.module}/templates/maas_controller.cloudinit.cfg", { @@ -112,39 +160,76 @@ resource "libvirt_cloudinit_disk" "maas_controller_cloudinit" { }) } +resource "libvirt_volume" "maas_controller_cloudinit_vol" { + name = "maas_controller_cloudinit_vol" + pool = libvirt_pool.sunbeam.name + create = { + content = { + url = libvirt_cloudinit_disk.maas_controller_cloudinit.path + } + } +} + resource "libvirt_domain" "maas_controller" { - name = "maas-controller" + name = "maas-controller" memory = var.maas_controller_mem vcpu = var.maas_controller_vcpu - disk { - volume_id = libvirt_volume.maas_controller_vol.id - scsi = "true" - } - cloudinit = libvirt_cloudinit_disk.maas_controller_cloudinit.id - boot_device { - dev = [ "hd"] - } - network_interface { - network_id = libvirt_network.generic_net.id - hostname = var.maas_hostname - addresses = [local.maas_controller_ip_addr] - mac = var.maas_controller_mac_address - } - console { - type = "pty" - target_type = "serial" - target_port = "0" - } + running = true + type = "kvm" + cpu = { mode = "host-passthrough" } - console { - type = "pty" - target_type = "virtio" - target_port = "1" + os = { + type = "hvm" + type_arch = "x86_64" + type_machine = "q35" + boot_devices = [{ dev = "hd" }] } - graphics { - type = "spice" - listen_type = "address" - autoport = true + devices = { + disks = [ + { + source = { + volume = { + pool = libvirt_pool.sunbeam.name + volume = libvirt_volume.maas_controller_vol.name + } + } + target = { dev = "sda", bus = "virtio" } + driver = { name = "qemu", type = "qcow2" } + }, + { + source = { + volume = { + pool = libvirt_pool.sunbeam.name + volume = libvirt_volume.maas_controller_cloudinit_vol.name + } + } + target = { dev = "hdd", bus = "virtio" } + } + ] + interfaces = [ + { + model = { type = "virtio"} + source = { + network = { + network = libvirt_network.generic_net.name + } + } + mac = { address = var.maas_controller_mac_address } + } + ] + consoles = [ + { target = { type = "serial" } }, + { target = { type = "virtio", port = "1" }} + ] + graphics = [ + { + type = "vnc" + vnc = { + autoport = true + listen = "127.0.0.1" + } + }, + ] } connection { @@ -170,41 +255,74 @@ resource "libvirt_domain" "node" { memory = var.node_mem vcpu = var.node_vcpu running = false - disk { - volume_id = libvirt_volume.node_vol[count.index].id - scsi = "true" - } - disk { - volume_id = libvirt_volume.node_vol_secondary[count.index].id - scsi = "true" - } - boot_device { - dev = [ "network"] + type = "kvm" + cpu = { + mode = "host-passthrough" } - network_interface { - network_id = libvirt_network.generic_net.id - hostname = "node-${count.index}" - mac = format("AA:BB:CC:11:22:%02d", count.index + 10) + os = { + type = "hvm" + type_arch = "x86_64" + type_machine = "q35" + boot_devices = [{ dev = "network" }, { dev = "hd" }] } - network_interface { - network_id = libvirt_network.external_net.id - mac = format("AA:BB:CC:33:44:%02d", count.index + 10) - } - console { - type = "pty" - target_type = "serial" - target_port = "0" - } - - console { - type = "pty" - target_type = "virtio" - target_port = "1" - } - graphics { - type = "spice" - listen_type = "address" - autoport = true + devices = { + disks = [ + { + serial = format("DISK-ROOT-%06d", count.index) + source = { + volume = { + pool = libvirt_pool.sunbeam.name + volume = libvirt_volume.node_vol[count.index].name + } + } + target = { dev = "sda", bus = "virtio" } + driver = { name = "qemu", type = "qcow2" } + }, + { + serial = format("DISK-CEPH-%06d", count.index) + source = { + volume = { + pool = libvirt_pool.sunbeam.name + volume = libvirt_volume.node_vol_secondary[count.index].name + } + } + target = { dev = "sdb", bus = "virtio" } + driver = { name = "qemu", type = "qcow2" } + } + ] + interfaces = [ + { + model = { type = "virtio"} + source = { + network = { + network = libvirt_network.generic_net.name + } + } + mac = { address = format("52:54:00:11:22:%02d", count.index + 10) } + }, + { + model = { type = "virtio"} + source = { + network = { + network = libvirt_network.external_net.name + } + } + mac = { address = format("52:54:00:33:44:%02d", count.index + 10) } + } + ] + consoles = [ + { target = { type = "serial" } }, + { target = { type = "virtio", port = "1" }} + ] + graphics = [ + { + type = "vnc" + vnc = { + autoport = true + listen = "127.0.0.1" + } + }, + ] } } diff --git a/testing/units/virtualnodes/outputs.tf b/testing/units/virtualnodes/outputs.tf index 1a96b370c..5ffdda974 100644 --- a/testing/units/virtualnodes/outputs.tf +++ b/testing/units/virtualnodes/outputs.tf @@ -9,7 +9,7 @@ output "nodes" { value = [ for node in libvirt_domain.node : { name = node.name - mac_address = node.network_interface[0].mac + mac_address = node.devices.interfaces[0].mac.address } ] depends_on = [libvirt_domain.node] From 353c716c2d8ec214e2ca3ef53561055cd2ec7ce9 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 17 Dec 2025 16:46:35 -0300 Subject: [PATCH 46/50] testing: make artifacts dir relative to the collect-logs.sh script Signed-off-by: Felipe Reyes --- testing/collect-logs.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/testing/collect-logs.sh b/testing/collect-logs.sh index 70fa0fb30..1e4095fcd 100755 --- a/testing/collect-logs.sh +++ b/testing/collect-logs.sh @@ -1,5 +1,9 @@ #!/bin/bash -ux +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +ARTIFACTS_DIR=$(realpath $SCRIPT_DIR/../artifacts) +mkdir -p $ARTIFACTS_DIR + ## Collect relevant files (if possible) sudo mkdir /tmp/sosreport/ sudo sosreport -a --batch --label hypervisor --all-logs --tmp-dir=/tmp/sosreport/ From aa4b928e7c7f0cd4fb83c62d6e74276e3c64e15e Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 17 Dec 2025 16:47:30 -0300 Subject: [PATCH 47/50] testing: Block until port 22 is open on the testing instance Signed-off-by: Felipe Reyes --- testing/test-multinode-maas.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/testing/test-multinode-maas.sh b/testing/test-multinode-maas.sh index a00ba055c..ceb42803d 100755 --- a/testing/test-multinode-maas.sh +++ b/testing/test-multinode-maas.sh @@ -30,6 +30,21 @@ if [[ ! -f "$HOME/.ssh/passwordless" ]]; then ssh-keygen -b 2048 -t rsa -f $HOME/.ssh/passwordless -q -N "" fi +function timeout_loop () { + local TIMEOUT=$1; shift + while [ "$TIMEOUT" -gt 0 ]; do + if "$@" > /dev/null 2>&1; then + echo "OK" + return 0 + fi + TIMEOUT=$((TIMEOUT - 1)) + sleep 1 + done + echo "ERROR: $* FAILED" + ret=1 + return 1 +} + function run_snap_daemon { sg snap_daemon -c "$*" } @@ -67,6 +82,10 @@ sleep 300 source demo-openrc openstack console log show --lines 200 test demo_floating_ip="$(openstack floating ip list -c 'Floating IP Address' -f value | head -n1)" + +TIMEOUT=120 +echo "Block until ${demo_floating_ip} on port 22 is reachable (timeout: ${TIMEOUT})" +timeout_loop ${TIMEOUT} nc -v -z ${demo_floating_ip} ssh -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -i ~/snap/openstack/current/sunbeam "ubuntu@${demo_floating_ip}" true run_snap_daemon sunbeam enable orchestration From a83ca564c74e2b497e193cdc623e34812b062791 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Wed, 17 Dec 2025 16:48:06 -0300 Subject: [PATCH 48/50] testing: bump up output timeout to 180m Signed-off-by: Felipe Reyes --- .github/workflows/testflinger/job.yaml.tpl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/testflinger/job.yaml.tpl b/.github/workflows/testflinger/job.yaml.tpl index 7f9773cb0..cdda85af7 100644 --- a/.github/workflows/testflinger/job.yaml.tpl +++ b/.github/workflows/testflinger/job.yaml.tpl @@ -3,7 +3,7 @@ job_queue: openstack provision_data: distro: noble global_timeout: 14400 # 4 hours -output_timeout: 5400 # 90 min +output_timeout: 10800 # 180 min test_data: attachments: - local: repository.tar.gz @@ -66,11 +66,12 @@ test_data: scp -r "ubuntu@${DEVICE_IP}:repository/artifacts/" artifacts/ || true find artifacts/ else - ssh ubuntu@${DEVICE_IP} /home/ubuntu/repository/testing/collect-logs.sh - scp -r "ubuntu@${DEVICE_IP}:repository/artifacts/" artifacts/ || true - find artifacts/ echo "blocking until file /tmp/.continue shows up in ${DEVICE_IP}" echo ssh ubuntu@${DEVICE_IP} ssh ubuntu@${DEVICE_IP} "until test -f /tmp/.continue; do sleep 10;done" + + ssh ubuntu@${DEVICE_IP} /home/ubuntu/repository/testing/collect-logs.sh + scp -r "ubuntu@${DEVICE_IP}:repository/artifacts/" artifacts/ || true + find artifacts/ exit 1 fi From bda71a21dff6b9933f8ddb3b2fe7204c964ed8cc Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Thu, 18 Dec 2025 16:32:24 -0300 Subject: [PATCH 49/50] testing: fix formatting in netplan template Force mac address to be rendered as a literal string Signed-off-by: Felipe Reyes --- .../virtualnodes/templates/maas_controller.netplan.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/units/virtualnodes/templates/maas_controller.netplan.yaml b/testing/units/virtualnodes/templates/maas_controller.netplan.yaml index eb6a43b07..281999461 100644 --- a/testing/units/virtualnodes/templates/maas_controller.netplan.yaml +++ b/testing/units/virtualnodes/templates/maas_controller.netplan.yaml @@ -4,12 +4,12 @@ network: ethernets: virtio-nic: match: - macaddress: ${mac_address} + macaddress: "${mac_address}" addresses: - - ${ip_address}/24 + - "${ip_address}/24" routes: - to: 0.0.0.0/0 via: 172.16.1.1 nameservers: addresses: - - ${dns_server} + - "${dns_server}" From fdb4655551841d31fbf82a0dffe11787308797d8 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Tue, 3 Mar 2026 18:00:45 -0300 Subject: [PATCH 50/50] testing: Tag MAAS instances Signed-off-by: Felipe Reyes --- testing/README.md | 1 + testing/units/maas/main.tf | 121 ++++++++++++++++++++- testing/units/maas/terragrunt.hcl | 10 ++ testing/units/maas/variables.tf | 5 + testing/units/virtualnodes/maas_apikey.txt | 0 testing/units/virtualnodes/main.tf | 31 +++--- testing/units/virtualnodes/outputs.tf | 7 ++ testing/units/virtualnodes/variables.tf | 14 +-- 8 files changed, 167 insertions(+), 22 deletions(-) create mode 100644 testing/units/virtualnodes/maas_apikey.txt diff --git a/testing/README.md b/testing/README.md index 283379915..9f67dedf9 100644 --- a/testing/README.md +++ b/testing/README.md @@ -23,6 +23,7 @@ snapcraft pack --use-lxd ## TODO * [ ] Expose a knob to turn on/off the log level of terragrunt/terraform. +* [ ] Redirect libvirt instances' console to a log file ## Known Issues diff --git a/testing/units/maas/main.tf b/testing/units/maas/main.tf index 326454427..4216fa914 100644 --- a/testing/units/maas/main.tf +++ b/testing/units/maas/main.tf @@ -5,6 +5,8 @@ locals { generic_dhcp_end = cidrhost(local.generic_net_addresses[0], 254) generic_reserved_start = cidrhost(local.generic_net_addresses[0], 1) generic_reserved_end = cidrhost(local.generic_net_addresses[0], 5) + external_reserved_start = cidrhost(local.external_net_addresses[0], 1) + external_reserved_end = cidrhost(local.external_net_addresses[0], 5) } resource "maas_configuration" "kernel_opts" { @@ -145,7 +147,7 @@ resource "maas_vlan" "generic_vlan" { space = maas_space.space_generic.name } -# Subent - generic +# Subnet - generic data "maas_subnet" "generic_subnet" { cidr = local.generic_net_addresses[0] @@ -175,6 +177,7 @@ resource "maas_subnet_ip_range" "generic_subnet_reserved_range" { start_ip = local.generic_reserved_start end_ip = local.generic_reserved_end type = "reserved" + comment = "Internal API" } resource "maas_vlan_dhcp" "generic_vlan_dhcp" { @@ -184,6 +187,68 @@ resource "maas_vlan_dhcp" "generic_vlan_dhcp" { ip_ranges = [maas_subnet_ip_range.generic_subnet_dhcp_range.id] } +# # Fabric - external +# data "maas_fabric" "external_fabric" { +# name = "fabric-0" # TODO: fix name +# } + +# import { +# to = maas_fabric.external_fabric +# id = "${data.maas_fabric.external_fabric.id}" +# } + +# resource "maas_fabric" "external_fabric" { +# name = "fabric-0" # TODO: fix name +# } + +# # VLAN - Generic +# data "maas_vlan" "external_vlan" { +# fabric = resource.maas_fabric.external_fabric.id +# vlan = 0 +# } + +# import { +# to = maas_vlan.external_vlan +# id = "${data.maas_fabric.external_fabric.name}:0" +# } + +# resource "maas_vlan" "external_vlan" { +# fabric = maas_fabric.external_fabric.id +# vid = 0 +# name = "untagged" +# space = maas_space.space_external.name +# } + +# # Subnet - external + +# data "maas_subnet" "external_subnet" { +# cidr = local.external_net_addresses[0] +# } + +# import { +# to = maas_subnet.external_subnet +# id = "${data.maas_subnet.external_subnet.cidr}" +# } + +# resource "maas_subnet" "external_subnet" { +# name = local.external_net_addresses[0] +# cidr = local.external_net_addresses[0] +# fabric = maas_fabric.external_fabric.id +# vlan = maas_vlan.external_vlan.vid +# } + + +# resource "maas_subnet_ip_range" "external_subnet_reserved_range" { +# subnet = maas_subnet.external_subnet.id +# start_ip = local.external_reserved_start +# end_ip = local.external_reserved_end +# type = "reserved" +# comment = "Public API" +# } + + +# Nodes configuration + resource "maas_machine" "node" { count = length(var.nodes) hostname = var.nodes[count.index].name @@ -194,3 +259,57 @@ resource "maas_machine" "node" { }) pxe_mac_address = var.nodes[count.index].mac_address } + +resource "maas_tag" "openstack" { + name = "openstack-sunbeam" + machines = [for node in maas_machine.node : node.id] +} + +resource "maas_tag" "juju" { + name = "juju-controller" + machines = [maas_machine.node[0].id, maas_machine.node[1].id, maas_machine.node[2].id] +} + +resource "maas_tag" "sunbeam" { + name = "sunbeam" + machines = [maas_machine.node[0].id, maas_machine.node[1].id, maas_machine.node[2].id] +} + +resource "maas_tag" "control" { + name = "control" + machines = [maas_machine.node[0].id, maas_machine.node[1].id, maas_machine.node[2].id] +} + +resource "maas_tag" "compute" { + name = "compute" + machines = [maas_machine.node[3].id, maas_machine.node[4].id, maas_machine.node[5].id] +} + +resource "maas_tag" "storage" { + name = "storage" + machines = [maas_machine.node[3].id, maas_machine.node[4].id, maas_machine.node[5].id] +} + +locals { + osd_hosts = flatten([for node in var.nodes : [for osd_disk in node.osd_disks : { hostname = node.name, disk_serial = osd_disk.serial, disk_size = osd_disk.size } ]]) +} + + +# import block devices +import { + to = maas_block_device.osd + id = "${data.maas_block_device.generic_fabric.id}" +} + +resource "maas_block_device" "osd" { + depends_on = [maas_machine.node] + count = length(local.osd_hosts) + machine = local.osd_hosts[count.index].hostname + name = substr(local.osd_hosts[count.index].disk_serial, 0, 20) + + id_path = "/dev/disk/by-id/virtio-${substr(local.osd_hosts[count.index].disk_serial, 0, 20)}" + size_gigabytes = local.osd_hosts[count.index].disk_size + tags = [ + "ceph", + ] +} diff --git a/testing/units/maas/terragrunt.hcl b/testing/units/maas/terragrunt.hcl index 02cfcf9d5..c00f9396a 100644 --- a/testing/units/maas/terragrunt.hcl +++ b/testing/units/maas/terragrunt.hcl @@ -5,6 +5,16 @@ include "stack" { terraform { source = "." + before_hook "ensure_authorized_keys" { + commands = ["apply", "plan"] + execute = ["bash", "-c", <<-EOT + mkdir -p $HOME/.ssh + chmod 700 $HOME/.ssh + touch $HOME/.ssh/authorized_keys + chmod 600 $HOME/.ssh/authorized_keys + EOT + ] + } } dependency "virtualnodes" { diff --git a/testing/units/maas/variables.tf b/testing/units/maas/variables.tf index a86117a69..efc5953b9 100644 --- a/testing/units/maas/variables.tf +++ b/testing/units/maas/variables.tf @@ -24,6 +24,11 @@ variable "nodes" { type = list(object({ name = string mac_address = string + osd_disks = list(object({ + serial = string + size = number + dev = string + })) })) } diff --git a/testing/units/virtualnodes/maas_apikey.txt b/testing/units/virtualnodes/maas_apikey.txt new file mode 100644 index 000000000..e69de29bb diff --git a/testing/units/virtualnodes/main.tf b/testing/units/virtualnodes/main.tf index 797ea409a..d2b3e5d24 100644 --- a/testing/units/virtualnodes/main.tf +++ b/testing/units/virtualnodes/main.tf @@ -96,18 +96,20 @@ resource "libvirt_pool" "sunbeam" { #### Volumes resource "libvirt_volume" "node_vol" { - name = "node_${count.index}.qcow2" - count = var.nodes_count - pool = libvirt_pool.sunbeam.name - capacity = var.node_rootfs_size + name = "node_${count.index}.qcow2" + count = var.nodes_count + pool = libvirt_pool.sunbeam.name + # transform GB to bytes + capacity = var.node_rootfs_size * 1024 * 1024 * 1024 target = { format = { type = "qcow2" } } } -resource "libvirt_volume" "node_vol_secondary" { - name = "node_${count.index}_secondary.qcow2" - count = var.nodes_count - pool = libvirt_pool.sunbeam.name - capacity = var.node_secondary_disk_size +resource "libvirt_volume" "node_vol_osd" { + name = "node_${count.index}_osd.qcow2" + count = var.nodes_count + pool = libvirt_pool.sunbeam.name + # transform GB to bytes + capacity = var.node_osd_disk_size * 1024 * 1024 * 1024 target = { format = { type = "qcow2" } } } @@ -126,7 +128,8 @@ resource "libvirt_volume" "ubuntu_noble" { resource "libvirt_volume" "maas_controller_vol" { name = "maas-controller-vol" pool = libvirt_pool.sunbeam.name - capacity = var.maas_controller_rootfs_size + # transform GB to bytes + capacity = var.maas_controller_rootfs_size * 1024 * 1024 * 1024 target = { format = { type = "qcow2" } } backing_store = { path = libvirt_volume.ubuntu_noble.path @@ -193,7 +196,7 @@ resource "libvirt_domain" "maas_controller" { volume = libvirt_volume.maas_controller_vol.name } } - target = { dev = "sda", bus = "virtio" } + target = { dev = "vda", bus = "virtio" } driver = { name = "qemu", type = "qcow2" } }, { @@ -275,7 +278,7 @@ resource "libvirt_domain" "node" { volume = libvirt_volume.node_vol[count.index].name } } - target = { dev = "sda", bus = "virtio" } + target = { dev = "vda", bus = "virtio" } driver = { name = "qemu", type = "qcow2" } }, { @@ -283,10 +286,10 @@ resource "libvirt_domain" "node" { source = { volume = { pool = libvirt_pool.sunbeam.name - volume = libvirt_volume.node_vol_secondary[count.index].name + volume = libvirt_volume.node_vol_osd[count.index].name } } - target = { dev = "sdb", bus = "virtio" } + target = { dev = "vdb", bus = "virtio" } driver = { name = "qemu", type = "qcow2" } } ] diff --git a/testing/units/virtualnodes/outputs.tf b/testing/units/virtualnodes/outputs.tf index 5ffdda974..294d312e8 100644 --- a/testing/units/virtualnodes/outputs.tf +++ b/testing/units/virtualnodes/outputs.tf @@ -10,6 +10,13 @@ output "nodes" { for node in libvirt_domain.node : { name = node.name mac_address = node.devices.interfaces[0].mac.address + osd_disks = [ + for disk in node.devices.disks : { + serial = disk.serial + size = var.node_osd_disk_size + dev = disk.target.dev + } if strcontains(disk.serial, "CEPH") + ] } ] depends_on = [libvirt_domain.node] diff --git a/testing/units/virtualnodes/variables.tf b/testing/units/virtualnodes/variables.tf index 4d8638474..1625ca496 100644 --- a/testing/units/virtualnodes/variables.tf +++ b/testing/units/virtualnodes/variables.tf @@ -36,15 +36,15 @@ variable "maas_hostname" { } variable "node_rootfs_size" { - description = "Node rootfs disk size (in bytes)" + description = "Node rootfs disk size (in Gigabytes)" type = number - default = 21474836480 # 20 GiB + default = 20 } -variable "node_secondary_disk_size" { - description = "Node secondary disk size (in bytes)" +variable "node_osd_disk_size" { + description = "Node secondary disk size (in Gigabytes)" type = number - default = 21474836480 # 20 GiB + default = 20 } variable "generic_net_domain" { @@ -89,7 +89,7 @@ variable "maas_controller_mac_address" { } variable "maas_controller_rootfs_size" { - description = "MAAS Controller rootfs disk size (in bytes)" + description = "MAAS Controller rootfs disk size (in Gigabytes)" type = number - default = 21474836480 # 20 GiB + default = 20 }