diff --git a/.github/workflows/build-snap.yml b/.github/workflows/build-snap.yml index 0ecde6f08..f4449febc 100644 --- a/.github/workflows/build-snap.yml +++ b/.github/workflows/build-snap.yml @@ -44,92 +44,7 @@ jobs: name: local-${{ needs.build.outputs.snap }} - name: test run: | - set -x - export COLUMNS=256 - - # Check docker, containerd and remove them if exists - sudo apt remove --purge docker.io containerd runc -y - sudo rm -rf /run/containerd - - # Allow lxd controller to reach to k8s controller on loadbalancer ip - # sudo nft insert rule ip filter FORWARD tcp dport 17070 accept - # sudo nft insert rule ip filter FORWARD tcp sport 17070 accept - # With above rules, got the following error: - # api.charmhub.io on 10.152.183.182:53: server misbehaving - # Accept all packets filtered for forward - sudo nft chain ip filter FORWARD '{policy accept;}' - - sudo snap remove --purge lxd - sudo snap install --channel 3.6 juju - - sudo snap install ${{ needs.build.outputs.snap }} --dangerous - sudo snap connect openstack:juju-bin juju:juju-bin - openstack.sunbeam prepare-node-script --bootstrap | bash -x - sudo snap connect openstack:dot-local-share-juju - sudo snap connect openstack:dot-config-openstack - sudo snap connect openstack:dot-local-share-openstack - - # Even though `--topology single --database single` is not used in the - # single-node tutorial, explicitly speficy it here to force the single - # mysql mode. - # The tutorial assumes ~16 GiB of memory where Sunbeam selects the singe - # mysql single mysql mode automatically. And self-hosted runners may - # have more than 32 GiB of memory where Sunbeam selects the multi mysql - # mode instead. So we have to override the Sunbeam's decision to be - # closer to the tutorial scenario. - sg snap_daemon "openstack.sunbeam cluster bootstrap --manifest .github/assets/testing/edge.yml --accept-defaults --topology single --database single" - sg snap_daemon "openstack.sunbeam cluster list" - # Note: Moving configure before enabling caas just to ensure caas images are not downloaded - # To download caas image, require ports to open on firewall to access fedora images. - sg snap_daemon "openstack.sunbeam configure --accept-defaults --openrc demo-openrc" - sg snap_daemon "openstack.sunbeam launch --name test" - # The cloud-init process inside the VM takes ~2 minutes to bring up the - # SSH service after the VM gets ACTIVE in OpenStack - sleep 300 - source demo-openrc - openstack console log show --lines 200 test - demo_floating_ip="$(openstack floating ip list -c 'Floating IP Address' -f value | head -n1)" - ssh -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -i ~/snap/openstack/current/sunbeam "ubuntu@${demo_floating_ip}" true - - sg snap_daemon "openstack.sunbeam enable orchestration" - sg snap_daemon "openstack.sunbeam enable loadbalancer" - sg snap_daemon "openstack.sunbeam enable dns testing.github." - # Disabled until https://github.com/canonical/mysql-router-k8s-operator/issues/452 - # or corresponding juju bug is fixed - # sg snap_daemon "openstack.sunbeam disable dns" - # sg snap_daemon "openstack.sunbeam disable loadbalancer" - # sg snap_daemon "openstack.sunbeam disable orchestration" - - # Vault has storage requirements > 15G - # Commenting as CI servers might not have enough disk space - # sg snap_daemon "openstack.sunbeam enable vault --dev-mode" - # sg snap_daemon "openstack.sunbeam enable secrets" - # sg snap_daemon "openstack.sunbeam disable secrets" - # sg snap_daemon "openstack.sunbeam disable vault" - - # Disable caas temporarily while MySQL memory gets adjusted - # sg snap_daemon "openstack.sunbeam enable caas" - # sg snap_daemon "openstack.sunbeam enable validation" - # If smoke tests fails, logs should be collected via sunbeam command in "Collect logs" - # sg snap_daemon "openstack.sunbeam validation run smoke" - # sg snap_daemon "openstack.sunbeam validation run --output tempest_validation.log" - # sg snap_daemon "openstack.sunbeam disable caas" - # sg snap_daemon "openstack.sunbeam disable validation" - - sg snap_daemon "openstack.sunbeam enable telemetry" - # Commenting observability as storage requirements ~6G - # sg snap_daemon "openstack.sunbeam enable observability embedded" - # Commented disabling observability due to LP#1998282 - # sg snap_daemon "openstack.sunbeam disable observability embedded" - # sg snap_daemon "openstack.sunbeam disable telemetry" - - # Commenting features as storage is full in CI machines - # sg snap_daemon "openstack.sunbeam enable resource-optimization" - # sg snap_daemon "openstack.sunbeam enable instance-recovery" - # Disable IR as the consul pods are stuck in getting terminated - # sg snap_daemon "openstack.sunbeam disable instance-recovery" - # sg snap_daemon "openstack.sunbeam disable resource-optimization" - + ./testing/test-standalone.sh local-${{ needs.build.outputs.snap }} - name: Collect logs if: always() run: | @@ -167,3 +82,46 @@ jobs: - name: Setup tmate session if: ${{ failure() && runner.debug }} uses: canonical/action-tmate@main + functional-test-maas: + needs: build + name: Functional test on MAAS + runs-on: [self-hosted, self-hosted-linux-amd64-noble-private-endpoint-medium] + env: + TESTFLINGER_DIR: .github/workflows/testflinger + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + path: repository + - name: Download snap artifact + id: download + uses: actions/download-artifact@v5 + with: + name: local-${{ needs.build.outputs.snap }} + path: repository + - name: Pack the repository + run: | + tar acf repository.tar.gz repository/ + - name: Create Testflinger job + env: + OPENSTACK_SNAP_PATH: local-${{ needs.build.outputs.snap }} + run: | + # Prepare job + envsubst '$OPENSTACK_SNAP_PATH' \ + < $TESTFLINGER_DIR/job.yaml.tpl \ + > $TESTFLINGER_DIR/job.yaml + - name: Submit job + uses: canonical/testflinger/.github/actions/submit@main + with: + poll: true + job-path: ${{ env.TESTFLINGER_DIR }}/job.yaml + - name: Upload logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: sunbeam_logs + path: logs + retention-days: 30 + - name: Setup tmate session + if: ${{ failure() && runner.debug }} + uses: canonical/action-tmate@main diff --git a/.github/workflows/testflinger/job.yaml.tpl b/.github/workflows/testflinger/job.yaml.tpl new file mode 100644 index 000000000..cdda85af7 --- /dev/null +++ b/.github/workflows/testflinger/job.yaml.tpl @@ -0,0 +1,77 @@ +# -*- mode: yaml -*- +job_queue: openstack +provision_data: + distro: noble +global_timeout: 14400 # 4 hours +output_timeout: 10800 # 180 min +test_data: + attachments: + - local: repository.tar.gz + test_cmds: | + set -ex + scp ./attachments/test/repository.tar.gz "ubuntu@${DEVICE_IP}:" + if ssh "ubuntu@${DEVICE_IP}" ' + set -ex + ssh-import-id lp:freyes + timeout_loop () { + local TIMEOUT=90 + while [ "$TIMEOUT" -gt 0 ]; do + if "$@" > /dev/null 2>&1; then + echo "OK" + return 0 + fi + TIMEOUT=$((TIMEOUT - 1)) + sleep 1 + done + echo "ERROR: $* FAILED" + ret=1 + return 1 + } + # http://pad.lv/2093303 + sudo mv -v /etc/apt/sources.list{,.bak} + # Workaround for: + # E: Failed to fetch http://... Hash Sum mismatch + timeout_loop sudo apt-get update -q + + # include ~/.local/bin in PATH + source ~/.profile + set -o pipefail + # LP: #2097451 + # LP: #2102175 + tar xzvf repository.tar.gz + cd repository/testing/ + + # generate passwordless key if needed + test -f ~/.ssh/passwordless || ssh-keygen -b 2048 -t rsa -f ~/.ssh/passwordless -q -N "" + + # Allow ssh connections to the virtual nodes without having host fingerprint issues. + echo "Host 172.16.1.* 172.16.2.*" >> ~/.ssh/config + echo " UserKnownHostsFile /dev/null" >> ~/.ssh/config + echo " StrictHostKeyChecking no" >> ~/.ssh/config + + # Install depependencies in the hypervisor. + ./install_deps.sh + + # Prepare the testing bed running terragrunt + # make the libvirt group effective in this shell, so terraform can talk to the libvirt unix socket + sudo su - ubuntu -c $(realpath ./deploy.sh) + cd ../ + + # Start the testing using the previously prepare test bed. + export TEST_SNAP_OPENSTACK=${OPENSTACK_SNAP_PATH} + export TEST_MAAS_API_KEY="$(cat /tmp/maas-api.key)" + export TEST_MAAS_URL="http://172.16.1.2:5240/MAAS" + ./testing/test-multinode-maas.sh ${OPENSTACK_SNAP_PATH} + '; then + scp -r "ubuntu@${DEVICE_IP}:repository/artifacts/" artifacts/ || true + find artifacts/ + else + echo "blocking until file /tmp/.continue shows up in ${DEVICE_IP}" + echo ssh ubuntu@${DEVICE_IP} + ssh ubuntu@${DEVICE_IP} "until test -f /tmp/.continue; do sleep 10;done" + + ssh ubuntu@${DEVICE_IP} /home/ubuntu/repository/testing/collect-logs.sh + scp -r "ubuntu@${DEVICE_IP}:repository/artifacts/" artifacts/ || true + find artifacts/ + exit 1 + fi diff --git a/.gitignore b/.gitignore index 084cc5996..3e557f4b4 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,5 @@ cython_debug/ .vscode/ .stestr/ + +.github/workflows/testflinger/*.yaml diff --git a/testing/.gitignore b/testing/.gitignore new file mode 100644 index 000000000..a8fd9076e --- /dev/null +++ b/testing/.gitignore @@ -0,0 +1,5 @@ +*.tfstate +*.tfstate.backup +*.lock.hcl +.terraform/ +.terragrunt-cache/ diff --git a/testing/README.md b/testing/README.md new file mode 100644 index 000000000..9f67dedf9 --- /dev/null +++ b/testing/README.md @@ -0,0 +1,33 @@ +# Testflinger Testing + +## Local Development/Testing + +To run the testing, it's possible to use the `./testing/local-testflinger.sh` script. + +Usage example: + +1. Install the testflinger-cli snap: `sudo snap install testflinger-cli`. +2. Make sure there is a copy of the openstack snap at the toplevel of the git + repo. Use `snap download` or `snapcraft pack`. + +``` sh +snap download --channel 2024.1/edge openstack +``` + +``` sh +snapcraft pack --use-lxd +``` + +3. Run `./testing/local-testflinger.sh`. + +## TODO + +* [ ] Expose a knob to turn on/off the log level of terragrunt/terraform. +* [ ] Redirect libvirt instances' console to a log file + + +## Known Issues + +* When a libvirt instance does PXE boot, there could be situations where it + doesn't boot and it just times out, making the whole deployment timeout or + fail when terraform's apply times out. diff --git a/testing/collect-logs.sh b/testing/collect-logs.sh new file mode 100755 index 000000000..1e4095fcd --- /dev/null +++ b/testing/collect-logs.sh @@ -0,0 +1,13 @@ +#!/bin/bash -ux + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +ARTIFACTS_DIR=$(realpath $SCRIPT_DIR/../artifacts) +mkdir -p $ARTIFACTS_DIR + +## Collect relevant files (if possible) +sudo mkdir /tmp/sosreport/ +sudo sosreport -a --batch --label hypervisor --all-logs --tmp-dir=/tmp/sosreport/ +sudo mv /tmp/sosreport/* $ARTIFACTS_DIR +ssh -i ~/.ssh/passwordless ubuntu@172.16.1.2 "sudo mkdir /tmp/sosreport; sudo sosreport -a --batch --label maas-controller --all-logs --tmp-dir=/tmp/sosreport/; sudo chmod +r /tmp/sosreport/" +scp -i ~/.ssh/passwordless ubuntu@172.16.1.2:"/tmp/sosreport/*" $ARTIFACTS_DIR +sudo chmod +r -R $ARTIFACTS_DIR diff --git a/testing/deploy.sh b/testing/deploy.sh new file mode 100755 index 000000000..7bf6a3d20 --- /dev/null +++ b/testing/deploy.sh @@ -0,0 +1,9 @@ +#!/bin/bash -exu + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +pushd $SCRIPT_DIR + +# export TERRAGRUNT_LOG_LEVEL=trace +# export TF_LOG=TRACE +terragrunt --non-interactive run-all apply diff --git a/testing/install_deps.sh b/testing/install_deps.sh new file mode 100755 index 000000000..8f335da4f --- /dev/null +++ b/testing/install_deps.sh @@ -0,0 +1,52 @@ +#!/bin/bash -x + +if [ "x$(which terragrunt)" != "x0" ]; then + sudo wget -O /usr/local/bin/terragrunt https://github.com/gruntwork-io/terragrunt/releases/download/v0.87.1/terragrunt_linux_amd64 + sudo chmod +x /usr/local/bin/terragrunt +fi + +# install opentofu +if [ "x$(which tofu)" != "x0" ]; then + sudo apt-get update + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg + sudo install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://get.opentofu.org/opentofu.gpg | sudo tee /etc/apt/keyrings/opentofu.gpg >/dev/null + curl -fsSL https://packages.opentofu.org/opentofu/tofu/gpgkey | sudo gpg --no-tty --batch --dearmor -o /etc/apt/keyrings/opentofu-repo.gpg >/dev/null + sudo chmod a+r /etc/apt/keyrings/opentofu.gpg /etc/apt/keyrings/opentofu-repo.gpg +echo \ + "deb [signed-by=/etc/apt/keyrings/opentofu.gpg,/etc/apt/keyrings/opentofu-repo.gpg] https://packages.opentofu.org/opentofu/tofu/any/ any main +deb-src [signed-by=/etc/apt/keyrings/opentofu.gpg,/etc/apt/keyrings/opentofu-repo.gpg] https://packages.opentofu.org/opentofu/tofu/any/ any main" | \ + sudo tee /etc/apt/sources.list.d/opentofu.list > /dev/null + + sudo chmod a+r /etc/apt/sources.list.d/opentofu.list + sudo apt-get update + sudo apt-get install -y -qq tofu +fi + +# install terraform +if [ "x$(which tofu)" != "x0" ]; then + sudo apt-get update -q + sudo apt-get install -yq gnupg software-properties-common + wget -O- https://apt.releases.hashicorp.com/gpg | \ + gpg --dearmor | \ + sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg > /dev/null + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(grep -oP '(?<=UBUNTU_CODENAME=).*' /etc/os-release || lsb_release -cs) main" | \ + sudo tee /etc/apt/sources.list.d/hashicorp.list + sudo apt-get update -q + sudo apt-get install -yq terraform +fi + +if [ "x$(which virsh)" != "x0" ]; then + sudo apt-get install -y -qq \ + sosreport \ + libvirt-daemon \ + libvirt-daemon-driver-qemu \ + libvirt-daemon-system \ + libvirt-clients \ + genisoimage + + # allow a non-root user to use libvirt/virsh easily with no permission issues + sudo sed -i '/^security_driver/d' /etc/libvirt/qemu.conf + echo 'security_driver = "none"' | sudo tee -a /etc/libvirt/qemu.conf >/dev/null + sudo systemctl restart libvirtd +fi diff --git a/testing/local-testflinger.sh b/testing/local-testflinger.sh new file mode 100755 index 000000000..17360a25f --- /dev/null +++ b/testing/local-testflinger.sh @@ -0,0 +1,21 @@ +#!/bin/bash -ex + +TMP_DIR=$(mktemp -d) +trap "rm -rf $TMP_DIR; echo 'Cleaned up temporary file.'" EXIT +cp -rf ../snap-openstack/ $TMP_DIR/repository +pushd $TMP_DIR +tar --exclude=repository/.tox --exclude=repository/.github/workflows/testflinger/repository.tar.gz --exclude=repository/.git -acf repository.tar.gz repository/ +ls -lh repository.tar.gz +popd +export TESTFLINGER_DIR=$(pwd)/.github/workflows/testflinger/ +cp $TMP_DIR/repository.tar.gz $TESTFLINGER_DIR +export OPENSTACK_SNAP_PATH=$(ls openstack_*.snap) +JOB_FILE=$TESTFLINGER_DIR/job.yaml +envsubst '$OPENSTACK_SNAP_PATH' \ + < $TESTFLINGER_DIR/job.yaml.tpl \ + > $JOB_FILE + +test -f $JOB_FILE +cd $TESTFLINGER_DIR +testflinger-cli -d submit --poll $JOB_FILE +rm -rf $TMP_DIR diff --git a/testing/private/.keep b/testing/private/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/testing/private/README.md b/testing/private/README.md new file mode 100644 index 000000000..c161760b9 --- /dev/null +++ b/testing/private/README.md @@ -0,0 +1 @@ +Directory to write temporary files that are private in nature, for example generated password-less ssh keys. diff --git a/testing/stack.hcl b/testing/stack.hcl new file mode 100644 index 000000000..7b3af0be5 --- /dev/null +++ b/testing/stack.hcl @@ -0,0 +1,20 @@ +locals { + units = { + virtualnodes = { + source = "./virtualnodes" + } + + maas = { + source = "./maas" + dependencies = ["virtualnodes"] + } + } + + # Stack-wide variables + stack_config = { + ssh_private_key_path = "~/.ssh/passwordless" + ssh_public_key_path = "~/.ssh/passwordless.pub" + libvirt_uri = get_env("LIBVIRT_DEFAULT_URI", "qemu:///system") + maas_hostname = "maas-controller" + } +} diff --git a/testing/templates/maas-provider.tf.tpl b/testing/templates/maas-provider.tf.tpl new file mode 100644 index 000000000..9d0294f7f --- /dev/null +++ b/testing/templates/maas-provider.tf.tpl @@ -0,0 +1,43 @@ +terraform { + required_version = ">= 0.14.0" + required_providers { + local = { + source = "hashicorp/local" + version = "2.5.3" + } + maas = { + source = "canonical/maas" + version = "2.6.0" + } + netparse = { + source = "gmeligio/netparse" + version = "0.0.4" + } + null = { + source = "hashicorp/null" + version = "3.2.4" + } + tls = { + source = "hashicorp/tls" + version = "4.1.0" + } + } +} + +provider "local" { +} + +provider "maas" { + api_version = "2.0" + api_key = "${api_key}" + api_url = "http://${controller_ip_address}:5240/MAAS" +} + +provider "null" { +} + +provider "netparse" { +} + +provider "tls" { +} diff --git a/testing/test-multinode-maas.sh b/testing/test-multinode-maas.sh new file mode 100755 index 000000000..ceb42803d --- /dev/null +++ b/testing/test-multinode-maas.sh @@ -0,0 +1,114 @@ +#!/bin/bash -eux + +export COLUMNS=256 + +if [ $# -ne 1 ]; then + echo "Invalid number of arguments" >&2 + echo "Usage:" + echo " $0 " +fi + +TEST_SNAP_OPENSTACK=${1} +TEST_JUJU_CHANNEL=${TEST_JUJU_CHANNEL:-3.6} + +if [[ ! -f "${TEST_SNAP_OPENSTACK}" ]]; then + echo "${TEST_SNAP_OPENSTACK}: No such file or directory" >&2 + exit 1 +fi + +if [[ -z "$TEST_MAAS_API_KEY" ]];then + echo "Error: Please define the TEST_MAAS_API_KEY environment variable" >&1 + exit 1 +fi + +if [[ -z "$TEST_MAAS_URL" ]];then + echo "Error: Please define the TEST_MAAS_URL environment variable" >&1 + exit 1 +fi + +if [[ ! -f "$HOME/.ssh/passwordless" ]]; then + ssh-keygen -b 2048 -t rsa -f $HOME/.ssh/passwordless -q -N "" +fi + +function timeout_loop () { + local TIMEOUT=$1; shift + while [ "$TIMEOUT" -gt 0 ]; do + if "$@" > /dev/null 2>&1; then + echo "OK" + return 0 + fi + TIMEOUT=$((TIMEOUT - 1)) + sleep 1 + done + echo "ERROR: $* FAILED" + ret=1 + return 1 +} + +function run_snap_daemon { + sg snap_daemon -c "$*" +} + +sudo snap install --channel ${TEST_JUJU_CHANNEL} juju +sudo snap install --dangerous ${TEST_SNAP_OPENSTACK} +sudo snap connect openstack:juju-bin juju:juju-bin +openstack.sunbeam prepare-node-script --bootstrap | bash -x + +# connect plugs manually since the snap is installed from a locally built one. +sudo snap connect openstack:dot-local-share-juju +sudo snap connect openstack:dot-config-openstack +sudo snap connect openstack:dot-local-share-openstack +sudo snap alias openstack.sunbeam sunbeam + +run_snap_daemon sunbeam deployment add maas mymaas ${TEST_MAAS_API_KEY} ${TEST_MAAS_URL} +run_snap_daemon sunbeam deployment space map space-generic:data +run_snap_daemon sunbeam deployment space map space-generic:internal +run_snap_daemon sunbeam deployment space map space-generic:management +run_snap_daemon sunbeam deployment space map space-generic:storage +run_snap_daemon sunbeam deployment space map space-generic:storage-cluster +run_snap_daemon sunbeam deployment space map space-external:public + +run_snap_daemon sunbeam deployment validate + +run_snap_daemon sunbeam cluster bootstrap + +run_snap_daemon sunbeam cluster list + +run_snap_daemon sunbeam configure --accept-defaults --openrc demo-openrc +run_snap_daemon sunbeam launch --name test +# The cloud-init process inside the VM takes ~2 minutes to bring up the +# SSH service after the VM gets ACTIVE in OpenStack +sleep 300 +source demo-openrc +openstack console log show --lines 200 test +demo_floating_ip="$(openstack floating ip list -c 'Floating IP Address' -f value | head -n1)" + +TIMEOUT=120 +echo "Block until ${demo_floating_ip} on port 22 is reachable (timeout: ${TIMEOUT})" +timeout_loop ${TIMEOUT} nc -v -z ${demo_floating_ip} +ssh -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -i ~/snap/openstack/current/sunbeam "ubuntu@${demo_floating_ip}" true + +run_snap_daemon sunbeam enable orchestration +run_snap_daemon sunbeam enable loadbalancer +run_snap_daemon sunbeam enable dns testing.github. + +run_snap_daemon sunbeam enable vault --dev-mode +run_snap_daemon sunbeam enable secrets +run_snap_daemon sunbeam disable secrets +run_snap_daemon sunbeam disable vault + +sg snap_daemon "openstack.sunbeam enable validation" +# sg snap_daemon "openstack.sunbeam validation run smoke" +# sg snap_daemon "openstack.sunbeam validation run --output tempest_validation.log" + +sg snap_daemon "openstack.sunbeam enable telemetry" +sg snap_daemon "openstack.sunbeam enable observability embedded" +sg snap_daemon "openstack.sunbeam disable observability embedded" +sg snap_daemon "openstack.sunbeam disable telemetry" + +# Commenting features as storage is full in CI machines +# sg snap_daemon "openstack.sunbeam enable resource-optimization" +# sg snap_daemon "openstack.sunbeam enable instance-recovery" +# Disable IR as the consul pods are stuck in getting terminated +# sg snap_daemon "openstack.sunbeam disable instance-recovery" +# sg snap_daemon "openstack.sunbeam disable resource-optimization" diff --git a/testing/test-standalone.sh b/testing/test-standalone.sh new file mode 100755 index 000000000..98fa43423 --- /dev/null +++ b/testing/test-standalone.sh @@ -0,0 +1,99 @@ +#!/bin/bash +set -x +export COLUMNS=256 + +if [$# -ne 1]; then + echo "Invalid number of arguments" >&2 + echo "Usage:" + echo " $0 " +fi + +TEST_SNAP_OPENSTACK=${1} + +if [[ ! -f "${TEST_SNAP_OPENSTACK}" ]]; then + echo "${TEST_SNAP_OPENSTACK}: No such file or directory" >&2 + exit 1 +fi + +# Check docker, containerd and remove them if exists +sudo apt remove --purge docker.io containerd runc -y +sudo rm -rf /run/containerd + +# Allow lxd controller to reach to k8s controller on loadbalancer ip +# sudo nft insert rule ip filter FORWARD tcp dport 17070 accept +# sudo nft insert rule ip filter FORWARD tcp sport 17070 accept +# With above rules, got the following error: +# api.charmhub.io on 10.152.183.182:53: server misbehaving +# Accept all packets filtered for forward +sudo nft chain ip filter FORWARD '{policy accept;}' + +sudo snap remove --purge lxd +sudo snap install --channel 3.6 juju + +sudo snap install --dangerous ${TEST_SNAP_OPENSTACK} +sudo snap connect openstack:juju-bin juju:juju-bin +openstack.sunbeam prepare-node-script --bootstrap | bash -x +sudo snap connect openstack:dot-local-share-juju +sudo snap connect openstack:dot-config-openstack +sudo snap connect openstack:dot-local-share-openstack + +# Even though `--topology single --database single` is not used in the +# single-node tutorial, explicitly speficy it here to force the single +# mysql mode. +# The tutorial assumes ~16 GiB of memory where Sunbeam selects the singe +# mysql single mysql mode automatically. And self-hosted runners may +# have more than 32 GiB of memory where Sunbeam selects the multi mysql +# mode instead. So we have to override the Sunbeam's decision to be +# closer to the tutorial scenario. +sg snap_daemon "openstack.sunbeam cluster bootstrap --manifest .github/assets/testing/edge.yml --accept-defaults --topology single --database single" +sg snap_daemon "openstack.sunbeam cluster list" +# Note: Moving configure before enabling caas just to ensure caas images are not downloaded +# To download caas image, require ports to open on firewall to access fedora images. +sg snap_daemon "openstack.sunbeam configure --accept-defaults --openrc demo-openrc" +sg snap_daemon "openstack.sunbeam launch --name test" +# The cloud-init process inside the VM takes ~2 minutes to bring up the +# SSH service after the VM gets ACTIVE in OpenStack +sleep 300 +source demo-openrc +openstack console log show --lines 200 test +demo_floating_ip="$(openstack floating ip list -c 'Floating IP Address' -f value | head -n1)" +ssh -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -i ~/snap/openstack/current/sunbeam "ubuntu@${demo_floating_ip}" true + +sg snap_daemon "openstack.sunbeam enable orchestration" +sg snap_daemon "openstack.sunbeam enable loadbalancer" +sg snap_daemon "openstack.sunbeam enable dns testing.github." +# Disabled until https://github.com/canonical/mysql-router-k8s-operator/issues/452 +# or corresponding juju bug is fixed +# sg snap_daemon "openstack.sunbeam disable dns" +# sg snap_daemon "openstack.sunbeam disable loadbalancer" +# sg snap_daemon "openstack.sunbeam disable orchestration" + +# Vault has storage requirements > 15G +# Commenting as CI servers might not have enough disk space +# sg snap_daemon "openstack.sunbeam enable vault --dev-mode" +# sg snap_daemon "openstack.sunbeam enable secrets" +# sg snap_daemon "openstack.sunbeam disable secrets" +# sg snap_daemon "openstack.sunbeam disable vault" + +# Disable caas temporarily while MySQL memory gets adjusted +# sg snap_daemon "openstack.sunbeam enable caas" +# sg snap_daemon "openstack.sunbeam enable validation" +# If smoke tests fails, logs should be collected via sunbeam command in "Collect logs" +# sg snap_daemon "openstack.sunbeam validation run smoke" +# sg snap_daemon "openstack.sunbeam validation run --output tempest_validation.log" +# sg snap_daemon "openstack.sunbeam disable caas" +# sg snap_daemon "openstack.sunbeam disable validation" + +sg snap_daemon "openstack.sunbeam enable telemetry" +# Commenting observability as storage requirements ~6G +# sg snap_daemon "openstack.sunbeam enable observability embedded" +# Commented disabling observability due to LP#1998282 +# sg snap_daemon "openstack.sunbeam disable observability embedded" +# sg snap_daemon "openstack.sunbeam disable telemetry" + +# Commenting features as storage is full in CI machines +# sg snap_daemon "openstack.sunbeam enable resource-optimization" +# sg snap_daemon "openstack.sunbeam enable instance-recovery" +# Disable IR as the consul pods are stuck in getting terminated +# sg snap_daemon "openstack.sunbeam disable instance-recovery" +# sg snap_daemon "openstack.sunbeam disable resource-optimization" diff --git a/testing/units/maas/main.tf b/testing/units/maas/main.tf new file mode 100644 index 000000000..4216fa914 --- /dev/null +++ b/testing/units/maas/main.tf @@ -0,0 +1,315 @@ +locals { + generic_net_addresses = ["172.16.1.0/24"] + external_net_addresses = ["172.16.2.0/24"] + generic_dhcp_start = cidrhost(local.generic_net_addresses[0], 200) + generic_dhcp_end = cidrhost(local.generic_net_addresses[0], 254) + generic_reserved_start = cidrhost(local.generic_net_addresses[0], 1) + generic_reserved_end = cidrhost(local.generic_net_addresses[0], 5) + external_reserved_start = cidrhost(local.external_net_addresses[0], 1) + external_reserved_end = cidrhost(local.external_net_addresses[0], 5) +} + +resource "maas_configuration" "kernel_opts" { + key = "kernel_opts" + value = "console=ttyS0 console=tty0" +} + +resource "maas_configuration" "dnssec_disable" { + key = "dnssec_validation" + value = "no" +} + +resource "maas_configuration" "completed_intro" { + key = "completed_intro" + value = "true" +} + +resource "maas_configuration" "upstream_dns" { + key = "upstream_dns" + value = var.upstream_dns_server +} + +# NOTE(freyes): this selection is made automatically by MAAS when installed, +# running this block raises the following error: +# +# Error: error creating ubuntu noble: ServerError: 400 Bad Request ({"__all__": +# ["Boot source selection with this Boot source, Os and Release already +# exists."]}) +# +# It's possible to use the `import` block, although there seems to not be a need +# of making this a managed resource at the moment. +# +# data "maas_boot_source" "boot_source" {} +# +# resource "maas_boot_source_selection" "amd64" { +# boot_source = data.maas_boot_source.boot_source.id +# os = "ubuntu" +# release = "noble" +# arches = ["amd64"] +# } + + +# Generate SSH key pair +resource "tls_private_key" "ssh_key" { + algorithm = "RSA" + rsa_bits = 4096 +} + +# Save private key to local file +resource "local_file" "private_key" { + content = tls_private_key.ssh_key.private_key_pem + filename = "${path.module}/../../private/id_rsa" + file_permission = "0600" +} + +# Read existing file content +data "local_file" "existing_keys" { + filename = pathexpand("~/.ssh/authorized_keys") +} + +locals { + existing_content = try(data.local_file.existing_keys.content, "") + new_key = tls_private_key.ssh_key.public_key_openssh + all_keys = "${local.existing_content}${local.new_key}" + + kvm_host_addr = provider::netparse::parse_url(var.libvirt_uri).host +} + +resource "local_file" "updated_keys" { + content = local.all_keys + filename = pathexpand("~/.ssh/authorized_keys") +} + +resource "null_resource" "maas_controller_null" { + + connection { + type = "ssh" + user = "ubuntu" + private_key = file(var.ssh_private_key_path) + host = var.maas_controller_ip_address + } + + provisioner "remote-exec" { + inline = [ + "#!/bin/bash", + "sudo mkdir -p /var/snap/maas/current/root/.ssh", + "echo '${tls_private_key.ssh_key.private_key_openssh}' | sudo tee /var/snap/maas/current/root/.ssh/id_rsa", + "sudo chmod 700 /var/snap/maas/current/root/.ssh", + "sudo chmod 600 /var/snap/maas/current/root/.ssh/id_rsa", + "ssh-keyscan -H ${local.kvm_host_addr} | sudo tee -a /var/snap/maas/current/root/.ssh/known_hosts", + "sudo chmod 600 /var/snap/maas/current/root/.ssh/known_hosts", + ] + } +} + + +data "maas_rack_controller" "primary" { + hostname = var.maas_hostname +} + +resource "maas_space" "space_external" { + name = "space-external" +} + +resource "maas_space" "space_generic" { + name = "space-generic" +} + +# Fabric - generic +data "maas_fabric" "generic_fabric" { + name = "fabric-0" +} + +import { + to = maas_fabric.generic_fabric + id = "${data.maas_fabric.generic_fabric.id}" +} + +resource "maas_fabric" "generic_fabric" { + name = "fabric-0" +} + +# VLAN - Generic +data "maas_vlan" "generic_vlan" { + fabric = resource.maas_fabric.generic_fabric.id + vlan = 0 +} + +import { + to = maas_vlan.generic_vlan + id = "${data.maas_fabric.generic_fabric.name}:0" +} + +resource "maas_vlan" "generic_vlan" { + fabric = maas_fabric.generic_fabric.id + vid = 0 + name = "untagged" + space = maas_space.space_generic.name +} + +# Subnet - generic + +data "maas_subnet" "generic_subnet" { + cidr = local.generic_net_addresses[0] +} + +import { + to = maas_subnet.generic_subnet + id = "${data.maas_subnet.generic_subnet.cidr}" +} + +resource "maas_subnet" "generic_subnet" { + name = local.generic_net_addresses[0] + cidr = local.generic_net_addresses[0] + fabric = maas_fabric.generic_fabric.id + vlan = maas_vlan.generic_vlan.vid +} + +resource "maas_subnet_ip_range" "generic_subnet_dhcp_range" { + subnet = maas_subnet.generic_subnet.id + start_ip = local.generic_dhcp_start + end_ip = local.generic_dhcp_end + type = "dynamic" +} + +resource "maas_subnet_ip_range" "generic_subnet_reserved_range" { + subnet = maas_subnet.generic_subnet.id + start_ip = local.generic_reserved_start + end_ip = local.generic_reserved_end + type = "reserved" + comment = "Internal API" +} + +resource "maas_vlan_dhcp" "generic_vlan_dhcp" { + fabric = maas_fabric.generic_fabric.id + vlan = maas_vlan.generic_vlan.vid + primary_rack_controller = data.maas_rack_controller.primary.id + ip_ranges = [maas_subnet_ip_range.generic_subnet_dhcp_range.id] +} + +# # Fabric - external +# data "maas_fabric" "external_fabric" { +# name = "fabric-0" # TODO: fix name +# } + +# import { +# to = maas_fabric.external_fabric +# id = "${data.maas_fabric.external_fabric.id}" +# } + +# resource "maas_fabric" "external_fabric" { +# name = "fabric-0" # TODO: fix name +# } + +# # VLAN - Generic +# data "maas_vlan" "external_vlan" { +# fabric = resource.maas_fabric.external_fabric.id +# vlan = 0 +# } + +# import { +# to = maas_vlan.external_vlan +# id = "${data.maas_fabric.external_fabric.name}:0" +# } + +# resource "maas_vlan" "external_vlan" { +# fabric = maas_fabric.external_fabric.id +# vid = 0 +# name = "untagged" +# space = maas_space.space_external.name +# } + +# # Subnet - external + +# data "maas_subnet" "external_subnet" { +# cidr = local.external_net_addresses[0] +# } + +# import { +# to = maas_subnet.external_subnet +# id = "${data.maas_subnet.external_subnet.cidr}" +# } + +# resource "maas_subnet" "external_subnet" { +# name = local.external_net_addresses[0] +# cidr = local.external_net_addresses[0] +# fabric = maas_fabric.external_fabric.id +# vlan = maas_vlan.external_vlan.vid +# } + + +# resource "maas_subnet_ip_range" "external_subnet_reserved_range" { +# subnet = maas_subnet.external_subnet.id +# start_ip = local.external_reserved_start +# end_ip = local.external_reserved_end +# type = "reserved" +# comment = "Public API" +# } + + +# Nodes configuration + +resource "maas_machine" "node" { + count = length(var.nodes) + hostname = var.nodes[count.index].name + power_type = "virsh" + power_parameters = jsonencode({ + power_address = var.libvirt_uri + power_id = var.nodes[count.index].name + }) + pxe_mac_address = var.nodes[count.index].mac_address +} + +resource "maas_tag" "openstack" { + name = "openstack-sunbeam" + machines = [for node in maas_machine.node : node.id] +} + +resource "maas_tag" "juju" { + name = "juju-controller" + machines = [maas_machine.node[0].id, maas_machine.node[1].id, maas_machine.node[2].id] +} + +resource "maas_tag" "sunbeam" { + name = "sunbeam" + machines = [maas_machine.node[0].id, maas_machine.node[1].id, maas_machine.node[2].id] +} + +resource "maas_tag" "control" { + name = "control" + machines = [maas_machine.node[0].id, maas_machine.node[1].id, maas_machine.node[2].id] +} + +resource "maas_tag" "compute" { + name = "compute" + machines = [maas_machine.node[3].id, maas_machine.node[4].id, maas_machine.node[5].id] +} + +resource "maas_tag" "storage" { + name = "storage" + machines = [maas_machine.node[3].id, maas_machine.node[4].id, maas_machine.node[5].id] +} + +locals { + osd_hosts = flatten([for node in var.nodes : [for osd_disk in node.osd_disks : { hostname = node.name, disk_serial = osd_disk.serial, disk_size = osd_disk.size } ]]) +} + + +# import block devices +import { + to = maas_block_device.osd + id = "${data.maas_block_device.generic_fabric.id}" +} + +resource "maas_block_device" "osd" { + depends_on = [maas_machine.node] + count = length(local.osd_hosts) + machine = local.osd_hosts[count.index].hostname + name = substr(local.osd_hosts[count.index].disk_serial, 0, 20) + + id_path = "/dev/disk/by-id/virtio-${substr(local.osd_hosts[count.index].disk_serial, 0, 20)}" + size_gigabytes = local.osd_hosts[count.index].disk_size + tags = [ + "ceph", + ] +} diff --git a/testing/units/maas/terragrunt.hcl b/testing/units/maas/terragrunt.hcl new file mode 100644 index 000000000..c00f9396a --- /dev/null +++ b/testing/units/maas/terragrunt.hcl @@ -0,0 +1,52 @@ +include "stack" { + path = find_in_parent_folders("stack.hcl") + expose = true +} + +terraform { + source = "." + before_hook "ensure_authorized_keys" { + commands = ["apply", "plan"] + execute = ["bash", "-c", <<-EOT + mkdir -p $HOME/.ssh + chmod 700 $HOME/.ssh + touch $HOME/.ssh/authorized_keys + chmod 600 $HOME/.ssh/authorized_keys + EOT + ] + } +} + +dependency "virtualnodes" { + config_path = "../virtualnodes" + + mock_outputs = { + maas_api_key = "ConsumerSecret:TokenKey:TokenSecret" + maas_controller_ip_address = "1.2.3.4" + nodes = [] + } + mock_outputs_allowed_terraform_commands = ["plan", "validate"] +} + +# Generate provider configuration using dependency outputs +generate "provider" { + path = "provider.tf" + if_exists = "overwrite" + contents = templatefile("${get_parent_terragrunt_dir()}/templates/maas-provider.tf.tpl", { + api_key = dependency.virtualnodes.outputs.maas_api_key + controller_ip_address = dependency.virtualnodes.outputs.maas_controller_ip_address + }) +} + +inputs = merge( + include.stack.locals.stack_config, + { + maas_api_key = dependency.virtualnodes.outputs.maas_api_key + maas_controller_ip_address = dependency.virtualnodes.outputs.maas_controller_ip_address + nodes = dependency.virtualnodes.outputs.nodes + # TODO(freyes): make `172.16.1.1` dynamic, it's the gateway of the generic + # network created by libvirt, so this should be an output of the + # virtualnodes' unit. + libvirt_uri = format("qemu+ssh://%s@%s/system", get_env("USER"), "172.16.1.1") + } +) diff --git a/testing/units/maas/variables.tf b/testing/units/maas/variables.tf new file mode 100644 index 000000000..efc5953b9 --- /dev/null +++ b/testing/units/maas/variables.tf @@ -0,0 +1,45 @@ +variable "libvirt_uri" { + description = "Libvirt connection URI" + type = string + default = "qemu:///system" +} + +variable "maas_api_key" { + description = "MAAS Admin API Key" + type = string +} + +variable "maas_controller_ip_address" { + description = "MAAS Controller IP Address" + type = string +} + +variable "maas_hostname" { + description = "MAAS controller hostname" + type = string +} + +variable "nodes" { + description = "List of (virtual) nodes" + type = list(object({ + name = string + mac_address = string + osd_disks = list(object({ + serial = string + size = number + dev = string + })) + })) +} + +variable "ssh_private_key_path" { + description = "Path to the SSH private key to use in deployed nodes" + type = string + default = "~/.ssh/id_ecdsa" +} + +variable "upstream_dns_server" { + description = "upstream dns server to use in MAAS" + type = string + default = "8.8.8.8" +} diff --git a/testing/units/virtualnodes/maas_apikey.txt b/testing/units/virtualnodes/maas_apikey.txt new file mode 100644 index 000000000..e69de29bb diff --git a/testing/units/virtualnodes/main.tf b/testing/units/virtualnodes/main.tf new file mode 100644 index 000000000..d2b3e5d24 --- /dev/null +++ b/testing/units/virtualnodes/main.tf @@ -0,0 +1,343 @@ +terraform { + required_version = ">= 0.14.0" + required_providers { + libvirt = { + source = "dmacvicar/libvirt" + version = "0.9.1" + } + external = { + source = "hashicorp/external" + version = "2.3.5" + } + } +} + +provider "libvirt" { + uri = var.libvirt_uri +} + +#### Locals +locals { + maas_controller_ip_addr = "172.16.1.2" + generic_net_addresses = { + start = "172.16.1.2", + end = "172.16.1.254", + cidr = "172.16.1.0/24", + gateway = "172.16.1.1" + } + external_net_addresses = { + start = "172.16.2.2", + end = "172.16.2.254", + cidr = "172.16.2.0/24", + gateway = "172.16.2.1" + } +} + +#### Networks + +resource "libvirt_network" "generic_net" { + name = "generic_net" + autostart = true + + domain = { + name = var.generic_net_domain + } + forward = { + nat = { + ports = [ + { + start = 1024 + end = 65535 + } + ] + } + } + ips = [ + { + address = local.generic_net_addresses.gateway + prefix = 24 + } + ] +} + +resource "libvirt_network" "external_net" { + name = "external_net" + autostart = true + + domain = { + name = var.external_net_domain + } + forward = { + nat = { + ports = [ + { + start = 1024 + end = 65535 + } + ] + } + } + ips = [ + { + address = local.external_net_addresses.gateway + prefix = 24 + } + ] +} + +resource "libvirt_pool" "sunbeam" { + name = "sunbeam" + type = "dir" + target = { + path = var.storage_pool_path + } +} + +#### Volumes + +resource "libvirt_volume" "node_vol" { + name = "node_${count.index}.qcow2" + count = var.nodes_count + pool = libvirt_pool.sunbeam.name + # transform GB to bytes + capacity = var.node_rootfs_size * 1024 * 1024 * 1024 + target = { format = { type = "qcow2" } } +} + +resource "libvirt_volume" "node_vol_osd" { + name = "node_${count.index}_osd.qcow2" + count = var.nodes_count + pool = libvirt_pool.sunbeam.name + # transform GB to bytes + capacity = var.node_osd_disk_size * 1024 * 1024 * 1024 + target = { format = { type = "qcow2" } } +} + + +resource "libvirt_volume" "ubuntu_noble" { + name = "ubuntu-noble.qcow2" + pool = libvirt_pool.sunbeam.name + target = { format = { type = "qcow2" } } + create = { + content = { + url = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" + } + } +} + +resource "libvirt_volume" "maas_controller_vol" { + name = "maas-controller-vol" + pool = libvirt_pool.sunbeam.name + # transform GB to bytes + capacity = var.maas_controller_rootfs_size * 1024 * 1024 * 1024 + target = { format = { type = "qcow2" } } + backing_store = { + path = libvirt_volume.ubuntu_noble.path + format = { type = "qcow2" } + } +} + +#### Virtual machines (domains) + +resource "libvirt_cloudinit_disk" "maas_controller_cloudinit" { + name = "maas_controller_cloudinit.iso" + meta_data = yamlencode({ + instance-id = "maas-controller" + local-hostname = "maas-controller" + }) + user_data = templatefile( + "${path.module}/templates/maas_controller.cloudinit.cfg", + { + address = local.maas_controller_ip_addr + dns_server = var.upstream_dns_server + maas_hostname = var.maas_hostname + networks = "generic:172.16.1.0/24" + ssh_public_key = file(var.ssh_public_key_path) + }) + network_config = templatefile( + "${path.module}/templates/maas_controller.netplan.yaml", + { + dns_server = var.upstream_dns_server + ip_address = local.maas_controller_ip_addr + mac_address = var.maas_controller_mac_address + }) +} + +resource "libvirt_volume" "maas_controller_cloudinit_vol" { + name = "maas_controller_cloudinit_vol" + pool = libvirt_pool.sunbeam.name + create = { + content = { + url = libvirt_cloudinit_disk.maas_controller_cloudinit.path + } + } +} + +resource "libvirt_domain" "maas_controller" { + name = "maas-controller" + memory = var.maas_controller_mem + vcpu = var.maas_controller_vcpu + running = true + type = "kvm" + cpu = { mode = "host-passthrough" } + + os = { + type = "hvm" + type_arch = "x86_64" + type_machine = "q35" + boot_devices = [{ dev = "hd" }] + } + devices = { + disks = [ + { + source = { + volume = { + pool = libvirt_pool.sunbeam.name + volume = libvirt_volume.maas_controller_vol.name + } + } + target = { dev = "vda", bus = "virtio" } + driver = { name = "qemu", type = "qcow2" } + }, + { + source = { + volume = { + pool = libvirt_pool.sunbeam.name + volume = libvirt_volume.maas_controller_cloudinit_vol.name + } + } + target = { dev = "hdd", bus = "virtio" } + } + ] + interfaces = [ + { + model = { type = "virtio"} + source = { + network = { + network = libvirt_network.generic_net.name + } + } + mac = { address = var.maas_controller_mac_address } + } + ] + consoles = [ + { target = { type = "serial" } }, + { target = { type = "virtio", port = "1" }} + ] + graphics = [ + { + type = "vnc" + vnc = { + autoport = true + listen = "127.0.0.1" + } + }, + ] + } + + connection { + type = "ssh" + user = "ubuntu" + private_key = file(var.ssh_private_key_path) + host = local.maas_controller_ip_addr + } + + provisioner "remote-exec" { + inline = [ + "until test -f /tmp/.i_am_done; do sleep 10;done", + ] + } +} + +resource "libvirt_domain" "node" { + depends_on = [ + libvirt_domain.maas_controller, + ] + count = var.nodes_count + name = "node-${count.index}" + memory = var.node_mem + vcpu = var.node_vcpu + running = false + type = "kvm" + cpu = { + mode = "host-passthrough" + } + os = { + type = "hvm" + type_arch = "x86_64" + type_machine = "q35" + boot_devices = [{ dev = "network" }, { dev = "hd" }] + } + devices = { + disks = [ + { + serial = format("DISK-ROOT-%06d", count.index) + source = { + volume = { + pool = libvirt_pool.sunbeam.name + volume = libvirt_volume.node_vol[count.index].name + } + } + target = { dev = "vda", bus = "virtio" } + driver = { name = "qemu", type = "qcow2" } + }, + { + serial = format("DISK-CEPH-%06d", count.index) + source = { + volume = { + pool = libvirt_pool.sunbeam.name + volume = libvirt_volume.node_vol_osd[count.index].name + } + } + target = { dev = "vdb", bus = "virtio" } + driver = { name = "qemu", type = "qcow2" } + } + ] + interfaces = [ + { + model = { type = "virtio"} + source = { + network = { + network = libvirt_network.generic_net.name + } + } + mac = { address = format("52:54:00:11:22:%02d", count.index + 10) } + }, + { + model = { type = "virtio"} + source = { + network = { + network = libvirt_network.external_net.name + } + } + mac = { address = format("52:54:00:33:44:%02d", count.index + 10) } + } + ] + consoles = [ + { target = { type = "serial" } }, + { target = { type = "virtio", port = "1" }} + ] + graphics = [ + { + type = "vnc" + vnc = { + autoport = true + listen = "127.0.0.1" + } + }, + ] + } +} + +data "external" "remote_command" { + depends_on = [ + libvirt_domain.maas_controller + ] + program = ["bash", "-c", <<-EOF + # Block until the api.key file shows up + API_KEY_FILE=/tmp/maas-api.key + ssh -i ${var.ssh_private_key_path} ubuntu@${local.maas_controller_ip_addr} 'touch /home/ubuntu/api.key; until [ -s /home/ubuntu/api.key ]; do sleep 5;done; cat /home/ubuntu/api.key' > $API_KEY_FILE + cat $API_KEY_FILE 2>&1 | jq -R '{apikey: .}' 2>&1 + EOF + ] +} diff --git a/testing/units/virtualnodes/outputs.tf b/testing/units/virtualnodes/outputs.tf new file mode 100644 index 000000000..294d312e8 --- /dev/null +++ b/testing/units/virtualnodes/outputs.tf @@ -0,0 +1,30 @@ +output "maas_controller_ip_address" { + description = "MAAS Controller IP Address." + value = local.maas_controller_ip_addr + depends_on = [libvirt_domain.maas_controller] +} + +output "nodes" { + description = "List of (virtual) nodes" + value = [ + for node in libvirt_domain.node : { + name = node.name + mac_address = node.devices.interfaces[0].mac.address + osd_disks = [ + for disk in node.devices.disks : { + serial = disk.serial + size = var.node_osd_disk_size + dev = disk.target.dev + } if strcontains(disk.serial, "CEPH") + ] + } + ] + depends_on = [libvirt_domain.node] +} + +output "maas_api_key" { + description = "MAAS Admin API Key" + value = data.external.remote_command.result.apikey + sensitive = true + depends_on = [libvirt_domain.maas_controller] +} diff --git a/testing/units/virtualnodes/templates/maas_controller.cloudinit.cfg b/testing/units/virtualnodes/templates/maas_controller.cloudinit.cfg new file mode 100644 index 000000000..e175e1a15 --- /dev/null +++ b/testing/units/virtualnodes/templates/maas_controller.cloudinit.cfg @@ -0,0 +1,81 @@ +#cloud-config +ssh_pwauth: True +ssh_authorized_keys: + - ${ssh_public_key} +chpasswd: + list: | + ubuntu:linux + expire: False + +hostname: ${maas_hostname} +create_hostname_file: true +locale: C.UTF-8 +package_update: true +package_upgrade: true +packages: + - openssh-server + - snapd + - jq + - git + - python3-pip + - python3-venv + - moreutils +write_files: + - content: | + #!/bin/bash + set -x + sudo snap install maas --channel 3.6/stable + sudo snap install maas-test-db --channel 3.6/stable + + sudo -E maas init region+rack --database-uri maas-test-db:/// --maas-url http://${address}:5240/MAAS + sleep 25 + sudo maas createadmin --username admin --password admin --email admin + sudo maas apikey --username admin > /home/ubuntu/api.key + profile="admin" + maas login $profile http://${address}:5240/MAAS $(cat /home/ubuntu/api.key) + # # Configure MAAS DNS forwarding to OpenStack network DNS server + # maas $profile maas set-config name=upstream_dns value="${dns_server}" + # # TODO: Find how to properly configure DNSSEC + # maas $profile maas set-config name=dnssec_validation value=no + # # Download noble image + # # maas $profile boot-source-selections create 1 os="ubuntu" release=noble arches=amd64 subarches="*" labels="*" + # # To make 'openstack console log show' work + # maas $profile maas set-config name=kernel_opts value="console=ttyS0 console=tty0" + # # Skip setup screen + # maas admin maas set-config name=completed_intro value=true + + # PRIMARY_RACK=$(maas $profile rack-controllers read | jq -r ".[] | .system_id") + + # networks=(${networks}) + # for network in "$${networks[@]}" ; do + # SUBNET=$${network#*:} + # SPACE=$${network%%:*} + # space=space-$SPACE + # FABRIC_ID=$(maas admin subnet read "$SUBNET" | jq -r ".vlan.fabric_id") + # VLAN_TAG=$(maas admin subnet read "$SUBNET" | jq -r ".vlan.vid") + # maas $profile spaces create name=$space + # maas $profile fabric update $FABRIC_ID name=fabric-$SPACE + # if [ "$SPACE" != "pxe-boot" ]; then + # maas $profile vlan update $FABRIC_ID $VLAN_TAG space=$space primary_rack="$PRIMARY_RACK" + # continue + # fi + # DHCP_START=$(python3 -c "import ipaddress as ip;print(list(ip.ip_network('$SUBNET').hosts())[-10])") + # DHCP_END=$(python3 -c "import ipaddress as ip;print(list(ip.ip_network('$SUBNET').hosts())[-1])") + # maas $profile ipranges create type=dynamic start_ip="$DHCP_START" end_ip="$DHCP_END" + # maas $profile vlan update $FABRIC_ID $VLAN_TAG space=$space dhcp_on=True primary_rack="$PRIMARY_RACK" + # done + + # Wait for boot os to be downloaded + while [ "x$(maas $profile boot-resources is-importing | cat)" != "xfalse" ]; + do + echo "$(date) - waiting for boot resources to import" + sleep 15 + done + # # Temporary workaround to get MAAS DHCP service started + # sudo snap restart maas + touch /tmp/.i_am_done + path: /usr/local/share/init.sh + owner: root:root + permissions: "0555" +runcmd: + - sudo -i -u ubuntu /usr/local/share/init.sh diff --git a/testing/units/virtualnodes/templates/maas_controller.netplan.yaml b/testing/units/virtualnodes/templates/maas_controller.netplan.yaml new file mode 100644 index 000000000..281999461 --- /dev/null +++ b/testing/units/virtualnodes/templates/maas_controller.netplan.yaml @@ -0,0 +1,15 @@ +network: + version: 2 + renderer: networkd + ethernets: + virtio-nic: + match: + macaddress: "${mac_address}" + addresses: + - "${ip_address}/24" + routes: + - to: 0.0.0.0/0 + via: 172.16.1.1 + nameservers: + addresses: + - "${dns_server}" diff --git a/testing/units/virtualnodes/terragrunt.hcl b/testing/units/virtualnodes/terragrunt.hcl new file mode 100644 index 000000000..230ab3ebb --- /dev/null +++ b/testing/units/virtualnodes/terragrunt.hcl @@ -0,0 +1,30 @@ +include "stack" { + path = find_in_parent_folders("stack.hcl") + + # NOTE(freyes): expose shouldn't be needed to access `locals` in `stack.hcl`, + # although without it `stack` is `null`. + expose = true +} + +terraform { + source = "./" + before_hook "create_directories" { + commands = ["apply", "plan"] + execute = ["bash", "-c", <<-EOT + chmod 755 $HOME + mkdir -p $HOME/sunbeam_storage + chmod 775 $HOME/sunbeam_storage + chgrp libvirt $HOME/sunbeam_storage + echo "Directory $HOME/sunbeam_storage created" + ls -ld $HOME/sunbeam_storage + EOT + ] + } +} + +inputs = merge( + include.stack.locals.stack_config, + { + storage_pool_path = format("%s/sunbeam_storage", get_env("HOME")) + } +) diff --git a/testing/units/virtualnodes/variables.tf b/testing/units/virtualnodes/variables.tf new file mode 100644 index 000000000..1625ca496 --- /dev/null +++ b/testing/units/virtualnodes/variables.tf @@ -0,0 +1,95 @@ +variable "libvirt_uri" { + description = "Libvirt connection URI" + type = string + default = "qemu:///system" +} + +variable "nodes_count" { + type = number + default = 6 +} + +variable "node_mem" { + type = string + default = "16777216" # 16GiB +} + +variable "node_vcpu" { + type = number + default = 4 +} + +variable "maas_controller_mem" { + type = string + default = "8388608" # 8GiB +} + +variable "maas_controller_vcpu" { + type = number + default = 4 +} + +variable "maas_hostname" { + description = "MAAS Controller hostname" + type = string + default = "maas-controller" +} + +variable "node_rootfs_size" { + description = "Node rootfs disk size (in Gigabytes)" + type = number + default = 20 +} + +variable "node_osd_disk_size" { + description = "Node secondary disk size (in Gigabytes)" + type = number + default = 20 +} + +variable "generic_net_domain" { + description = "" + type = string + default = "generic.maas" +} + +variable "external_net_domain" { + description = "" + type = string + default = "external.maas" +} + +variable "ssh_public_key_path" { + description = "Path to the SSH public key to use in deployed nodes" + type = string + default = "~/.ssh/id_ecdsa.pub" +} + +variable "ssh_private_key_path" { + description = "Path to the SSH private key to use in deployed nodes" + type = string + default = "~/.ssh/id_ecdsa" +} + +variable "storage_pool_path" { + description = "Path to the storage pool used by the virtual nodes" + type = string +} + +variable "upstream_dns_server" { + description = "upstream dns server to use in MAAS" + type = string + default = "8.8.8.8" +} + +variable "maas_controller_mac_address" { + description = "MAC address to assign to the maas controller nic in the management network" + type = string + default = "52:54:00:11:11:01" +} + +variable "maas_controller_rootfs_size" { + description = "MAAS Controller rootfs disk size (in Gigabytes)" + type = number + default = 20 +}