From 075fc82bde927c26b20493b7d44c269e35398802 Mon Sep 17 00:00:00 2001 From: Michal Opala Date: Thu, 12 Mar 2026 10:43:43 +0100 Subject: [PATCH 1/5] M #-: Add Open vSwitch role - Handle both kernel and DPDK networking in OVS - Allow for creation of arbitrary number of bonds and bridges - Assert that 'ovs' structure is valid before applying any changes - Fix up checksum offloading for Debian-like distros - Make OVS configuration persistent (across reboots) Signed-off-by: Michal Opala --- playbooks/site.yml | 9 + roles/openvswitch/README.md | 99 +++++++ roles/openvswitch/defaults/main.yml | 41 +++ roles/openvswitch/meta/main.yml | 6 + roles/openvswitch/tasks/main.yml | 259 ++++++++++++++++++ .../templates/opennebula-ovs.service.jinja | 30 ++ .../templates/opennebula-ovs.sh.jinja | 156 +++++++++++ 7 files changed, 600 insertions(+) create mode 100644 roles/openvswitch/README.md create mode 100644 roles/openvswitch/defaults/main.yml create mode 100644 roles/openvswitch/meta/main.yml create mode 100644 roles/openvswitch/tasks/main.yml create mode 100644 roles/openvswitch/templates/opennebula-ovs.service.jinja create mode 100644 roles/openvswitch/templates/opennebula-ovs.sh.jinja diff --git a/playbooks/site.yml b/playbooks/site.yml index afc0cf9e..a419de73 100644 --- a/playbooks/site.yml +++ b/playbooks/site.yml @@ -15,6 +15,15 @@ - role: repository tags: [preinstall, prometheus] +- hosts: + - "{{ frontend_group | d('frontend') }}" + - "{{ node_group | d('node') }}" + collections: + - opennebula.deploy + roles: + - role: openvswitch + tags: [network, openvswitch] + - hosts: "{{ frontend_group | d('frontend') }}" tags: [frontend, stage1] collections: diff --git a/roles/openvswitch/README.md b/roles/openvswitch/README.md new file mode 100644 index 00000000..d9d84609 --- /dev/null +++ b/roles/openvswitch/README.md @@ -0,0 +1,99 @@ +Role: opennebula.deploy.openvswitch +=================================== + +A role that **replaces** OS default networking with OVS/DPDK. + +Requirements +------------ + +N/A + +Role Variables +-------------- + +| Name | Type | Default | Example | Description | +|----------------|--------|-----------------------|---------------|------------------------------------------| +| `ovs` | `dict` | (check role defaults) | (check below) | OVS/DPDK config. | +| `ovs_packages` | `dict` | (check role defaults) | | OVS/DPDK packages grouped per OS distro. | + +Dependencies +------------ + +N/A + +Example Playbook +---------------- + + - hosts: node + vars: + kernel_ok_to_reboot: true + kernel_params: + - default_hugepagesz: "1G" + - hugepagesz: "1G" + - hugepages: 3 + - intel_iommu: "on" + kernel_modules: + - load: vfio-pci + - load: vfio_iommu_type1 + options: ["allow_unsafe_interrupts=1"] # for virtio-net-pci devices + opennebula_repo_pre_enable: + AlmaLinux: + extra_rpms: + '10': [centos-release-nfv-openvswitch] + config_manager: + '10': [crb, epel, highavailability, centos-nfv-openvswitch] + RedHat: + subscription_manager: + '9': + - codeready-builder-for-rhel-9-x86_64-rpms + - rhel-9-for-x86_64-highavailability-rpms + - fast-datapath-for-rhel-9-x86_64-rpms + ovs: + set: + - other_config:dpdk-init: 'true' + - other_config:dpdk-socket-mem: '1024,0' + iface: + dpdk-p0: + set: + - type: dpdkvhostuserclient + - options:vhost-server-path: /var/tmp/dpdk-p0 + dpdk-p1: + set: + - type: dpdk + - options:dpdk-devargs: '0000:02:00.0' + dpdk-p2: + set: + - type: dpdk + - options:dpdk-devargs: '0000:03:00.0' + eth3: {} # non-DPDK device + bond: + bond0: + ifaces: [dpdk-p1, dpdk-p2] + mode: active-backup + br: + ovsbr0: + ports: [dpdk-p0, bond0] + set: + - datapath_type: netdev + addrs: + - cidr: "{{ ansible_default_ipv4.address ~ '/' ~ ansible_default_ipv4.prefix }}" + metric: 400 + gw: "{{ ansible_default_ipv4.gateway }}" + dns: ["{{ ansible_default_ipv4.gateway }}"] + ovsbr1: # non-DPDK bridge + ports: [eth3] + roles: + - role: opennebula.deploy.helper.facts + - role: opennebula.deploy.helper.kernel + - role: opennebula.deploy.repository + - role: opennebula.deploy.openvswitch + +License +------- + +Apache-2.0 + +Author Information +------------------ + +[OpenNebula Systems](https://opennebula.io/) diff --git a/roles/openvswitch/defaults/main.yml b/roles/openvswitch/defaults/main.yml new file mode 100644 index 00000000..78000a73 --- /dev/null +++ b/roles/openvswitch/defaults/main.yml @@ -0,0 +1,41 @@ +--- +ovs_defaults: + iface: {} + bond: {} + br: {} + +ovs_packages: + AlmaLinux: + - ethtool + - iproute + - iputils + - openvswitch3.5 + Debian: + - ethtool + - iproute2 + - iputils-arping + - openvswitch-switch + RedHat: + - ethtool + - iproute + - iputils + - openvswitch3.6 + +ovs_packages_dpdk: + AlmaLinux: + - dpdk-tools + - ethtool + - iproute + - iputils + - openvswitch3.5 + Debian: + - ethtool + - iproute2 + - iputils-arping + - openvswitch-switch-dpdk + RedHat: + - dpdk-tools + - ethtool + - iproute + - iputils + - openvswitch3.6 diff --git a/roles/openvswitch/meta/main.yml b/roles/openvswitch/meta/main.yml new file mode 100644 index 00000000..e233e79d --- /dev/null +++ b/roles/openvswitch/meta/main.yml @@ -0,0 +1,6 @@ +--- +collections: + - opennebula.deploy + +dependencies: + - role: opennebula.deploy.common diff --git a/roles/openvswitch/tasks/main.yml b/roles/openvswitch/tasks/main.yml new file mode 100644 index 00000000..eaa17537 --- /dev/null +++ b/roles/openvswitch/tasks/main.yml @@ -0,0 +1,259 @@ +--- +- name: Compute facts (OVS) + ansible.builtin.set_fact: + ovs: "{{ ovs_defaults | combine(ovs | d({}), recursive=true) }}" + +- when: ((ovs.iface | count) + (ovs.bond | count) + (ovs.br | count)) > 0 + vars: + # helpers + _pci_addr_regex: >- + ^[0-9a-fA-F]{4}[:][0-9a-fA-F]{2}[:][0-9a-fA-F]{2}[.][0-9a-fA-F]{1,2}$ + # general + _dpdk_enabled: >- + {{ ovs.set | d([]) + | selectattr('other_config:dpdk-init', 'defined') + | selectattr('other_config:dpdk-init', 'in', ['true']) + | count > 0 }} + # iface + _dpdk_iface: >- + {%- set output = [] -%} + {%- for k, v in ovs.iface.items() -%} + {%- for u in v.set | d([]) | selectattr('type', 'defined') -%} + {%- if u.type.startswith('dpdk') -%} + {{- output.append([k, v]) -}} + {%- endif -%} + {%- endfor -%} + {%- endfor -%} + {{- dict(output) -}} + _dpdk_pci_addrs_raw: >- + {%- set output = [] -%} + {%- for k, v in _dpdk_iface.items() -%} + {%- for u in v.set | selectattr('options:dpdk-devargs', 'defined') -%} + {%- if u['options:dpdk-devargs'] | string | regex_search(_pci_addr_regex) -%} + {{- output.append(u['options:dpdk-devargs']) -}} + {%- else -%} + {{- output.append(None) -}} + {%- endif -%} + {%- endfor -%} + {%- endfor -%} + {{- output -}} + _dpdk_pci_addrs: >- + {{ _dpdk_pci_addrs_raw | select | unique }} + _iface: >- + {{ ovs.iface | dict2items + | rejectattr('key', 'in', _dpdk_iface.keys()) + | items2dict }} + _pci_addrs: >- + {{ command_udevadm_pci_addresses.stdout_lines | d([]) + | select + | map('regex_replace', '^pci-', '') }} + # bond + _dpdk_bond: >- + {{ ovs.bond | dict2items + | selectattr('value.ifaces', 'defined') + | selectattr('value.ifaces', 'subset', _dpdk_iface.keys()) + | items2dict }} + _bond: >- + {{ ovs.bond | dict2items + | selectattr('value.ifaces', 'defined') + | selectattr('value.ifaces', 'subset', _iface.keys()) + | items2dict }} + _all_bond_ifaces: >- + {{ ovs.bond | dict2items + | selectattr('value.ifaces', 'defined') + | map(attribute='value.ifaces') + | flatten }} + # br + _dpdk_br: >- + {%- set output = [] -%} + {%- for k, v in ovs.br.items() -%} + {%- for u in v.set | d([]) | selectattr('datapath_type', 'defined') -%} + {%- if u.datapath_type in ['netdev'] -%} + {{- output.append([k, v]) -}} + {%- endif -%} + {%- endfor -%} + {%- endfor -%} + {{- dict(output) -}} + _br: >- + {{ ovs.br | dict2items + | rejectattr('key', 'in', _dpdk_br.keys()) + | items2dict }} + _all_br_ports: >- + {{ ovs.br | dict2items + | selectattr('value.ports', 'defined') + | map(attribute='value.ports') + | flatten }} + block: + - name: Assert DPDK is enabled when DPDK resources are defined + ansible.builtin.assert: + that: (_dpdk_iface | count == 0) + or + (_dpdk_enabled is true) + fail_msg: Please enable DPDK (define other_config:dpdk-init) or remove DPDK resources from OVS config. + + - name: Assert all provided DPDK PCI addresses are valid and unique + ansible.builtin.assert: + that: _dpdk_pci_addrs_raw == _dpdk_pci_addrs # no invalid, no duplicated + fail_msg: Please remove invalid / duplicated DPDK PCI addresses from OVS config. + + - name: Query lspci if DPDK PCI addresses do exist + ansible.builtin.shell: + cmd: | + set -o errexit + {% for v in _dpdk_pci_addrs %} + STDOUT="$(lspci -vmm -nkD -s '{{ v }}')" + if [[ -n "$STDOUT" ]]; then + echo "$STDOUT" + echo + else + echo "Could not find '{{ v }}'" >&2 + exit 1 + fi + {% endfor %} + executable: /bin/bash + changed_when: false + + - name: Query udev for PCI addresses + ansible.builtin.command: + cmd: "udevadm info --query=property --property=ID_PATH --value {{ _paths | join(' ') }}" + when: _paths | count > 0 + vars: + _paths: >- + {{ _iface.keys() | map('regex_replace', '^(.*)$', "-p '/sys/class/net/\g<1>'") }} + register: command_udevadm_pci_addresses + changed_when: false + + - name: Assert DPDK and non-DPDK devices do not share PCI addresses + ansible.builtin.assert: + that: _dpdk_pci_addrs | intersect(_pci_addrs) | count == 0 + fail_msg: Please remove conflicting (PCI address) DPDK vs non-DPDK device definitions from OVS config. + + - name: Assert all bond 'ifaces' are unique subset of ovs.iface devices + ansible.builtin.assert: + that: + - _all_bond_ifaces == _all_bond_ifaces | unique # no duplicated + - _all_bond_ifaces is subset(ovs.iface.keys()) + fail_msg: Please ensure ALL bond 'ifaces' are declared in ovs.iface and uniquely distributed among bond resources. + + - name: Assert each bond resource contains at least 2 'ifaces' + ansible.builtin.assert: + that: + - 0 not in _ifaces_counts + - 1 not in _ifaces_counts + fail_msg: Please ensure each bond resource contains at least 2 'ifaces'. + vars: + _ifaces_counts: >- + {{ ovs.bond | dict2items + | selectattr('value.ifaces', 'defined') + | map(attribute='value.ifaces') + | map('count') + | unique }} + + - name: Assert each bond 'ifaces' are not of mixed DPDK / non-DPDK types + ansible.builtin.assert: + that: + - _bond_keys == _bond_keys | unique # there are no mixed 'ifaces' + - _bond_keys == _all_bond_keys # every bond resource has some valid-ish 'ifaces' + fail_msg: Please ensure each declared bond resource has 'ifaces' of either DPDK or non-DPDK types (never both). + vars: + _bond_keys: >- + {{ _dpdk_bond.keys() | list + _bond.keys() | list }} + _all_bond_keys: >- + {{ ovs.bond.keys() }} + + - name: Assert all br 'ports' are unique subset of ovs.iface and ovs.bond devices + ansible.builtin.assert: + that: + - _all_br_ports == _all_br_ports | unique # no duplicated + - _all_br_ports is subset(ovs.iface.keys() | list + ovs.bond.keys() | list) + fail_msg: Please ensure ALL br 'ports' are declared in ovs.iface and ovs.bond and uniquely distributed among br resources. + + - name: Assert each non-DPDK br 'ports' do not contain DPDK resources + ansible.builtin.assert: + that: _keys | intersect(_dpdk_keys) | count == 0 + fail_msg: Please ensure no DPDK resource is inserted into non-DPDK br resources. + vars: + _keys: >- + {{ _br | dict2items + | map(attribute='value.ports') + | flatten }} + _dpdk_keys: >- + {{ _dpdk_bond.keys() | list + _dpdk_iface.keys() | list }} + + - name: Assert addrs/gw/dns are not declared for br 'ports' and bond 'ifaces' + ansible.builtin.assert: + that: _matched | count == 0 + fail_msg: Please remove ALL addrs/gw/dns attributes from br 'ports' and bond 'ifaces'. + vars: + _keys: >- + {{ (_all_bond_ifaces + _all_br_ports) | difference(ovs.bond) }} + _matched: >- + {{ _keys | map('extract', ovs.iface) + | map('dict2items') + | flatten + | selectattr('key', 'in', ['addrs', 'gw', 'dns']) }} + + - name: Install OVS packages + ansible.builtin.package: + name: "{{ _specific[ansible_distribution] | d(_specific[ansible_os_family]) }}" + vars: + _specific: "{{ _dpdk_enabled | ternary(ovs_packages_dpdk, ovs_packages) }}" + register: package + until: package is success + retries: 12 + delay: 5 + + - when: + - ansible_os_family in ['Debian'] + - _dpdk_enabled is true + block: + - name: Use DPDK version of ovs-vswitchd + community.general.alternatives: + name: ovs-vswitchd + path: /usr/lib/openvswitch-switch-dpdk/ovs-vswitchd-dpdk + register: alternatives_ovs_vswitchd + + - name: Enable / (Re)Start OVS (NOW) + ansible.builtin.systemd_service: + name: "{{ _specific[ansible_os_family] }}" + state: >- + {{ 'restarted' if _changed else 'started' }} + enabled: true + vars: + _specific: + Debian: openvswitch-switch.service + RedHat: openvswitch.service + _changed: >- + {{ (alternatives_ovs_vswitchd is defined) + and + (alternatives_ovs_vswitchd is changed) }} + + - name: Install OVS-related scripts + ansible.builtin.template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: "{{ item.mode }}" + owner: 0 + group: 0 + loop: + - src: opennebula-ovs.sh.jinja + dest: /usr/local/sbin/opennebula-ovs.sh + mode: u=rwx,go=rx + - src: opennebula-ovs.service.jinja + dest: /etc/systemd/system/opennebula-ovs.service + mode: u=rw,go=r + register: template + + - name: Switch to OVS networking + ansible.builtin.systemd_service: + daemon_reload: "{{ item.daemon_reload | d(omit) }}" + name: "{{ item.name | d(omit) }}" + state: "{{ item.state | d(omit) }}" + enabled: "{{ item.enabled | d(omit) }}" + loop: + - daemon_reload: "{{ _changed }}" + - name: opennebula-ovs.service + state: "{{ 'restarted' if _changed else 'started' }}" + enabled: true + vars: + _changed: "{{ template is changed }}" diff --git a/roles/openvswitch/templates/opennebula-ovs.service.jinja b/roles/openvswitch/templates/opennebula-ovs.service.jinja new file mode 100644 index 00000000..c931d683 --- /dev/null +++ b/roles/openvswitch/templates/opennebula-ovs.service.jinja @@ -0,0 +1,30 @@ +[Unit] +Description=OVS Bridge Interface Network configuration +{% if ansible_os_family == 'Debian' %} +After=openvswitch-switch.service network-pre.target +Wants=network-pre.target +Before=network.target +Requires=openvswitch-switch.service +{% endif %} +{% if ansible_os_family == 'RedHat' %} +After=openvswitch.service network-pre.target +Wants=network-pre.target +Before=network.target +Requires=openvswitch.service +{% endif %} + +[Service] +Type=oneshot +RemainAfterExit=yes +EnvironmentFile= +ExecStart=/usr/local/sbin/opennebula-ovs.sh +TimeoutStartSec=120 +StandardOutput=journal +StandardError=journal +# Ensure the service runs in a clean environment +Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +# Network operations require root +User=root + +[Install] +WantedBy=multi-user.target diff --git a/roles/openvswitch/templates/opennebula-ovs.sh.jinja b/roles/openvswitch/templates/opennebula-ovs.sh.jinja new file mode 100644 index 00000000..e7f95eb9 --- /dev/null +++ b/roles/openvswitch/templates/opennebula-ovs.sh.jinja @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +set -o errexit -o pipefail + +# --- Helper functions + +log() { echo "$*" >&2; } +die() { log "ERROR: $*"; exit 1; } + +# --- Basic assertions + +type -p arping find ip jq ovs-vsctl &>/dev/null +{% if _dpdk_enabled %} +type -p dpdk-devbind.py ethtool &>/dev/null +{% endif %} + +{% for iface in _iface %} +# {{ iface }} +log 'Asserting {{ iface }} exists and is not used outside OVS' +ip --json link show dev '{{ iface }}' | jq -re '.[0].master as $m | [null, "ovs-system"] | any(. == $m)' &>/dev/null +{% endfor %} + +# --- OVS / DPDK initialization + +{% for addr in _dpdk_pci_addrs %} +# {{ addr }} +log 'Calling dpdk-devbind.py on {{ addr }}' +dpdk-devbind.py --force --bind=vfio-pci '{{ addr }}' || die 'Failed to bind PCI device {{ addr }}' +{% endfor %} + +{% for v in ovs.set | d([]) | map('dict2items') | flatten %} +log 'Setting {{ v.key }}={{ v.value }}' +ovs-vsctl --no-wait set Open_vSwitch . '{{ v.key }}={{ v.value }}' || die 'Failed to set {{ v.key }}={{ v.value }}' +{% endfor %} + +# --- Bridge / Bond creation + +{% for br in ovs.br %} +# {{ br }} +if ovs-vsctl br-exists '{{ br }}' &>/dev/null; then + log 'WARNING: Bridge {{ br }} already exists, skipping creation' +else + log 'Creating OVS bridge {{ br }}' + ovs-vsctl add-br '{{ br }}' \ +{% for v in ovs.br[br].set | d([]) | map('dict2items') | flatten %} + -- set bridge '{{ br }}' '{{ v.key }}={{ v.value }}' \ +{% endfor %} + || die 'Failed to create bridge {{ br }}' +fi +{% for port in ovs.br[br].ports %} +if ovs-vsctl port-to-br '{{ port }}' &>/dev/null; then + log 'WARNING: Port {{ port }} already exists, skipping creation' +else +{% if port in ovs.iface %} + log 'Adding port {{ port }} to {{ br }}' + ovs-vsctl add-port '{{ br }}' '{{ port }}' \ +{% for v in ovs.iface[port].set | d([]) | map('dict2items') | flatten %} + -- set Interface '{{ port }}' '{{ v.key }}={{ v.value }}' \ +{% endfor %} + || die 'Failed to add port {{ port }}' +{% endif %} +{% if port in ovs.bond %} + log 'Adding bond {{ port }} to {{ br }}' + ovs-vsctl add-bond '{{ br }}' '{{ port }}' \ +{% for iface in ovs.bond[port].ifaces %} + '{{ iface }}' \ +{% endfor %} +{% for v in ovs.bond[port].set | d([]) | map('dict2items') | flatten %} + '{{ v.key }}={{ v.value }}' \ +{% endfor %} +{% for iface in ovs.bond[port].ifaces %} +{% for v in ovs.iface[iface].set | d([]) | map('dict2items') | flatten %} + -- set Interface '{{ iface }}' '{{ v.key }}={{ v.value }}' \ +{% endfor %} +{% endfor %} + || die 'Failed to add bond {{ port }}' +{% endif %} +fi +{% endfor %} +{% endfor %} + +# --- Checksum offload (internal) + +{% for br in _dpdk_br %} +log 'Disabling checksum offloading for {{ br }}' +ethtool --offload '{{ br }}' tx off rx off || log 'WARNING: Failed to disable checksum offloading for {{ br }}' +{% endfor %} + +# --- Networking cleanup + +if type -p systemctl &>/dev/null && systemctl is-active --quiet NetworkManager; then + log 'Stopping and disabling NetworkManager' + systemctl disable NetworkManager || log 'WARNING: Failed to disable NetworkManager' + systemctl stop NetworkManager || log 'WARNING: Failed to stop NetworkManager' +fi + +if type -p netplan &>/dev/null && netplan status -f json | jq -re '."netplan-global-state"."online"' &>/dev/null; then + log 'Disabling Netplan' + find /etc/netplan/ -type f -name '*.yaml' -exec mv {} {}.bak \; + netplan apply || log 'WARNING: Failed to disable Netplan' +fi + +# --- IP configuration + +{% for dev, v in (_iface.items() | list + ovs.br.items() | list) %} +# {{ dev }} +log 'Flushing {{ dev }}' +ip addr flush dev '{{ dev }}' || die 'Failed to flush {{ dev }}' +{% for a in v.addrs | d([]) %} +log 'Adding IP address {{ a.cidr }} to {{ dev }}' +ip addr add '{{ a.cidr }}' dev '{{ dev }}' \ + {{ "metric '{}'".format(a.metric) if a.metric is defined else "" }} || die 'Failed to add IP address to {{ dev }}' +{% endfor %} +log 'Bringing up {{ dev }}' +ip link set dev '{{ dev }}' up || die 'Failed to bring up {{ dev }}' +{% endfor %} + +# --- Default route + +{% set gw = ((ovs.iface.values() | list + ovs.br.values() | list) | selectattr('gw', 'defined') | first).gw | d(None) %} +{% if gw is not none %} +if ip --json route show default | jq -re '. != []' &>/dev/null; then + log 'WARNING: Default route already exists' +else + log 'Adding default route via {{ gw }}' + ip route add default via '{{ gw }}' || die 'Failed to add default route {{ gw }}' +fi +{% endif %} + +# --- Nameserver configuration + +if type -p systemctl resolvectl &>/dev/null && systemctl is-active --quiet systemd-resolved; then +{% for dev, v in (ovs.iface.items() | list + ovs.br.items() | list) %} + # {{ dev }} +{% if v.dns is defined and v.dns is sequence %} + log 'Setting nameservers for {{ dev }}' + resolvectl dns '{{ dev }}' \ + {{ v.dns | map('regex_replace', '^(.*)$', "'\g<1>'") | join(' ') }} \ + || die 'Failed to setup nameservers' +{% endif %} +{% endfor %} +fi + +# --- Networking refresh + +{% for br, v in ovs.br.items() %} +# {{ br }} +{% for a in v.addrs | d([]) %} +log 'Sending gratuitous ARP for {{ a.cidr }} on {{ br }}' +arping -c 3 -A -I '{{ br }}' '{{ a.cidr.split('/') | first }}' || log 'WARNING: arping failed' +{% endfor %} +{% endfor %} + +# --- + +log 'OVS network configuration completed successfully' +exit 0 From 3c703e2962dca8c987f2877faf2405a4b69d206a Mon Sep 17 00:00:00 2001 From: Michal Opala Date: Mon, 23 Mar 2026 11:38:47 +0100 Subject: [PATCH 2/5] M #-: Add auto-cleanup of orphaned one-deploy OVS resources - Use external_ids to mark and manage only subset of OVS resources - Auto-cleanup OVS ports and bridges managed by one-deploy - Allow for interface and bridge re-configurations (limited) - Re-assembly bond resources on interface changes (fix) - Make it possible to move interfaces between different OVS bridges / bonds - Improve readability of opennebula-ovs.sh.jinja template Signed-off-by: Michal Opala --- .../templates/opennebula-ovs.sh.jinja | 157 ++++++++++++------ 1 file changed, 108 insertions(+), 49 deletions(-) diff --git a/roles/openvswitch/templates/opennebula-ovs.sh.jinja b/roles/openvswitch/templates/opennebula-ovs.sh.jinja index e7f95eb9..4d5bdad0 100644 --- a/roles/openvswitch/templates/opennebula-ovs.sh.jinja +++ b/roles/openvswitch/templates/opennebula-ovs.sh.jinja @@ -8,6 +8,7 @@ die() { log "ERROR: $*"; exit 1; } # --- Basic assertions +log 'Asserting required binaries and scripts are available' type -p arping find ip jq ovs-vsctl &>/dev/null {% if _dpdk_enabled %} type -p dpdk-devbind.py ethtool &>/dev/null @@ -32,72 +33,134 @@ log 'Setting {{ v.key }}={{ v.value }}' ovs-vsctl --no-wait set Open_vSwitch . '{{ v.key }}={{ v.value }}' || die 'Failed to set {{ v.key }}={{ v.value }}' {% endfor %} -# --- Bridge / Bond creation +# --- Bridge / Port cleanup + +declare -A _ALL_BOND_IFACES=({{ _all_bond_ifaces | map('regex_replace', '^(.*)$', '[\g<1>]=1') | join(' ') }}) + +ovs-vsctl --no-headings --columns=name find Interface 'external_ids:one-deploy-bond!=""' | while read -r IFACE; do + if [[ -n "$IFACE" ]] && [[ ! -v _ALL_BOND_IFACES[$IFACE] ]]; then + log "Deleting leftover iface $IFACE" + ovs-vsctl --with-iface del-port "$IFACE" || die "Failed to delete iface $IFACE" + fi +done + +declare -A _ALL_BR_PORTS=({{ _all_br_ports | map('regex_replace', '^(.*)$', '[\g<1>]=1') | join(' ') }}) + +ovs-vsctl --no-headings --columns=name find Port 'external_ids:one-deploy-bond=""' | while read -r PORT; do + if [[ -n "$PORT" ]] && [[ ! -v _ALL_BR_PORTS[$PORT] ]]; then + log "Deleting leftover port $PORT" + ovs-vsctl del-port "$PORT" || die "Failed to delete port $PORT" + fi +done + +declare -A _ALL_BR=({{ ovs.br.keys() | map('regex_replace', '^(.*)$', '[\g<1>]=1') | join(' ') }}) + +ovs-vsctl --no-headings --columns=name find Bridge 'external_ids:one-deploy-br!=""' | while read -r BR; do + if [[ -n "$BR" ]] && [[ ! -v _ALL_BR[$BR] ]]; then + log "Deleting leftover br $BR" + ovs-vsctl del-br "$BR" || die "Failed to delete br $BR" + fi +done + +# --- Networking cleanup + +if type -p systemctl &>/dev/null && systemctl is-active --quiet NetworkManager; then + log 'Stopping and disabling NetworkManager' + systemctl disable NetworkManager || log 'WARNING: Failed to disable NetworkManager' + systemctl stop NetworkManager || log 'WARNING: Failed to stop NetworkManager' +fi + +if type -p netplan &>/dev/null && netplan status -f json | jq -re '."netplan-global-state"."online"' &>/dev/null; then + log 'Disabling Netplan' + find /etc/netplan/ -type f -name '*.yaml' -exec mv {} {}.bak \; + netplan apply || log 'WARNING: Failed to disable Netplan' +fi + +# --- Bridge creation {% for br in ovs.br %} # {{ br }} -if ovs-vsctl br-exists '{{ br }}' &>/dev/null; then - log 'WARNING: Bridge {{ br }} already exists, skipping creation' -else - log 'Creating OVS bridge {{ br }}' - ovs-vsctl add-br '{{ br }}' \ +log 'Creating OVS bridge {{ br }}' +ovs-vsctl --may-exist add-br '{{ br }}' \ + -- set Bridge '{{ br }}' 'external_ids:one-deploy-br={{ br }}' \ {% for v in ovs.br[br].set | d([]) | map('dict2items') | flatten %} - -- set bridge '{{ br }}' '{{ v.key }}={{ v.value }}' \ + -- set Bridge '{{ br }}' '{{ v.key }}={{ v.value }}' \ {% endfor %} - || die 'Failed to create bridge {{ br }}' -fi -{% for port in ovs.br[br].ports %} -if ovs-vsctl port-to-br '{{ port }}' &>/dev/null; then - log 'WARNING: Port {{ port }} already exists, skipping creation' -else -{% if port in ovs.iface %} - log 'Adding port {{ port }} to {{ br }}' - ovs-vsctl add-port '{{ br }}' '{{ port }}' \ + || die 'Failed to configure bridge {{ br }}' +{% endfor %} + +# --- Port creation + +{% for br in ovs.br %} +# {{ br }} +{% for port in ovs.br[br].ports | select('in', ovs.iface) %} +log 'Adding port {{ port }} to {{ br }}' +ovs-vsctl --may-exist add-port '{{ br }}' '{{ port }}' \ + 'external_ids:one-deploy-br={{ br }}' \ + 'external_ids:one-deploy-bond=""' \ + -- set Interface '{{ port }}' 'external_ids:one-deploy-br={{ br }}' \ + -- set Interface '{{ port }}' 'external_ids:one-deploy-bond=""' \ {% for v in ovs.iface[port].set | d([]) | map('dict2items') | flatten %} - -- set Interface '{{ port }}' '{{ v.key }}={{ v.value }}' \ + -- set Interface '{{ port }}' '{{ v.key }}={{ v.value }}' \ {% endfor %} - || die 'Failed to add port {{ port }}' -{% endif %} -{% if port in ovs.bond %} - log 'Adding bond {{ port }} to {{ br }}' - ovs-vsctl add-bond '{{ br }}' '{{ port }}' \ + || die 'Failed to add port {{ port }}' +{% endfor %} +{% endfor %} + +# --- Bond creation + +{% for br in ovs.br %} +# {{ br }} +{% for port in ovs.br[br].ports | select('in', ovs.bond) %} +log 'Adding bond {{ port }} to {{ br }}' +ovs-vsctl --if-exists del-port '{{ port }}' -- add-bond '{{ br }}' '{{ port }}' \ {% for iface in ovs.bond[port].ifaces %} - '{{ iface }}' \ + '{{ iface }}' \ {% endfor %} + 'external_ids:one-deploy-br={{ br }}' \ + 'external_ids:one-deploy-bond={{ port }}' \ {% for v in ovs.bond[port].set | d([]) | map('dict2items') | flatten %} - '{{ v.key }}={{ v.value }}' \ + '{{ v.key }}={{ v.value }}' \ {% endfor %} {% for iface in ovs.bond[port].ifaces %} + -- set Interface '{{ iface }}' 'external_ids:one-deploy-br={{ br }}' \ + -- set Interface '{{ iface }}' 'external_ids:one-deploy-bond={{ port }}' \ {% for v in ovs.iface[iface].set | d([]) | map('dict2items') | flatten %} - -- set Interface '{{ iface }}' '{{ v.key }}={{ v.value }}' \ + -- set Interface '{{ iface }}' '{{ v.key }}={{ v.value }}' \ {% endfor %} {% endfor %} - || die 'Failed to add bond {{ port }}' -{% endif %} -fi + || die 'Failed to add bond {{ port }}' {% endfor %} {% endfor %} -# --- Checksum offload (internal) +# --- Bridge / Interface reconfiguration -{% for br in _dpdk_br %} -log 'Disabling checksum offloading for {{ br }}' -ethtool --offload '{{ br }}' tx off rx off || log 'WARNING: Failed to disable checksum offloading for {{ br }}' +{% for br in ovs.br | dict2items | selectattr('value.set', 'defined') | map(attribute='key') %} +# {{ br }} +log 'Reconfiguring OVS bridge {{ br }}' +ovs-vsctl \ +{% for v in ovs.br[br].set | d([]) | map('dict2items') | flatten %} + -- set Bridge '{{ br }}' '{{ v.key }}={{ v.value }}' \ +{% endfor %} + || die 'Failed to reconfigure {{ br }}' {% endfor %} -# --- Networking cleanup +{% for iface in ovs.iface | dict2items | selectattr('value.set', 'defined') | map(attribute='key') %} +# {{ iface }} +log 'Reconfiguring interface {{ iface }}' +ovs-vsctl \ +{% for v in ovs.iface[iface].set | d([]) | map('dict2items') | flatten %} + -- --if-exists set Interface '{{ iface }}' '{{ v.key }}={{ v.value }}' \ +{% endfor %} + || die 'Failed to reconfigure {{ iface }}' +{% endfor %} -if type -p systemctl &>/dev/null && systemctl is-active --quiet NetworkManager; then - log 'Stopping and disabling NetworkManager' - systemctl disable NetworkManager || log 'WARNING: Failed to disable NetworkManager' - systemctl stop NetworkManager || log 'WARNING: Failed to stop NetworkManager' -fi +# --- Checksum offload (kernel) -if type -p netplan &>/dev/null && netplan status -f json | jq -re '."netplan-global-state"."online"' &>/dev/null; then - log 'Disabling Netplan' - find /etc/netplan/ -type f -name '*.yaml' -exec mv {} {}.bak \; - netplan apply || log 'WARNING: Failed to disable Netplan' -fi +{% for br in _dpdk_br %} +log 'Disabling checksum offloading for {{ br }}' +ethtool --offload '{{ br }}' tx off rx off || log 'WARNING: Failed to disable checksum offloading for {{ br }}' +{% endfor %} # --- IP configuration @@ -118,12 +181,8 @@ ip link set dev '{{ dev }}' up || die 'Failed to bring up {{ dev }}' {% set gw = ((ovs.iface.values() | list + ovs.br.values() | list) | selectattr('gw', 'defined') | first).gw | d(None) %} {% if gw is not none %} -if ip --json route show default | jq -re '. != []' &>/dev/null; then - log 'WARNING: Default route already exists' -else - log 'Adding default route via {{ gw }}' - ip route add default via '{{ gw }}' || die 'Failed to add default route {{ gw }}' -fi +log 'Replacing default route to via {{ gw }}' +ip route replace default via '{{ gw }}' || die 'Failed to replace default route {{ gw }}' {% endif %} # --- Nameserver configuration From e2f47b36e3fb7fe100e21e028e7bd5d9ef53dc12 Mon Sep 17 00:00:00 2001 From: Michal Opala Date: Mon, 23 Mar 2026 15:38:21 +0100 Subject: [PATCH 3/5] M #-: Exclude internal ports from PCI queries - Allow for setting options on "internal" ports - Recognize "internal" ports automatically matching br names - Update README.md to include "internal" port example --- roles/openvswitch/README.md | 6 ++++++ roles/openvswitch/tasks/main.yml | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/roles/openvswitch/README.md b/roles/openvswitch/README.md index d9d84609..94a7392a 100644 --- a/roles/openvswitch/README.md +++ b/roles/openvswitch/README.md @@ -53,17 +53,23 @@ Example Playbook - other_config:dpdk-init: 'true' - other_config:dpdk-socket-mem: '1024,0' iface: + ovsbr0: # "internal" port + set: + - mtu_request: 1500 dpdk-p0: set: - type: dpdkvhostuserclient + - mtu_request: 9000 - options:vhost-server-path: /var/tmp/dpdk-p0 dpdk-p1: set: - type: dpdk + - mtu_request: 9000 - options:dpdk-devargs: '0000:02:00.0' dpdk-p2: set: - type: dpdk + - mtu_request: 9000 - options:dpdk-devargs: '0000:03:00.0' eth3: {} # non-DPDK device bond: diff --git a/roles/openvswitch/tasks/main.yml b/roles/openvswitch/tasks/main.yml index eaa17537..e5c3d92e 100644 --- a/roles/openvswitch/tasks/main.yml +++ b/roles/openvswitch/tasks/main.yml @@ -39,9 +39,14 @@ {{- output -}} _dpdk_pci_addrs: >- {{ _dpdk_pci_addrs_raw | select | unique }} + _internal_iface: >- + {{ ovs.iface | dict2items + | selectattr('key', 'in', ovs.br.keys()) + | items2dict }} _iface: >- {{ ovs.iface | dict2items | rejectattr('key', 'in', _dpdk_iface.keys()) + | rejectattr('key', 'in', _internal_iface.keys()) | items2dict }} _pci_addrs: >- {{ command_udevadm_pci_addresses.stdout_lines | d([]) From 6a91a50b0fe00c1b2e86f92ff197ac31d85bbef3 Mon Sep 17 00:00:00 2001 From: Michal Opala Date: Mon, 23 Mar 2026 18:15:51 +0100 Subject: [PATCH 4/5] M #-: Fix incorrect bond_mode example in README.md Signed-off-by: Michal Opala --- roles/openvswitch/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/openvswitch/README.md b/roles/openvswitch/README.md index 94a7392a..4b605b07 100644 --- a/roles/openvswitch/README.md +++ b/roles/openvswitch/README.md @@ -75,7 +75,8 @@ Example Playbook bond: bond0: ifaces: [dpdk-p1, dpdk-p2] - mode: active-backup + set: + - bond_mode: active-backup br: ovsbr0: ports: [dpdk-p0, bond0] From f29dc8f077ebb680a9e2d8a52299422f72bc2a6f Mon Sep 17 00:00:00 2001 From: Michal Opala Date: Mon, 30 Mar 2026 17:39:50 +0200 Subject: [PATCH 5/5] M #-: Add ability to reconfigure specific OVS ports - Add ovs.port dictionary - Remove all/ALL prefixes from variable names and messages Signed-off-by: Michal Opala --- roles/openvswitch/README.md | 4 +++ roles/openvswitch/defaults/main.yml | 1 + roles/openvswitch/tasks/main.yml | 28 +++++++++++-------- .../templates/opennebula-ovs.sh.jinja | 26 +++++++++++------ 4 files changed, 40 insertions(+), 19 deletions(-) diff --git a/roles/openvswitch/README.md b/roles/openvswitch/README.md index 4b605b07..cc3154c2 100644 --- a/roles/openvswitch/README.md +++ b/roles/openvswitch/README.md @@ -52,6 +52,10 @@ Example Playbook set: - other_config:dpdk-init: 'true' - other_config:dpdk-socket-mem: '1024,0' + port: + ovsbr0: # "internal" port + set: + - tag: 123 iface: ovsbr0: # "internal" port set: diff --git a/roles/openvswitch/defaults/main.yml b/roles/openvswitch/defaults/main.yml index 78000a73..b629fdc9 100644 --- a/roles/openvswitch/defaults/main.yml +++ b/roles/openvswitch/defaults/main.yml @@ -3,6 +3,7 @@ ovs_defaults: iface: {} bond: {} br: {} + port: {} ovs_packages: AlmaLinux: diff --git a/roles/openvswitch/tasks/main.yml b/roles/openvswitch/tasks/main.yml index e5c3d92e..68855aa1 100644 --- a/roles/openvswitch/tasks/main.yml +++ b/roles/openvswitch/tasks/main.yml @@ -63,7 +63,7 @@ | selectattr('value.ifaces', 'defined') | selectattr('value.ifaces', 'subset', _iface.keys()) | items2dict }} - _all_bond_ifaces: >- + _bond_ifaces: >- {{ ovs.bond | dict2items | selectattr('value.ifaces', 'defined') | map(attribute='value.ifaces') @@ -83,7 +83,7 @@ {{ ovs.br | dict2items | rejectattr('key', 'in', _dpdk_br.keys()) | items2dict }} - _all_br_ports: >- + _br_ports: >- {{ ovs.br | dict2items | selectattr('value.ports', 'defined') | map(attribute='value.ports') @@ -136,9 +136,9 @@ - name: Assert all bond 'ifaces' are unique subset of ovs.iface devices ansible.builtin.assert: that: - - _all_bond_ifaces == _all_bond_ifaces | unique # no duplicated - - _all_bond_ifaces is subset(ovs.iface.keys()) - fail_msg: Please ensure ALL bond 'ifaces' are declared in ovs.iface and uniquely distributed among bond resources. + - _bond_ifaces == _bond_ifaces | unique # no duplicated + - _bond_ifaces is subset(ovs.iface.keys()) + fail_msg: Please ensure bond 'ifaces' are declared in ovs.iface and uniquely distributed among bond resources. - name: Assert each bond resource contains at least 2 'ifaces' ansible.builtin.assert: @@ -166,12 +166,18 @@ _all_bond_keys: >- {{ ovs.bond.keys() }} - - name: Assert all br 'ports' are unique subset of ovs.iface and ovs.bond devices + - name: Assert ovs.port is a subset of available ports (including internal) ansible.builtin.assert: that: - - _all_br_ports == _all_br_ports | unique # no duplicated - - _all_br_ports is subset(ovs.iface.keys() | list + ovs.bond.keys() | list) - fail_msg: Please ensure ALL br 'ports' are declared in ovs.iface and ovs.bond and uniquely distributed among br resources. + - ovs.port.keys() is subset(ovs.br.keys() | list + _br_ports) + fail_msg: Please ensure ovs.port is a subset of available ports. + + - name: Assert br 'ports' are unique subset of ovs.bond + ovs.iface resources + ansible.builtin.assert: + that: + - _br_ports == _br_ports | unique # no duplicated + - _br_ports is subset(ovs.bond.keys() | list + ovs.iface.keys() | list) + fail_msg: Please ensure br 'ports' are declared in ovs.bond and ovs.iface and uniquely distributed among br resources. - name: Assert each non-DPDK br 'ports' do not contain DPDK resources ansible.builtin.assert: @@ -188,10 +194,10 @@ - name: Assert addrs/gw/dns are not declared for br 'ports' and bond 'ifaces' ansible.builtin.assert: that: _matched | count == 0 - fail_msg: Please remove ALL addrs/gw/dns attributes from br 'ports' and bond 'ifaces'. + fail_msg: Please remove addrs/gw/dns attributes from br 'ports' and bond 'ifaces'. vars: _keys: >- - {{ (_all_bond_ifaces + _all_br_ports) | difference(ovs.bond) }} + {{ (_bond_ifaces + _br_ports) | difference(ovs.bond) }} _matched: >- {{ _keys | map('extract', ovs.iface) | map('dict2items') diff --git a/roles/openvswitch/templates/opennebula-ovs.sh.jinja b/roles/openvswitch/templates/opennebula-ovs.sh.jinja index 4d5bdad0..bdb2f0e7 100644 --- a/roles/openvswitch/templates/opennebula-ovs.sh.jinja +++ b/roles/openvswitch/templates/opennebula-ovs.sh.jinja @@ -35,28 +35,28 @@ ovs-vsctl --no-wait set Open_vSwitch . '{{ v.key }}={{ v.value }}' || die 'Faile # --- Bridge / Port cleanup -declare -A _ALL_BOND_IFACES=({{ _all_bond_ifaces | map('regex_replace', '^(.*)$', '[\g<1>]=1') | join(' ') }}) +declare -A _BOND_IFACES=({{ _bond_ifaces | map('regex_replace', '^(.*)$', '[\g<1>]=1') | join(' ') }}) ovs-vsctl --no-headings --columns=name find Interface 'external_ids:one-deploy-bond!=""' | while read -r IFACE; do - if [[ -n "$IFACE" ]] && [[ ! -v _ALL_BOND_IFACES[$IFACE] ]]; then + if [[ -n "$IFACE" ]] && [[ ! -v _BOND_IFACES[$IFACE] ]]; then log "Deleting leftover iface $IFACE" ovs-vsctl --with-iface del-port "$IFACE" || die "Failed to delete iface $IFACE" fi done -declare -A _ALL_BR_PORTS=({{ _all_br_ports | map('regex_replace', '^(.*)$', '[\g<1>]=1') | join(' ') }}) +declare -A _BR_PORTS=({{ _br_ports | map('regex_replace', '^(.*)$', '[\g<1>]=1') | join(' ') }}) ovs-vsctl --no-headings --columns=name find Port 'external_ids:one-deploy-bond=""' | while read -r PORT; do - if [[ -n "$PORT" ]] && [[ ! -v _ALL_BR_PORTS[$PORT] ]]; then + if [[ -n "$PORT" ]] && [[ ! -v _BR_PORTS[$PORT] ]]; then log "Deleting leftover port $PORT" ovs-vsctl del-port "$PORT" || die "Failed to delete port $PORT" fi done -declare -A _ALL_BR=({{ ovs.br.keys() | map('regex_replace', '^(.*)$', '[\g<1>]=1') | join(' ') }}) +declare -A _BR=({{ ovs.br.keys() | map('regex_replace', '^(.*)$', '[\g<1>]=1') | join(' ') }}) ovs-vsctl --no-headings --columns=name find Bridge 'external_ids:one-deploy-br!=""' | while read -r BR; do - if [[ -n "$BR" ]] && [[ ! -v _ALL_BR[$BR] ]]; then + if [[ -n "$BR" ]] && [[ ! -v _BR[$BR] ]]; then log "Deleting leftover br $BR" ovs-vsctl del-br "$BR" || die "Failed to delete br $BR" fi @@ -133,18 +133,28 @@ ovs-vsctl --if-exists del-port '{{ port }}' -- add-bond '{{ br }}' '{{ port }}' {% endfor %} {% endfor %} -# --- Bridge / Interface reconfiguration +# --- Bridge / Port / Interface reconfiguration {% for br in ovs.br | dict2items | selectattr('value.set', 'defined') | map(attribute='key') %} # {{ br }} log 'Reconfiguring OVS bridge {{ br }}' ovs-vsctl \ {% for v in ovs.br[br].set | d([]) | map('dict2items') | flatten %} - -- set Bridge '{{ br }}' '{{ v.key }}={{ v.value }}' \ + -- --if-exists set Bridge '{{ br }}' '{{ v.key }}={{ v.value }}' \ {% endfor %} || die 'Failed to reconfigure {{ br }}' {% endfor %} +{% for port in ovs.port | dict2items | selectattr('value.set', 'defined') | map(attribute='key') %} +# {{ port }} +log 'Reconfiguring port {{ port }}' +ovs-vsctl \ +{% for v in ovs.port[port].set | d([]) | map('dict2items') | flatten %} + -- --if-exists set Port '{{ port }}' '{{ v.key }}={{ v.value }}' \ +{% endfor %} + || die 'Failed to reconfigure {{ port }}' +{% endfor %} + {% for iface in ovs.iface | dict2items | selectattr('value.set', 'defined') | map(attribute='key') %} # {{ iface }} log 'Reconfiguring interface {{ iface }}'