KVM: add configurable MAC/IP script hook for static ARP/NDP and routes (#13495)

* KVM: add configurable MAC/IP script hook for static ARP/NDP and routes

Introduces a new agent.properties option `vm.network.macip.static`
(false by default) that makes BridgeVifDriver invoke on modifymacip.sh
on every NIC plug (VM start) and unplug (VM stop).

This is very useful in EVPN+VXLAN environments as it can reduce BUM
traffic. By setting static ARP/NDP entries bridges can be configured
using 'neigh_suppress on' as the ARP/NDP entries are already set
statically by CloudStack.

Setting 'neigh_suppress on' requires a manual change in the modifyvxlan.sh
script as this is not the default behavior.

* vxlan: In EVPN mode, disable ARP/NDP learning

FRR populates the FDB via BGP EVPN, so kernel data-plane learning is
redundant and counterproductive.

Static ARP (IPv4) and NDP (IPv6) entries are added on startup of the
Instance and remove on shutdown.

FRR populates the FDB/neighbor table via control plane, and neigh_suppress
tells the kernel bridge to use that information instead of flooding.

This will vastly reduce BUM traffic with static ARP/NDP entries.

Author: wido

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -937,6 +937,21 @@ public Property<Integer> getWorkers() {
      * */
     public static final Property<Integer> INCREMENTAL_SNAPSHOT_RETRY_REBASE_WAIT = new Property<>("incremental.snapshot.retry.rebase.wait", 60);
 
+    /**
+     * When set to <code>true</code>, executes <code>modifymacip.sh</code> (resolved via the
+     * network scripts directory) on VM NIC plug (VM start) and unplug (VM stop) to manage static
+     * ARP/NDP entries and host routes for VM interfaces.<br>
+     * The script is invoked with:<br>
+     * &nbsp;&nbsp;add:    <code>-o add -b &lt;bridge&gt; -m &lt;mac&gt; [-4 &lt;ipv4&gt;] [-6 &lt;ipv6&gt;]</code><br>
+     * &nbsp;&nbsp;delete: <code>-o delete -b &lt;bridge&gt; -m &lt;mac&gt;</code><br>
+     * A bundled reference implementation is available at
+     * <code>scripts/vm/network/vnet/modifymacip.sh</code>.<br>
+     * Set to <code>false</code> or leave unset to disable this feature.<br>
+     * Data type: Boolean.<br>
+     * Default value: <code>false</code>
+     */
+    public static final Property<Boolean> VM_NETWORK_MACIP_STATIC = new Property<>("vm.network.macip.static", false, Boolean.class);
+
 
     public static class Property <T>{
         private String name;

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java

@@ -48,6 +48,7 @@ public class BridgeVifDriver extends VifDriverBase {
     private final Object _vnetBridgeMonitor = new Object();
     private String _modifyVlanPath;
     private String _modifyVxlanPath;
+    private String _macIpScriptPath;
     private String _controlCidr = NetUtils.getLinkLocalCIDR();
     private Long libvirtVersion;
 
@@ -83,6 +84,14 @@ public void configure(Map<String, Object> params) throws ConfigurationException
             throw new ConfigurationException("Unable to find " + vxlanScript);
         }
 
+        if (Boolean.TRUE.equals(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VM_NETWORK_MACIP_STATIC))) {
+            _macIpScriptPath = Script.findScript(networkScriptsDir, "modifymacip.sh");
+            if (_macIpScriptPath == null) {
+                throw new ConfigurationException("Unable to find modifymacip.sh");
+            }
+            logger.info("VM network MAC/IP static script configured: {}", _macIpScriptPath);
+        }
+
         libvirtVersion = (Long) params.get("libvirtVersion");
         if (libvirtVersion == null) {
             libvirtVersion = 0L;
@@ -279,11 +288,14 @@ public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicA
         }
         intf.setLinkStateUp(nic.isEnabled());
 
+        executeMacIpScript(intf.getBrName(), nic.getMac(), nic.getIp(), nic.getIp6Address(), nic.getNicSecIps());
+
         return intf;
     }
 
     @Override
     public void unplug(LibvirtVMDef.InterfaceDef iface, boolean deleteBr) {
+        executeMacIpScript(iface.getBrName(), iface.getMacAddress());
         deleteVnetBr(iface.getBrName(), deleteBr);
     }
 
@@ -403,6 +415,60 @@ private void deleteVnetBr(String brName, boolean deleteBr) {
         }
     }
 
+    private void executeMacIpScript(String brName, String mac) {
+        if (_macIpScriptPath == null || mac == null || brName == null) {
+            return;
+        }
+        try {
+            final Script command = new Script(_macIpScriptPath, _timeout, logger);
+            command.add("-o", "delete");
+            command.add("-b", brName);
+            command.add("-m", mac);
+            final String result = command.execute();
+            if (result != null) {
+                logger.warn("MAC/IP script returned error for delete on {}: {}", mac, result);
+            }
+        } catch (Exception e) {
+            // Managing host neighbour/route entries is best-effort and must never break VM lifecycle operations
+            logger.warn("Failed to run MAC/IP script for delete on {} ({})", mac, brName, e);
+        }
+    }
+
+    private void executeMacIpScript(String brName, String mac, String ipv4, String ipv6, List<String> secondaryIps) {
+        if (_macIpScriptPath == null || mac == null || brName == null) {
+            return;
+        }
+        try {
+            final Script command = new Script(_macIpScriptPath, _timeout, logger);
+            command.add("-o", "add");
+            command.add("-b", brName);
+            command.add("-m", mac);
+            if (ipv4 != null && !ipv4.isEmpty()) {
+                command.add("-4", ipv4);
+            }
+            command.add("-6", NetUtils.ipv6LinkLocal(mac).toString());
+            if (ipv6 != null && !ipv6.isEmpty()) {
+                command.add("-6", ipv6);
+            }
+            if (secondaryIps != null) {
+                for (String secIp : secondaryIps) {
+                    if (NetUtils.isValidIp6(secIp)) {
+                        command.add("-6", secIp);
+                    } else {
+                        command.add("-4", secIp);
+                    }
+                }
+            }
+            final String result = command.execute();
+            if (result != null) {
+                logger.warn("MAC/IP script returned error for add on {}: {}", mac, result);
+            }
+        } catch (Exception e) {
+            // Managing host neighbour/route entries is best-effort and must never break VM lifecycle operations
+            logger.warn("Failed to run MAC/IP script for add on {} ({})", mac, brName, e);
+        }
+    }
+
     private void deleteExistingLinkLocalRouteTable(String linkLocalBr) {
         Script command = new Script("/bin/bash", _timeout);
         command.add("-c");

scripts/vm/network/vnet/modifymacip.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# modifymacip.sh -- Manage static ARP/NDP entries and host routes for VM NICs
+#
+# Usage:
+#   add:    modifymacip.sh -o add    -b <bridge> -m <mac> [-4 <ipv4>] ... [-6 <ipv6>] ...
+#   delete: modifymacip.sh -o delete -b <bridge> -m <mac>
+#
+# Both -4 and -6 may be specified multiple times to cover primary and secondary
+# addresses (e.g. link-local + global unicast for IPv6).
+# On delete the bridge neighbour table is queried for all entries matching the
+# MAC address; no separate state file is required.
+
+usage() {
+    echo "Usage: $0 -o <add|delete> -b <bridge> -m <mac> [-4 <ipv4>] ... [-6 <ipv6>] ..."
+}
+
+OP=
+BRIDGE=
+MAC=
+IPV4_LIST=()
+IPV6_LIST=()
+
+while getopts 'o:b:m:4:6:' OPTION; do
+    case $OPTION in
+    o) OP="$OPTARG" ;;
+    b) BRIDGE="$OPTARG" ;;
+    m) MAC="$OPTARG" ;;
+    4) IPV4_LIST+=("$OPTARG") ;;
+    6) IPV6_LIST+=("$OPTARG") ;;
+    ?) usage; exit 2 ;;
+    esac
+done
+
+if [[ -z "$OP" || -z "$BRIDGE" || -z "$MAC" ]]; then
+    usage
+    exit 2
+fi
+
+add_entries() {
+    for addr in "${IPV4_LIST[@]}"; do
+        ip neigh replace "${addr}" lladdr "${MAC}" dev "${BRIDGE}" nud permanent
+        ip route replace "${addr}/32" dev "${BRIDGE}"
+    done
+
+    if [[ "${#IPV6_LIST[@]}" -gt 0 ]]; then
+        # Ensure IPv6 is enabled on the bridge before installing NDP entries
+        sysctl -qw "net.ipv6.conf.${BRIDGE}.disable_ipv6=0"
+        for addr in "${IPV6_LIST[@]}"; do
+            ip -6 neigh replace "${addr}" lladdr "${MAC}" dev "${BRIDGE}" nud permanent
+            ip -6 route replace "${addr}/128" dev "${BRIDGE}"
+        done
+    fi
+}
+
+delete_entries() {
+    # Find all IPv4 neighbour entries on the bridge matching this MAC and remove them
+    while read -r addr; do
+        ip neigh del "${addr}" dev "${BRIDGE}" 2>/dev/null || true
+        ip route del "${addr}/32" dev "${BRIDGE}" 2>/dev/null || true
+    done < <(ip neigh show dev "${BRIDGE}" | awk -v mac="${MAC}" 'tolower($3) == tolower(mac) {print $1}')
+
+    # Find all IPv6 neighbour entries on the bridge matching this MAC and remove them
+    while read -r addr; do
+        ip -6 neigh del "${addr}" dev "${BRIDGE}" 2>/dev/null || true
+        ip -6 route del "${addr}/128" dev "${BRIDGE}" 2>/dev/null || true
+    done < <(ip -6 neigh show dev "${BRIDGE}" | awk -v mac="${MAC}" 'tolower($3) == tolower(mac) {print $1}')
+}
+
+case "$OP" in
+    add)    add_entries ;;
+    delete) delete_entries ;;
+    *)      usage; exit 2 ;;
+esac

scripts/vm/network/vnet/modifyvxlan-evpn.sh

@@ -82,6 +82,8 @@ addVxlan() {
     bridge link show|grep ${VXLAN_BR}|awk '{print $2}'|grep "^${VXLAN_DEV}\$" > /dev/null
     if [[ $? -gt 0 ]]; then
         ip link set ${VXLAN_DEV} master ${VXLAN_BR}
+        bridge link set dev ${VXLAN_DEV} neigh_suppress on
+        bridge link set dev ${VXLAN_DEV} learning off
     fi
 }