Skip to content

Commit 501aa7c

Browse files
nvazqueznvazquez
authored
DPDK vHost User mode selection (#3153)
* DPDK vHost User mode selection * SQL text field and DPDK classes refactor * Fix NullPointerException after refactor * Fix unit test * Refactor details type
1 parent 4e8f149 commit 501aa7c

15 files changed

Lines changed: 615 additions & 80 deletions

File tree

‎engine/schema/src/main/resources/META-INF/db/schema-41200to41300.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,5 @@
1919
-- Schema upgrade from 4.12.0.0 to 4.13.0.0
2020
--;
2121

22+
-- DPDK client and server mode support
23+
ALTER TABLE `cloud`.`service_offering_details` CHANGE COLUMN `value` `value` TEXT NOT NULL;

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/dpdk/DPDKDriver.java‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package com.cloud.hypervisor.kvm.dpdk;
20+
21+
import com.cloud.utils.component.Adapter;
22+
23+
import java.util.Map;
24+
25+
public interface DPDKDriver extends Adapter {
26+
27+
/**
28+
* Get the next DPDK port name to be created
29+
*/
30+
String getNextDpdkPort();
31+
32+
/**
33+
* Get the latest DPDK port number created on a DPDK enabled host
34+
*/
35+
int getDpdkLatestPortNumberUsed();
36+
37+
/**
38+
* Add OVS port (if it does not exist) to bridge with DPDK support
39+
*/
40+
void addDpdkPort(String bridgeName, String port, String vlan, DPDKHelper.VHostUserMode vHostUserMode, String dpdkOvsPath);
41+
42+
/**
43+
* Since DPDK user client/server mode, retrieve the guest interfaces mode from the DPDK vHost User mode
44+
*/
45+
String getGuestInterfacesModeFromDPDKVhostUserMode(DPDKHelper.VHostUserMode dpdKvHostUserMode);
46+
47+
/**
48+
* Get DPDK vHost User mode from extra config. If it is not present, server is returned as default
49+
*/
50+
DPDKHelper.VHostUserMode getDPDKvHostUserMode(Map<String, String> extraConfig);
51+
52+
/**
53+
* Check for additional extra 'dpdk-interface' configurations, return them appended
54+
*/
55+
String getExtraDpdkProperties(Map<String, String> extraConfig);
56+
}

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/dpdk/DPDKDriverImpl.java‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package com.cloud.hypervisor.kvm.dpdk;
20+
21+
import com.cloud.utils.component.AdapterBase;
22+
import com.cloud.utils.script.Script;
23+
import org.apache.commons.lang.StringUtils;
24+
import org.apache.log4j.Logger;
25+
26+
import java.util.Map;
27+
28+
public class DPDKDriverImpl extends AdapterBase implements DPDKDriver {
29+
static final String DPDK_PORT_PREFIX = "csdpdk-";
30+
31+
private final String dpdkPortVhostUserType = "dpdkvhostuser";
32+
private final String dpdkPortVhostUserClientType = "dpdkvhostuserclient";
33+
34+
private static final Logger s_logger = Logger.getLogger(DPDKDriver.class);
35+
36+
public DPDKDriverImpl() {
37+
}
38+
39+
/**
40+
* Get the next DPDK port name to be created
41+
*/
42+
public String getNextDpdkPort() {
43+
int portNumber = getDpdkLatestPortNumberUsed();
44+
return DPDK_PORT_PREFIX + String.valueOf(portNumber + 1);
45+
}
46+
47+
/**
48+
* Get the latest DPDK port number created on a DPDK enabled host
49+
*/
50+
public int getDpdkLatestPortNumberUsed() {
51+
s_logger.debug("Checking the last DPDK port created");
52+
String cmd = "ovs-vsctl show | grep Port | grep " + DPDK_PORT_PREFIX + " | " +
53+
"awk '{ print $2 }' | sort -rV | head -1";
54+
String port = Script.runSimpleBashScript(cmd);
55+
int portNumber = 0;
56+
if (StringUtils.isNotBlank(port)) {
57+
String unquotedPort = port.replace("\"", "");
58+
String dpdkPortNumber = unquotedPort.split(DPDK_PORT_PREFIX)[1];
59+
portNumber = Integer.valueOf(dpdkPortNumber);
60+
}
61+
return portNumber;
62+
}
63+
64+
/**
65+
* Add OVS port (if it does not exist) to bridge with DPDK support
66+
*/
67+
public void addDpdkPort(String bridgeName, String port, String vlan, DPDKHelper.VHostUserMode vHostUserMode, String dpdkOvsPath) {
68+
String type = vHostUserMode == DPDKHelper.VHostUserMode.SERVER ?
69+
dpdkPortVhostUserType :
70+
dpdkPortVhostUserClientType;
71+
72+
StringBuilder stringBuilder = new StringBuilder();
73+
stringBuilder.append(String.format("ovs-vsctl add-port %s %s " +
74+
"vlan_mode=access tag=%s " +
75+
"-- set Interface %s type=%s", bridgeName, port, vlan, port, type));
76+
77+
if (vHostUserMode == DPDKHelper.VHostUserMode.CLIENT) {
78+
stringBuilder.append(String.format(" options:vhost-server-path=%s/%s",
79+
dpdkOvsPath, port));
80+
}
81+
82+
String cmd = stringBuilder.toString();
83+
s_logger.debug("DPDK property enabled, executing: " + cmd);
84+
Script.runSimpleBashScript(cmd);
85+
}
86+
87+
/**
88+
* Since DPDK user client/server mode, retrieve the guest interfaces mode from the DPDK vHost User mode
89+
*/
90+
public String getGuestInterfacesModeFromDPDKVhostUserMode(DPDKHelper.VHostUserMode dpdKvHostUserMode) {
91+
return dpdKvHostUserMode == DPDKHelper.VHostUserMode.CLIENT ? "server" : "client";
92+
}
93+
94+
/**
95+
* Get DPDK vHost User mode from extra config. If it is not present, server is returned as default
96+
*/
97+
public DPDKHelper.VHostUserMode getDPDKvHostUserMode(Map<String, String> extraConfig) {
98+
return extraConfig.containsKey(DPDKHelper.DPDK_VHOST_USER_MODE) ?
99+
DPDKHelper.VHostUserMode.fromValue(extraConfig.get(DPDKHelper.DPDK_VHOST_USER_MODE)) :
100+
DPDKHelper.VHostUserMode.SERVER;
101+
}
102+
103+
/**
104+
* Check for additional extra 'dpdk-interface' configurations, return them appended
105+
*/
106+
public String getExtraDpdkProperties(Map<String, String> extraConfig) {
107+
StringBuilder stringBuilder = new StringBuilder();
108+
for (String key : extraConfig.keySet()) {
109+
if (key.startsWith(DPDKHelper.DPDK_INTERFACE_PREFIX)) {
110+
stringBuilder.append(extraConfig.get(key));
111+
}
112+
}
113+
return stringBuilder.toString();
114+
}
115+
}

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@
4646
import javax.xml.parsers.DocumentBuilderFactory;
4747
import javax.xml.parsers.ParserConfigurationException;
4848

49+
import com.cloud.hypervisor.kvm.dpdk.DPDKHelper;
4950
import com.cloud.resource.RequestWrapper;
50-
import org.apache.cloudstack.api.ApiConstants;
5151
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
5252
import org.apache.cloudstack.storage.to.TemplateObjectTO;
5353
import org.apache.cloudstack.storage.to.VolumeObjectTO;
@@ -524,9 +524,6 @@ protected enum BridgeType {
524524

525525
protected boolean dpdkSupport = false;
526526
protected String dpdkOvsPath;
527-
protected static final String DPDK_NUMA = ApiConstants.EXTRA_CONFIG + "-dpdk-numa";
528-
protected static final String DPDK_HUGE_PAGES = ApiConstants.EXTRA_CONFIG + "-dpdk-hugepages";
529-
protected static final String DPDK_INTERFACE_PREFIX = ApiConstants.EXTRA_CONFIG + "-dpdk-interface-";
530527

531528
private String getEndIpFromStartIp(final String startIp, final int numIps) {
532529
final String[] tokens = startIp.split("[.]");
@@ -2073,7 +2070,7 @@ public LibvirtVMDef createVMFromSpec(final VirtualMachineTO vmTO) {
20732070
vm.setPlatformEmulator(vmTO.getPlatformEmulator());
20742071

20752072
Map<String, String> extraConfig = vmTO.getExtraConfig();
2076-
if (dpdkSupport && (!extraConfig.containsKey(DPDK_NUMA) || !extraConfig.containsKey(DPDK_HUGE_PAGES))) {
2073+
if (dpdkSupport && (!extraConfig.containsKey(DPDKHelper.DPDK_NUMA) || !extraConfig.containsKey(DPDKHelper.DPDK_HUGE_PAGES))) {
20772074
s_logger.info("DPDK is enabled but it needs extra configurations for CPU NUMA and Huge Pages for VM deployment");
20782075
}
20792076

@@ -2110,7 +2107,7 @@ public LibvirtVMDef createVMFromSpec(final VirtualMachineTO vmTO) {
21102107
grd.setVcpuNum(vcpus);
21112108
vm.addComp(grd);
21122109

2113-
if (!extraConfig.containsKey(DPDK_NUMA)) {
2110+
if (!extraConfig.containsKey(DPDKHelper.DPDK_NUMA)) {
21142111
final CpuModeDef cmd = new CpuModeDef();
21152112
cmd.setMode(_guestCpuMode);
21162113
cmd.setModel(_guestCpuModel);
@@ -2238,7 +2235,7 @@ protected void addExtraConfigComponent(Map<String, String> extraConfig, LibvirtV
22382235
if (MapUtils.isNotEmpty(extraConfig)) {
22392236
StringBuilder extraConfigBuilder = new StringBuilder();
22402237
for (String key : extraConfig.keySet()) {
2241-
if (!key.startsWith(DPDK_INTERFACE_PREFIX)) {
2238+
if (!key.startsWith(DPDKHelper.DPDK_INTERFACE_PREFIX) && !key.equals(DPDKHelper.DPDK_VHOST_USER_MODE)) {
22422239
extraConfigBuilder.append(extraConfig.get(key));
22432240
}
22442241
}

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1017,6 +1017,7 @@ enum HostNicType {
10171017
private String _dpdkSourcePath;
10181018
private String _dpdkSourcePort;
10191019
private String _dpdkExtraLines;
1020+
private String _interfaceMode;
10201021

10211022
public void defBridgeNet(String brName, String targetBrName, String macAddr, NicModel model) {
10221023
defBridgeNet(brName, targetBrName, macAddr, model, 0);
@@ -1031,14 +1032,16 @@ public void defBridgeNet(String brName, String targetBrName, String macAddr, Nic
10311032
_networkRateKBps = networkRateKBps;
10321033
}
10331034

1034-
public void defDpdkNet(String dpdkSourcePath, String dpdkPort, String macAddress, NicModel model, Integer networkRateKBps, String extra) {
1035+
public void defDpdkNet(String dpdkSourcePath, String dpdkPort, String macAddress, NicModel model,
1036+
Integer networkRateKBps, String extra, String interfaceMode) {
10351037
_netType = GuestNetType.VHOSTUSER;
10361038
_dpdkSourcePath = dpdkSourcePath;
10371039
_dpdkSourcePort = dpdkPort;
10381040
_macAddr = macAddress;
10391041
_model = model;
10401042
_networkRateKBps = networkRateKBps;
10411043
_dpdkExtraLines = extra;
1044+
_interfaceMode = interfaceMode;
10421045
}
10431046

10441047
public void defDirectNet(String sourceName, String targetName, String macAddr, NicModel model, String sourceMode) {
@@ -1184,7 +1187,8 @@ public String toString() {
11841187
} else if (_netType == GuestNetType.DIRECT) {
11851188
netBuilder.append("<source dev='" + _sourceName + "' mode='" + _netSourceMode + "'/>\n");
11861189
} else if (_netType == GuestNetType.VHOSTUSER) {
1187-
netBuilder.append("<source type='unix' path='"+ _dpdkSourcePath + _dpdkSourcePort + "' mode='client'/>\n");
1190+
netBuilder.append("<source type='unix' path='"+ _dpdkSourcePath + _dpdkSourcePort +
1191+
"' mode='" + _interfaceMode + "'/>\n");
11881192
}
11891193
if (_networkName != null) {
11901194
netBuilder.append("<target dev='" + _networkName + "'/>\n");

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/OvsVifDriver.java‎

Lines changed: 19 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@
2424

2525
import javax.naming.ConfigurationException;
2626

27+
import com.cloud.hypervisor.kvm.dpdk.DPDKDriver;
28+
import com.cloud.hypervisor.kvm.dpdk.DPDKDriverImpl;
29+
import com.cloud.hypervisor.kvm.dpdk.DPDKHelper;
2730
import com.cloud.utils.exception.CloudRuntimeException;
2831
import org.apache.commons.lang.StringUtils;
2932
import org.apache.log4j.Logger;
@@ -41,8 +44,7 @@
4144
public class OvsVifDriver extends VifDriverBase {
4245
private static final Logger s_logger = Logger.getLogger(OvsVifDriver.class);
4346
private int _timeout;
44-
45-
protected static final String DPDK_PORT_PREFIX = "csdpdk-";
47+
private DPDKDriver dpdkDriver;
4648

4749
@Override
4850
public void configure(Map<String, Object> params) throws ConfigurationException {
@@ -55,6 +57,11 @@ public void configure(Map<String, Object> params) throws ConfigurationException
5557
networkScriptsDir = "scripts/vm/network/vnet";
5658
}
5759

60+
String dpdk = (String) params.get("openvswitch.dpdk.enabled");
61+
if (StringUtils.isNotBlank(dpdk) && Boolean.parseBoolean(dpdk)) {
62+
dpdkDriver = new DPDKDriverImpl();
63+
}
64+
5865
String value = (String)params.get("scripts.timeout");
5966
_timeout = NumbersUtil.parseInt(value, 30 * 60) * 1000;
6067
}
@@ -80,55 +87,6 @@ public void getPifs() {
8087
s_logger.debug("done looking for pifs, no more bridges");
8188
}
8289

83-
/**
84-
* Get the latest DPDK port number created on a DPDK enabled host
85-
*/
86-
protected int getDpdkLatestPortNumberUsed() {
87-
s_logger.debug("Checking the last DPDK port created");
88-
String cmd = "ovs-vsctl show | grep Port | grep " + DPDK_PORT_PREFIX + " | " +
89-
"awk '{ print $2 }' | sort -rV | head -1";
90-
String port = Script.runSimpleBashScript(cmd);
91-
int portNumber = 0;
92-
if (StringUtils.isNotBlank(port)) {
93-
String unquotedPort = port.replace("\"", "");
94-
String dpdkPortNumber = unquotedPort.split(DPDK_PORT_PREFIX)[1];
95-
portNumber = Integer.valueOf(dpdkPortNumber);
96-
}
97-
return portNumber;
98-
}
99-
100-
/**
101-
* Get the next DPDK port name to be created
102-
*/
103-
protected String getNextDpdkPort() {
104-
int portNumber = getDpdkLatestPortNumberUsed();
105-
return DPDK_PORT_PREFIX + String.valueOf(portNumber + 1);
106-
}
107-
108-
/**
109-
* Add OVS port (if it does not exist) to bridge with DPDK support
110-
*/
111-
protected void addDpdkPort(String bridgeName, String port, String vlan) {
112-
String cmd = String.format("ovs-vsctl add-port %s %s " +
113-
"vlan_mode=access tag=%s " +
114-
"-- set Interface %s type=dpdkvhostuser", bridgeName, port, vlan, port);
115-
s_logger.debug("DPDK property enabled, executing: " + cmd);
116-
Script.runSimpleBashScript(cmd);
117-
}
118-
119-
/**
120-
* Check for additional extra 'dpdk-interface' configurations, return them appended
121-
*/
122-
private String getExtraDpdkProperties(Map<String, String> extraConfig) {
123-
StringBuilder stringBuilder = new StringBuilder();
124-
for (String key : extraConfig.keySet()) {
125-
if (key.startsWith(LibvirtComputingResource.DPDK_INTERFACE_PREFIX)) {
126-
stringBuilder.append(extraConfig.get(key));
127-
}
128-
}
129-
return stringBuilder.toString();
130-
}
131-
13290
@Override
13391
public InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException {
13492
s_logger.debug("plugging nic=" + nic);
@@ -158,12 +116,18 @@ public InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<S
158116
if (trafficLabel != null && !trafficLabel.isEmpty()) {
159117
if (_libvirtComputingResource.dpdkSupport && !nic.isDpdkDisabled()) {
160118
s_logger.debug("DPDK support enabled: configuring per traffic label " + trafficLabel);
161-
if (StringUtils.isBlank(_libvirtComputingResource.dpdkOvsPath)) {
119+
String dpdkOvsPath = _libvirtComputingResource.dpdkOvsPath;
120+
if (StringUtils.isBlank(dpdkOvsPath)) {
162121
throw new CloudRuntimeException("DPDK is enabled on the host but no OVS path has been provided");
163122
}
164-
String port = getNextDpdkPort();
165-
addDpdkPort(_pifs.get(trafficLabel), port, vlanId);
166-
intf.defDpdkNet(_libvirtComputingResource.dpdkOvsPath, port, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), 0, getExtraDpdkProperties(extraConfig));
123+
String port = dpdkDriver.getNextDpdkPort();
124+
DPDKHelper.VHostUserMode dpdKvHostUserMode = dpdkDriver.getDPDKvHostUserMode(extraConfig);
125+
dpdkDriver.addDpdkPort(_pifs.get(trafficLabel), port, vlanId, dpdKvHostUserMode, dpdkOvsPath);
126+
String interfaceMode = dpdkDriver.getGuestInterfacesModeFromDPDKVhostUserMode(dpdKvHostUserMode);
127+
intf.defDpdkNet(dpdkOvsPath, port, nic.getMac(),
128+
getGuestNicModel(guestOsType, nicAdapter), 0,
129+
dpdkDriver.getExtraDpdkProperties(extraConfig),
130+
interfaceMode);
167131
} else {
168132
s_logger.debug("creating a vlan dev and bridge for guest traffic per traffic label " + trafficLabel);
169133
intf.defBridgeNet(_pifs.get(trafficLabel), null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps);

0 commit comments

Comments
 (0)