Skip to content

Commit ee6720e

Browse files
sureshanapartisureshanaparti
committed
Support for parallel downloads of VMware VM disk files while exporting OVF from MS, and other changes below.
- Skip clone for powered off VMs - Fixes to support standalone host (with its default datacenter) - Some code improvements
1 parent 0c52c28 commit ee6720e

15 files changed

Lines changed: 263 additions & 122 deletions

File tree

‎api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -109,13 +109,13 @@ boolean attachRestoredVolumeToVirtualMachine(long zoneId, String location, Backu
109109

110110

111111
/**
112-
* Will perform a clone of a VM on an external host (if the guru can handle)
112+
* Will return the hypervisor VM (clone VM for PowerOn VMs), performs a clone of a VM if required on an external host (if the guru can handle)
113113
* @param hostIp VM's source host IP
114-
* @param vmName name of the source VM to clone from
114+
* @param vmName name of the source VM (clone VM name if cloned)
115115
* @param params hypervisor specific additional parameters
116-
* @return a reference to the cloned VM
116+
* @return a reference to the hypervisor or cloned VM, and cloned flag
117117
*/
118-
UnmanagedInstanceTO cloneHypervisorVMOutOfBand(String hostIp, String vmName, Map<String, String> params);
118+
Pair<UnmanagedInstanceTO, Boolean> getHypervisorVMOutOfBandAndCloneIfRequired(String hostIp, String vmName, Map<String, String> params);
119119

120120
/**
121121
* Removes a VM created as a clone of a VM on an external host
@@ -134,7 +134,7 @@ boolean attachRestoredVolumeToVirtualMachine(long zoneId, String location, Backu
134134
* @param templateLocation datastore to create the template file
135135
* @return the created template dir/name
136136
*/
137-
String createVMTemplateOutOfBand(String hostIp, String vmName, Map<String, String> params, DataStoreTO templateLocation);
137+
String createVMTemplateOutOfBand(String hostIp, String vmName, Map<String, String> params, DataStoreTO templateLocation, int threadsCountToExportOvf);
138138

139139
/**
140140
* Removes the template on the location

‎api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
package org.apache.cloudstack.vm;
1919

20+
import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
21+
2022
import java.util.List;
2123

2224
public class UnmanagedInstanceTO {
@@ -322,7 +324,7 @@ public int getDatastorePort() {
322324
public String toString() {
323325
return "Disk {" +
324326
"diskId='" + diskId + '\'' +
325-
", capacity=" + capacity +
327+
", capacity=" + toHumanReadableSize(capacity) +
326328
", controller='" + controller + '\'' +
327329
", controllerUnit=" + controllerUnit +
328330
"}";

‎api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,12 +45,22 @@ public interface UnmanagedVMsManager extends VmImportService, UnmanageVMService,
4545
ConfigKey.Scope.Global,
4646
null);
4747

48-
ConfigKey<Integer> ThreadsOnKVMHostToTransferVMwareVMFiles = new ConfigKey<>(Integer.class,
49-
"threads.on.kvm.host.to.transfer.vmware.vm.files",
48+
ConfigKey<Integer> ThreadsOnMSToDownloadVMwareVMFiles = new ConfigKey<>(Integer.class,
49+
"threads.on.ms.to.download.vmware.vm.files",
5050
"Advanced",
5151
"0",
52-
"Threads to use on the KVM host (by OVF Tool, supported from version >= 4.4.0) to transfer VMware VM files," +
53-
" less than zero - disabled, zero - uses total disks count, value should be less than 100 which is approximated to the number" +
52+
"Threads to use on the management server to download VMware VM files per import VM," +
53+
" single disk or less than zero - disabled, zero - uses total disks count, maximum value is 10. Default: 0.",
54+
true,
55+
ConfigKey.Scope.Global,
56+
null);
57+
58+
ConfigKey<Integer> ThreadsOnKVMHostToDownloadVMwareVMFiles = new ConfigKey<>(Integer.class,
59+
"threads.on.kvm.host.to.download.vmware.vm.files",
60+
"Advanced",
61+
"0",
62+
"Threads to use on the KVM host (by OVF Tool, supported from version >= 4.4.0) to download VMware VM files per import VM," +
63+
" single disk or less than zero - disabled, zero - uses total disks count, value should be less than 100 which is approximated to the number" +
5464
" of CPU cores minus one. Default: 0.",
5565
true,
5666
ConfigKey.Scope.Global,

‎plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java‎

Lines changed: 37 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1276,23 +1276,22 @@ private VirtualMachineMO createCloneFromSourceVM(String vmName, VirtualMachineMO
12761276
return clonedVM;
12771277
}
12781278

1279-
private String createOVFTemplateOfVM(VirtualMachineMO vmMO, DataStoreTO convertLocation) throws Exception {
1279+
private String createOVFTemplateOfVM(VirtualMachineMO vmMO, DataStoreTO convertLocation, int threadsCountToExportOvf) throws Exception {
12801280
String dataStoreUrl = getDataStoreUrlForTemplate(convertLocation);
12811281
String vmOvfName = UUID.randomUUID().toString();
12821282
String vmOvfCreationPath = createDirOnStorage(vmOvfName, dataStoreUrl, null);
12831283
s_logger.debug(String.format("Creating OVF %s for the VM %s at %s", vmOvfName, vmMO.getName(), vmOvfCreationPath));
1284-
vmMO.exportVm(vmOvfCreationPath, vmOvfName, false, false);
1284+
vmMO.exportVm(vmOvfCreationPath, vmOvfName, false, false, threadsCountToExportOvf);
12851285
s_logger.debug(String.format("Created OVF %s for the VM %s at %s", vmOvfName, vmMO.getName(), vmOvfCreationPath));
12861286
return vmOvfName;
12871287
}
12881288

12891289
@Override
1290-
public UnmanagedInstanceTO cloneHypervisorVMOutOfBand(String hostIp, String vmName, Map<String, String> params) {
1290+
public Pair<UnmanagedInstanceTO, Boolean> getHypervisorVMOutOfBandAndCloneIfRequired(String hostIp, String vmName, Map<String, String> params) {
12911291
String vcenter = params.get(VmDetailConstants.VMWARE_VCENTER_HOST);
12921292
String datacenter = params.get(VmDetailConstants.VMWARE_DATACENTER_NAME);
12931293
String username = params.get(VmDetailConstants.VMWARE_VCENTER_USERNAME);
12941294
String password = params.get(VmDetailConstants.VMWARE_VCENTER_PASSWORD);
1295-
s_logger.debug(String.format("Cloning VM %s at VMware host %s on vCenter %s", vmName, hostIp, vcenter));
12961295

12971296
try {
12981297
VmwareContext context = connectToVcenter(vcenter, username, password);
@@ -1303,22 +1302,41 @@ public UnmanagedInstanceTO cloneHypervisorVMOutOfBand(String hostIp, String vmNa
13031302
s_logger.error(err);
13041303
throw new CloudRuntimeException(err);
13051304
}
1305+
13061306
VirtualMachinePowerState sourceVmPowerState = vmMo.getPowerState();
1307-
if (sourceVmPowerState == VirtualMachinePowerState.POWERED_ON && isWindowsVm(vmMo)) {
1308-
String err = String.format("VM %s is a Windows VM and its Running, cannot be imported." +
1309-
"Please gracefully shut it down before attempting the import", vmName);
1310-
s_logger.error(err);
1311-
throw new CloudRuntimeException(err);
1307+
if (sourceVmPowerState == VirtualMachinePowerState.POWERED_OFF) {
1308+
// Don't clone for powered off VMs, can export OVF from it
1309+
UnmanagedInstanceTO instanceTO = VmwareHelper.getUnmanagedInstance(vmMo.getRunningHost(), vmMo);
1310+
return new Pair<>(instanceTO, false);
13121311
}
13131312

1313+
if (sourceVmPowerState == VirtualMachinePowerState.POWERED_ON) {
1314+
if (isWindowsVm(vmMo)) {
1315+
String err = String.format("VM %s is a Windows VM and its Running, cannot be imported." +
1316+
" Please gracefully shut it down before attempting the import", vmName);
1317+
s_logger.error(err);
1318+
throw new CloudRuntimeException(err);
1319+
}
1320+
1321+
if (isVMOnStandaloneHost(vmMo)) { // or datacenter.equalsIgnoreCase("ha-datacenter")? [Note: default datacenter name on standalone host: ha-datacenter]
1322+
String err = String.format("VM %s might be on standalone host and is Running, cannot be imported." +
1323+
" Please shut it down before attempting the import", vmName);
1324+
s_logger.error(err);
1325+
throw new CloudRuntimeException(err);
1326+
}
1327+
}
1328+
1329+
s_logger.debug(String.format("Cloning VM %s at VMware host %s on vCenter %s", vmName, hostIp, vcenter));
13141330
VirtualMachineMO clonedVM = createCloneFromSourceVM(vmName, vmMo, dataCenterMO);
1315-
s_logger.debug(String.format("VM %s cloned successfully", vmName));
1331+
s_logger.debug(String.format("VM %s cloned successfully, to VM %s", vmName, clonedVM.getName()));
13161332
UnmanagedInstanceTO clonedInstance = VmwareHelper.getUnmanagedInstance(vmMo.getRunningHost(), clonedVM);
13171333
setDisksFromSourceVM(clonedInstance, vmMo);
13181334
clonedInstance.setCloneSourcePowerState(sourceVmPowerState == VirtualMachinePowerState.POWERED_ON ? UnmanagedInstanceTO.PowerState.PowerOn : UnmanagedInstanceTO.PowerState.PowerOff);
1319-
return clonedInstance;
1335+
return new Pair<>(clonedInstance, true);
1336+
} catch (CloudRuntimeException cre) {
1337+
throw cre;
13201338
} catch (Exception e) {
1321-
String err = String.format("Error cloning VM: %s from vCenter %s: %s", vmName, vcenter, e.getMessage());
1339+
String err = String.format("Error while finding or cloning VM: %s from vCenter %s: %s", vmName, vcenter, e.getMessage());
13221340
s_logger.error(err, e);
13231341
throw new CloudRuntimeException(err, e);
13241342
}
@@ -1329,6 +1347,11 @@ private boolean isWindowsVm(VirtualMachineMO vmMo) throws Exception {
13291347
return sourceInstance.getOperatingSystem().toLowerCase().contains("windows");
13301348
}
13311349

1350+
private boolean isVMOnStandaloneHost(VirtualMachineMO vmMo) throws Exception {
1351+
UnmanagedInstanceTO sourceInstance = VmwareHelper.getUnmanagedInstance(vmMo.getRunningHost(), vmMo);
1352+
return StringUtils.isEmpty(sourceInstance.getClusterName());
1353+
}
1354+
13321355
private void setDisksFromSourceVM(UnmanagedInstanceTO clonedInstance, VirtualMachineMO vmMo) throws Exception {
13331356
UnmanagedInstanceTO sourceInstance = VmwareHelper.getUnmanagedInstance(vmMo.getRunningHost(), vmMo);
13341357
List<UnmanagedInstanceTO.Disk> sourceDisks = sourceInstance.getDisks();
@@ -1341,7 +1364,7 @@ private void setDisksFromSourceVM(UnmanagedInstanceTO clonedInstance, VirtualMac
13411364
}
13421365

13431366
@Override
1344-
public String createVMTemplateOutOfBand(String hostIp, String vmName, Map<String, String> params, DataStoreTO templateLocation) {
1367+
public String createVMTemplateOutOfBand(String hostIp, String vmName, Map<String, String> params, DataStoreTO templateLocation, int threadsCountToExportOvf) {
13451368
String vcenter = params.get(VmDetailConstants.VMWARE_VCENTER_HOST);
13461369
String datacenter = params.get(VmDetailConstants.VMWARE_DATACENTER_NAME);
13471370
String username = params.get(VmDetailConstants.VMWARE_VCENTER_USERNAME);
@@ -1357,7 +1380,7 @@ public String createVMTemplateOutOfBand(String hostIp, String vmName, Map<String
13571380
s_logger.error(err);
13581381
throw new CloudRuntimeException(err);
13591382
}
1360-
String ovaTemplate = createOVFTemplateOfVM(vmMo, templateLocation);
1383+
String ovaTemplate = createOVFTemplateOfVM(vmMo, templateLocation, threadsCountToExportOvf);
13611384
s_logger.debug(String.format("OVF %s created successfully on the datastore", ovaTemplate));
13621385
return ovaTemplate;
13631386
} catch (Exception e) {

‎plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/guru/VMwareGuruTest.java‎

Lines changed: 22 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import static org.junit.Assert.assertTrue;
2323
import static org.mockito.ArgumentMatchers.any;
2424
import static org.mockito.ArgumentMatchers.anyBoolean;
25+
import static org.mockito.ArgumentMatchers.anyInt;
2526
import static org.mockito.ArgumentMatchers.anyString;
2627
import static org.mockito.Mockito.spy;
2728
import static org.mockito.Mockito.when;
@@ -236,7 +237,7 @@ public void testCloneHypervisorVM_NoExternalVM() throws Exception {
236237
}); MockedConstruction<DatacenterMO> ignored3 = Mockito.mockConstruction(DatacenterMO.class, withSettings().spiedInstance(dataCenterMO), (mockDatacenterMO, contextDatacenterMO) -> {
237238
Mockito.doReturn(null).when(mockDatacenterMO).findVm(vmName);
238239
})) {
239-
vMwareGuru.cloneHypervisorVMOutOfBand(hostIp, vmName, params);
240+
vMwareGuru.getHypervisorVMOutOfBandAndCloneIfRequired(hostIp, vmName, params);
240241
}
241242
}
242243

@@ -280,7 +281,7 @@ public void testCloneHypervisorVM_WindowsVMRunning() throws Exception {
280281
}); MockedConstruction<DatacenterMO> ignored3 = Mockito.mockConstruction(DatacenterMO.class, withSettings().spiedInstance(dataCenterMO), (mockDatacenterMO, contextDatacenterMO) -> {
281282
Mockito.doReturn(vmMo).when(mockDatacenterMO).findVm(vmName);
282283
})) {
283-
vMwareGuru.cloneHypervisorVMOutOfBand(hostIp, vmName, params);
284+
vMwareGuru.getHypervisorVMOutOfBandAndCloneIfRequired(hostIp, vmName, params);
284285
}
285286
}
286287

@@ -307,8 +308,12 @@ public void testCloneHypervisorVM_GetDatastoresFailed() throws Exception {
307308
DatacenterMO dataCenterMO = spy(new DatacenterMO(vmwareContext, datacenterName));
308309
VirtualMachineMO vmMo = Mockito.mock(VirtualMachineMO.class);
309310
HostMO hostMo = Mockito.mock(HostMO.class);
310-
Mockito.doReturn(VirtualMachinePowerState.POWERED_OFF).when(vmMo).getPowerState();
311+
Mockito.doReturn(VirtualMachinePowerState.POWERED_ON).when(vmMo).getPowerState();
311312
Mockito.doReturn(hostMo).when(vmMo).getRunningHost();
313+
UnmanagedInstanceTO instance = Mockito.mock(UnmanagedInstanceTO.class);
314+
Mockito.doReturn("CentOS").when(instance).getOperatingSystem();
315+
Mockito.doReturn("test-cluster").when(instance).getClusterName();
316+
when(VmwareHelper.getUnmanagedInstance(hostMo, vmMo)).thenReturn(instance);
312317
List<DatastoreMO> datastores = new ArrayList<>();
313318
Mockito.doReturn(datastores).when(vmMo).getAllDatastores();
314319

@@ -323,7 +328,7 @@ public void testCloneHypervisorVM_GetDatastoresFailed() throws Exception {
323328
}); MockedConstruction<DatacenterMO> ignored3 = Mockito.mockConstruction(DatacenterMO.class, withSettings().spiedInstance(dataCenterMO), (mockDatacenterMO, contextDatacenterMO) -> {
324329
Mockito.doReturn(vmMo).when(mockDatacenterMO).findVm(vmName);
325330
})) {
326-
vMwareGuru.cloneHypervisorVMOutOfBand(hostIp, vmName, params);
331+
vMwareGuru.getHypervisorVMOutOfBandAndCloneIfRequired(hostIp, vmName, params);
327332
}
328333
}
329334

@@ -350,9 +355,13 @@ public void testCloneHypervisorVM_CloneVMFailed() throws Exception {
350355
DatacenterMO dataCenterMO = spy(new DatacenterMO(vmwareContext, datacenterName));
351356
VirtualMachineMO vmMo = Mockito.mock(VirtualMachineMO.class);
352357
HostMO hostMo = Mockito.mock(HostMO.class);
353-
Mockito.doReturn(VirtualMachinePowerState.POWERED_OFF).when(vmMo).getPowerState();
358+
Mockito.doReturn(VirtualMachinePowerState.POWERED_ON).when(vmMo).getPowerState();
354359
Mockito.doReturn(hostMo).when(vmMo).getRunningHost();
355360
Mockito.doReturn(mor).when(hostMo).getHyperHostOwnerResourcePool();
361+
UnmanagedInstanceTO instance = Mockito.mock(UnmanagedInstanceTO.class);
362+
Mockito.doReturn("CentOS").when(instance).getOperatingSystem();
363+
Mockito.doReturn("test-cluster").when(instance).getClusterName();
364+
when(VmwareHelper.getUnmanagedInstance(hostMo, vmMo)).thenReturn(instance);
356365
DatastoreMO datastoreMO = Mockito.mock(DatastoreMO.class);
357366
Mockito.doReturn(mor).when(datastoreMO).getMor();
358367
List<DatastoreMO> datastores = new ArrayList<>();
@@ -373,7 +382,7 @@ public void testCloneHypervisorVM_CloneVMFailed() throws Exception {
373382
Mockito.doReturn(mor).when(mockDatacenterMO).getVmFolder();
374383
Mockito.doReturn(mor).when(mockDatacenterMO).getMor();
375384
})) {
376-
vMwareGuru.cloneHypervisorVMOutOfBand(hostIp, vmName, params);
385+
vMwareGuru.getHypervisorVMOutOfBandAndCloneIfRequired(hostIp, vmName, params);
377386
}
378387
}
379388

@@ -400,7 +409,7 @@ public void testCloneHypervisorVM() throws Exception {
400409
DatacenterMO dataCenterMO = spy(new DatacenterMO(vmwareContext, datacenterName));
401410
VirtualMachineMO vmMo = Mockito.mock(VirtualMachineMO.class);
402411
HostMO hostMo = Mockito.mock(HostMO.class);
403-
Mockito.doReturn(VirtualMachinePowerState.POWERED_OFF).when(vmMo).getPowerState();
412+
Mockito.doReturn(VirtualMachinePowerState.POWERED_ON).when(vmMo).getPowerState();
404413
Mockito.doReturn(hostMo).when(vmMo).getRunningHost();
405414
Mockito.doReturn(mor).when(hostMo).getHyperHostOwnerResourcePool();
406415
Mockito.doReturn(mor).when(hostMo).getMor();
@@ -411,6 +420,8 @@ public void testCloneHypervisorVM() throws Exception {
411420
Mockito.doReturn(datastores).when(vmMo).getAllDatastores();
412421
Mockito.lenient().doReturn(true).when(vmMo).createFullClone(anyString(), any(ManagedObjectReference.class), any(ManagedObjectReference.class), any(ManagedObjectReference.class), any(Storage.ProvisioningType.class));
413422
UnmanagedInstanceTO instance = Mockito.mock(UnmanagedInstanceTO.class);
423+
Mockito.doReturn("CentOS").when(instance).getOperatingSystem();
424+
Mockito.doReturn("test-cluster").when(instance).getClusterName();
414425
when(VmwareHelper.getUnmanagedInstance(hostMo, vmMo)).thenReturn(instance);
415426
UnmanagedInstanceTO.Disk disk = Mockito.mock(UnmanagedInstanceTO.Disk.class);
416427
Mockito.doReturn("1").when(disk).getDiskId();
@@ -431,7 +442,7 @@ public void testCloneHypervisorVM() throws Exception {
431442
Mockito.doReturn(mor).when(mockDatacenterMO).getVmFolder();
432443
Mockito.doReturn(mor).when(mockDatacenterMO).getMor();
433444
})) {
434-
UnmanagedInstanceTO clonedVm = vMwareGuru.cloneHypervisorVMOutOfBand(hostIp, vmName, params);
445+
Pair<UnmanagedInstanceTO, Boolean> clonedVm = vMwareGuru.getHypervisorVMOutOfBandAndCloneIfRequired(hostIp, vmName, params);
435446
assertNotNull(clonedVm);
436447
}
437448
}
@@ -470,7 +481,7 @@ public void testCreateVMTemplateFileOutOfBand_NoClonedVM() throws Exception {
470481
}); MockedConstruction<DatacenterMO> ignored3 = Mockito.mockConstruction(DatacenterMO.class, withSettings().spiedInstance(dataCenterMO), (mockDatacenterMO, contextDatacenterMO) -> {
471482
Mockito.doReturn(null).when(mockDatacenterMO).findVm(vmName);
472483
})) {
473-
vMwareGuru.createVMTemplateOutOfBand(hostIp, vmName, params, dataStore);
484+
vMwareGuru.createVMTemplateOutOfBand(hostIp, vmName, params, dataStore, -1);
474485
}
475486
}
476487

@@ -496,7 +507,7 @@ public void testCreateVMTemplateFileOutOfBand() throws Exception {
496507
Mockito.doReturn(mor).when(vimClient).getDecendentMoRef(any(ManagedObjectReference.class), anyString(), anyString());
497508
DatacenterMO dataCenterMO = spy(new DatacenterMO(vmwareContext, datacenterName));
498509
VirtualMachineMO vmMo = Mockito.mock(VirtualMachineMO.class);
499-
Mockito.doNothing().when(vmMo).exportVm(anyString(), anyString(), anyBoolean(), anyBoolean());
510+
Mockito.doNothing().when(vmMo).exportVm(anyString(), anyString(), anyBoolean(), anyBoolean(), anyInt());
500511
NfsTO dataStore = Mockito.mock(NfsTO.class);
501512
Mockito.doReturn("nfs://10.1.1.4/testdir").when(dataStore).getUrl();
502513

@@ -511,7 +522,7 @@ public void testCreateVMTemplateFileOutOfBand() throws Exception {
511522
}); MockedConstruction<DatacenterMO> ignored3 = Mockito.mockConstruction(DatacenterMO.class, withSettings().spiedInstance(dataCenterMO), (mockDatacenterMO, contextDatacenterMO) -> {
512523
Mockito.doReturn(vmMo).when(mockDatacenterMO).findVm(vmName);
513524
})) {
514-
String vmTemplate = vMwareGuru.createVMTemplateOutOfBand(hostIp, vmName, params, dataStore);
525+
String vmTemplate = vMwareGuru.createVMTemplateOutOfBand(hostIp, vmName, params, dataStore, -1);
515526
assertNotNull(vmTemplate);
516527
assertTrue(UuidUtils.isUuid(vmTemplate));
517528
}

‎server/src/main/java/com/cloud/api/ApiResponseHelper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5127,7 +5127,7 @@ public UnmanagedInstanceResponse createUnmanagedInstanceResponse(UnmanagedInstan
51275127
} else if (instance.getHostName() != null) {
51285128
response.setHostName(instance.getHostName());
51295129
}
5130-
response.setPowerState(instance.getPowerState().toString());
5130+
response.setPowerState((instance.getPowerState() != null)? instance.getPowerState().toString() : UnmanagedInstanceTO.PowerState.PowerUnknown.toString());
51315131
response.setCpuCores(instance.getCpuCores());
51325132
response.setCpuSpeed(instance.getCpuSpeed());
51335133
response.setCpuCoresPerSocket(instance.getCpuCoresPerSocket());

0 commit comments

Comments
 (0)