Skip to content

Commit f5b3cb5

Browse files
SadiJrSadiJr
authored
[Veeam] enable volume attach/detach in VMs with Backup Offerings (#6581)
1 parent 2211182 commit f5b3cb5

6 files changed

Lines changed: 192 additions & 13 deletions

File tree

‎api/src/main/java/org/apache/cloudstack/backup/BackupManager.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,11 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer
5151
"300",
5252
"The backup and recovery background sync task polling interval in seconds.", true);
5353

54+
ConfigKey<Boolean> BackupEnableAttachDetachVolumes = new ConfigKey<>("Advanced", Boolean.class,
55+
"backup.enable.attach.detach.of.volumes",
56+
"false",
57+
"Enable volume attach/detach operations for VMs that are assigned to Backup Offerings.", true);
58+
5459
/**
5560
* List backup provider offerings
5661
* @param zoneId zone id

‎plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java‎

Lines changed: 47 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@
106106
import com.cloud.network.dao.PhysicalNetworkVO;
107107
import com.cloud.secstorage.CommandExecLogDao;
108108
import com.cloud.secstorage.CommandExecLogVO;
109+
import com.cloud.serializer.GsonHelper;
109110
import com.cloud.service.ServiceOfferingVO;
110111
import com.cloud.storage.DataStoreRole;
111112
import com.cloud.storage.DiskOfferingVO;
@@ -139,6 +140,7 @@
139140
import com.cloud.vm.VirtualMachineManager;
140141
import com.cloud.vm.VirtualMachineProfile;
141142
import com.cloud.vm.dao.UserVmDao;
143+
import com.cloud.vm.dao.VMInstanceDao;
142144
import com.google.gson.Gson;
143145
import com.vmware.vim25.ManagedObjectReference;
144146
import com.vmware.vim25.VirtualDevice;
@@ -153,6 +155,7 @@
153155

154156
public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Configurable {
155157
private static final Logger s_logger = Logger.getLogger(VMwareGuru.class);
158+
private static final Gson GSON = GsonHelper.getGson();
156159

157160

158161
@Inject VmwareVmImplementer vmwareVmImplementer;
@@ -161,6 +164,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
161164
@Inject ClusterDetailsDao _clusterDetailsDao;
162165
@Inject CommandExecLogDao _cmdExecLogDao;
163166
@Inject VmwareManager _vmwareMgr;
167+
@Inject VMInstanceDao vmDao;
164168
@Inject SecondaryStorageVmManager _secStorageMgr;
165169
@Inject PhysicalNetworkTrafficTypeDao _physicalNetworkTrafficTypeDao;
166170
@Inject VirtualMachineManager vmManager;
@@ -697,7 +701,7 @@ private VolumeVO createVolumeRecord(Volume.Type type, String volumeName, long zo
697701
volumeVO.setTemplateId(templateId);
698702
volumeVO.setAttached(new Date());
699703
volumeVO.setRemoved(null);
700-
volumeVO.setChainInfo(new Gson().toJson(diskInfo));
704+
volumeVO.setChainInfo(GSON.toJson(diskInfo));
701705
if (unitNumber != null) {
702706
volumeVO.setDeviceId(unitNumber.longValue());
703707
}
@@ -736,12 +740,14 @@ protected VolumeVO updateVolume(VirtualDisk disk, Map<VirtualDisk, VolumeVO> dis
736740
volume.setPath(volumeName);
737741
volume.setPoolId(poolId);
738742
VirtualMachineDiskInfo diskInfo = getDiskInfo(vmToImport, poolId, volumeName);
739-
volume.setChainInfo(new Gson().toJson(diskInfo));
743+
volume.setChainInfo(GSON.toJson(diskInfo));
740744
volume.setInstanceId(vm.getId());
741745
volume.setState(Volume.State.Ready);
742746
volume.setAttached(new Date());
743747
_volumeDao.update(volume.getId(), volume);
744748
if (volume.getRemoved() != null) {
749+
s_logger.debug(String.format("Marking volume [uuid: %s] of restored VM [%s] as non removed.", volume.getUuid(),
750+
ReflectionToStringBuilderUtils.reflectOnlySelectedFields(vm, "uuid", "instanceName")));
745751
_volumeDao.unremove(volume.getId());
746752
if (vm.getType() == Type.User) {
747753
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), volume.getDiskOfferingId(), null, volume.getSize(),
@@ -812,6 +818,23 @@ private VolumeVO createVolume(VirtualDisk disk, VirtualMachineMO vmToImport, lon
812818
return createVolumeRecord(type, volumeName, zoneId, domainId, accountId, diskOfferingId, provisioningType, size, instanceId, poolId, templateId, unitNumber, diskInfo);
813819
}
814820

821+
protected String createVolumeInfoFromVolumes(List<VolumeVO> vmVolumes) {
822+
try {
823+
List<Backup.VolumeInfo> list = new ArrayList<>();
824+
for (VolumeVO vol : vmVolumes) {
825+
list.add(new Backup.VolumeInfo(vol.getUuid(), vol.getPath(), vol.getVolumeType(), vol.getSize()));
826+
}
827+
return GSON.toJson(list.toArray(), Backup.VolumeInfo[].class);
828+
} catch (Exception e) {
829+
if (CollectionUtils.isEmpty(vmVolumes) || vmVolumes.get(0).getInstanceId() == null) {
830+
s_logger.error(String.format("Failed to create VolumeInfo of VM [id: null] volumes due to: [%s].", e.getMessage()), e);
831+
} else {
832+
s_logger.error(String.format("Failed to create VolumeInfo of VM [id: %s] volumes due to: [%s].", vmVolumes.get(0).getInstanceId(), e.getMessage()), e);
833+
}
834+
throw e;
835+
}
836+
}
837+
815838
/**
816839
* Get physical network ID from zoneId and Vmware label
817840
*/
@@ -919,11 +942,25 @@ private Map<VirtualDisk, VolumeVO> getDisksMapping(Backup backup, List<VirtualDi
919942
if (vm == null) {
920943
throw new CloudRuntimeException("Failed to find the volumes details from the VM backup");
921944
}
945+
922946
List<Backup.VolumeInfo> backedUpVolumes = vm.getBackupVolumeList();
923947
Map<String, Boolean> usedVols = new HashMap<>();
924948
Map<VirtualDisk, VolumeVO> map = new HashMap<>();
925949

926950
for (Backup.VolumeInfo backedUpVol : backedUpVolumes) {
951+
VolumeVO volumeExtra = _volumeDao.findByUuid(backedUpVol.getUuid());
952+
if (volumeExtra != null) {
953+
s_logger.debug(String.format("Marking volume [id: %s] of VM [%s] as removed for the backup process.", backedUpVol.getUuid(), ReflectionToStringBuilderUtils.reflectOnlySelectedFields(vm, "uuid", "instanceName")));
954+
_volumeDao.remove(volumeExtra.getId());
955+
956+
if (vm.getType() == Type.User) {
957+
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_DETACH, volumeExtra.getAccountId(), volumeExtra.getDataCenterId(), volumeExtra.getId(),
958+
volumeExtra.getName(), volumeExtra.getDiskOfferingId(), null, volumeExtra.getSize(), Volume.class.getName(),
959+
volumeExtra.getUuid(), volumeExtra.isDisplayVolume());
960+
_resourceLimitService.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.volume, volumeExtra.isDisplayVolume());
961+
_resourceLimitService.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.primary_storage, volumeExtra.isDisplayVolume(), volumeExtra.getSize());
962+
}
963+
}
927964
for (VirtualDisk disk : virtualDisks) {
928965
if (!map.containsKey(disk) && backedUpVol.getSize().equals(disk.getCapacityInBytes()) && !usedVols.containsKey(backedUpVol.getUuid())) {
929966
String volId = backedUpVol.getUuid();
@@ -1022,7 +1059,8 @@ public VirtualMachine importVirtualMachineFromBackup(long zoneId, long domainId,
10221059
return vm;
10231060
}
10241061

1025-
@Override public boolean attachRestoredVolumeToVirtualMachine(long zoneId, String location, Backup.VolumeInfo volumeInfo, VirtualMachine vm, long poolId, Backup backup)
1062+
@Override
1063+
public boolean attachRestoredVolumeToVirtualMachine(long zoneId, String location, Backup.VolumeInfo volumeInfo, VirtualMachine vm, long poolId, Backup backup)
10261064
throws Exception {
10271065
DatacenterMO dcMo = getDatacenterMO(zoneId);
10281066
VirtualMachineMO vmRestored = findVM(dcMo, location);
@@ -1061,6 +1099,12 @@ public VirtualMachine importVirtualMachineFromBackup(long zoneId, long domainId,
10611099
}
10621100
createVolume(attachedDisk, vmMo, vm.getDomainId(), vm.getDataCenterId(), vm.getAccountId(), vm.getId(), poolId, vm.getTemplateId(), backup, false);
10631101

1102+
if (vm.getBackupOfferingId() == null) {
1103+
return true;
1104+
}
1105+
VMInstanceVO vmVO = (VMInstanceVO)vm;
1106+
vmVO.setBackupVolumes(createVolumeInfoFromVolumes(_volumeDao.findByInstance(vm.getId())));
1107+
vmDao.update(vmVO.getId(), vmVO);
10641108
return true;
10651109
}
10661110

‎plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/guru/VMwareGuruTest.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,11 @@
2121
import com.cloud.dc.ClusterDetailsDao;
2222
import com.cloud.host.HostVO;
2323
import com.cloud.host.dao.HostDao;
24+
import com.cloud.storage.Storage.ProvisioningType;
2425
import com.cloud.storage.StoragePool;
2526
import com.cloud.storage.StoragePoolHostVO;
2627
import com.cloud.storage.Volume;
28+
import com.cloud.storage.VolumeVO;
2729
import com.cloud.storage.dao.StoragePoolHostDao;
2830
import com.cloud.utils.Pair;
2931
import com.cloud.vm.VirtualMachine;
@@ -44,6 +46,8 @@
4446
import org.springframework.test.context.ContextConfiguration;
4547
import org.springframework.test.context.support.AnnotationConfigContextLoader;
4648

49+
import static org.junit.Assert.assertEquals;
50+
4751
import java.util.ArrayList;
4852
import java.util.HashMap;
4953
import java.util.List;
@@ -116,4 +120,33 @@ public void finalizeMigrateForLocalStorageToHaveTargetHostGuid(){
116120
Assert.assertEquals("HostSystem:host-a@x.x.x.x", migrateVmToPoolCommand.getHostGuidInTargetCluster());
117121
}
118122

123+
@Test
124+
public void createVolumeInfoFromVolumesTestEmptyVolumeListReturnEmptyArray() {
125+
String volumeInfo = vMwareGuru.createVolumeInfoFromVolumes(new ArrayList<>());
126+
assertEquals("[]", volumeInfo);
127+
}
128+
129+
@Test(expected = NullPointerException.class)
130+
public void createVolumeInfoFromVolumesTestNullVolume() {
131+
vMwareGuru.createVolumeInfoFromVolumes(null);
132+
}
133+
134+
@Test
135+
public void createVolumeInfoFromVolumesTestCorrectlyConvertOfVolumes() {
136+
List<VolumeVO> volumesToTest = new ArrayList<>();
137+
138+
VolumeVO root = new VolumeVO("test", 1l, 1l, 1l, 1l, 1l, "test", "/root/dir", ProvisioningType.THIN, 555l, Volume.Type.ROOT);
139+
String rootUuid = root.getUuid();
140+
141+
VolumeVO data = new VolumeVO("test", 1l, 1l, 1l, 1l, 1l, "test", "/root/dir/data", ProvisioningType.THIN, 1111000l, Volume.Type.DATADISK);
142+
String dataUuid = data.getUuid();
143+
144+
volumesToTest.add(root);
145+
volumesToTest.add(data);
146+
147+
String result = vMwareGuru.createVolumeInfoFromVolumes(volumesToTest);
148+
String expected = String.format("[{\"uuid\":\"%s\",\"type\":\"ROOT\",\"size\":555,\"path\":\"/root/dir\"},{\"uuid\":\"%s\",\"type\":\"DATADISK\",\"size\":1111000,\"path\":\"/root/dir/data\"}]", rootUuid, dataUuid);
149+
150+
assertEquals(expected, result);
151+
}
119152
}

‎server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java‎

Lines changed: 41 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@
4545
import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd;
4646
import org.apache.cloudstack.api.command.user.volume.UploadVolumeCmd;
4747
import org.apache.cloudstack.api.response.GetUploadParamsResponse;
48+
import org.apache.cloudstack.backup.Backup;
49+
import org.apache.cloudstack.backup.BackupManager;
4850
import org.apache.cloudstack.context.CallContext;
4951
import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
5052
import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo;
@@ -97,9 +99,8 @@
9799
import org.apache.commons.collections.CollectionUtils;
98100
import org.apache.commons.collections.MapUtils;
99101
import org.apache.commons.lang3.ObjectUtils;
102+
import org.apache.commons.lang3.BooleanUtils;
100103
import org.apache.commons.lang3.StringUtils;
101-
import org.apache.commons.lang3.builder.ToStringBuilder;
102-
import org.apache.commons.lang3.builder.ToStringStyle;
103104
import org.apache.log4j.Logger;
104105
import org.jetbrains.annotations.NotNull;
105106
import org.jetbrains.annotations.Nullable;
@@ -2477,6 +2478,36 @@ private void checkDeviceId(Long deviceId, VolumeInfo volumeToAttach, UserVmVO vm
24772478
return volumeToAttach;
24782479
}
24792480

2481+
protected void validateIfVmHasBackups(UserVmVO vm, boolean attach) {
2482+
if ((vm.getBackupOfferingId() == null || CollectionUtils.isEmpty(vm.getBackupVolumeList())) || BooleanUtils.isTrue(BackupManager.BackupEnableAttachDetachVolumes.value())) {
2483+
return;
2484+
}
2485+
String errorMsg = String.format("Unable to detach volume, cannot detach volume from a VM that has backups. First remove the VM from the backup offering or "
2486+
+ "set the global configuration '%s' to true.", BackupManager.BackupEnableAttachDetachVolumes.key());
2487+
if (attach) {
2488+
errorMsg = String.format("Unable to attach volume, please specify a VM that does not have any backups or set the global configuration "
2489+
+ "'%s' to true.", BackupManager.BackupEnableAttachDetachVolumes.key());
2490+
}
2491+
throw new InvalidParameterValueException(errorMsg);
2492+
}
2493+
2494+
protected String createVolumeInfoFromVolumes(List<VolumeVO> vmVolumes) {
2495+
try {
2496+
List<Backup.VolumeInfo> list = new ArrayList<>();
2497+
for (VolumeVO vol : vmVolumes) {
2498+
list.add(new Backup.VolumeInfo(vol.getUuid(), vol.getPath(), vol.getVolumeType(), vol.getSize()));
2499+
}
2500+
return GsonHelper.getGson().toJson(list.toArray(), Backup.VolumeInfo[].class);
2501+
} catch (Exception e) {
2502+
if (CollectionUtils.isEmpty(vmVolumes) || vmVolumes.get(0).getInstanceId() == null) {
2503+
s_logger.error(String.format("Failed to create VolumeInfo of VM [id: null] volumes due to: [%s].", e.getMessage()), e);
2504+
} else {
2505+
s_logger.error(String.format("Failed to create VolumeInfo of VM [id: %s] volumes due to: [%s].", vmVolumes.get(0).getInstanceId(), e.getMessage()), e);
2506+
}
2507+
throw e;
2508+
}
2509+
}
2510+
24802511
@Override
24812512
@ActionEvent(eventType = EventTypes.EVENT_VOLUME_UPDATE, eventDescription = "updating volume", async = true)
24822513
public Volume updateVolume(long volumeId, String path, String state, Long storageId, Boolean displayVolume,
@@ -2649,13 +2680,11 @@ public Volume detachVolumeFromVM(DetachVolumeCmd cmmd) {
26492680

26502681
// Don't allow detach if target VM has associated VM snapshots
26512682
List<VMSnapshotVO> vmSnapshots = _vmSnapshotDao.findByVm(vmId);
2652-
if (vmSnapshots.size() > 0) {
2683+
if (CollectionUtils.isNotEmpty(vmSnapshots)) {
26532684
throw new InvalidParameterValueException("Unable to detach volume, please specify a VM that does not have VM snapshots");
26542685
}
26552686

2656-
if (vm.getBackupOfferingId() != null || vm.getBackupVolumeList().size() > 0) {
2657-
throw new InvalidParameterValueException("Unable to detach volume, cannot detach volume from a VM that has backups. First remove the VM from the backup offering.");
2658-
}
2687+
validateIfVmHasBackups(vm, false);
26592688

26602689
AsyncJobExecutionContext asyncExecutionContext = AsyncJobExecutionContext.getCurrentExecutionContext();
26612690
if (asyncExecutionContext != null) {
@@ -2705,6 +2734,10 @@ public Volume detachVolumeFromVM(DetachVolumeCmd cmmd) {
27052734
vol = _volsDao.findById((Long)jobResult);
27062735
}
27072736
}
2737+
if (vm.getBackupOfferingId() != null) {
2738+
vm.setBackupVolumes(createVolumeInfoFromVolumes(_volsDao.findByInstance(vm.getId())));
2739+
_vmInstanceDao.update(vm.getId(), vm);
2740+
}
27082741
return vol;
27092742
}
27102743
}
@@ -2846,6 +2879,7 @@ private Volume orchestrateDetachVolumeFromVM(long vmId, long volumeId) {
28462879
if (volumePool != null && hostId != null) {
28472880
handleTargetsForVMware(hostId, volumePool.getHostAddress(), volumePool.getPort(), volume.get_iScsiName());
28482881
}
2882+
28492883
return _volsDao.findById(volumeId);
28502884
} else {
28512885

@@ -3025,7 +3059,7 @@ public Volume migrateVolume(MigrateVolumeCmd cmd) {
30253059
snapshotHelper.checkKvmVolumeSnapshotsOnlyInPrimaryStorage(vol, _volsDao.getHypervisorType(vol.getId()));
30263060
} catch (CloudRuntimeException ex) {
30273061
throw new CloudRuntimeException(String.format("Unable to migrate %s to the destination storage pool [%s] due to [%s]", vol,
3028-
new ToStringBuilder(destPool, ToStringStyle.JSON_STYLE).append("uuid", destPool.getUuid()).append("name", destPool.getName()).toString(), ex.getMessage()), ex);
3062+
ReflectionToStringBuilderUtils.reflectOnlySelectedFields(destPool, "uuid", "name"), ex.getMessage()), ex);
30293063
}
30303064

30313065
DiskOfferingVO diskOffering = _diskOfferingDao.findById(vol.getDiskOfferingId());

‎server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -587,6 +587,7 @@ public boolean restoreBackup(final Long backupId) {
587587
if (offering == null) {
588588
throw new CloudRuntimeException("Failed to find backup offering of the VM backup");
589589
}
590+
590591
final BackupProvider backupProvider = getBackupProvider(offering.getProvider());
591592
if (!backupProvider.restoreVMFromBackup(vm, backup)) {
592593
throw new CloudRuntimeException("Error restoring VM from backup ID " + backup.getId());
@@ -619,7 +620,7 @@ public boolean restoreBackupVolumeAndAttachToVM(final String backedUpVolumeUuid,
619620
final VMInstanceVO vm = findVmById(vmId);
620621
accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm);
621622

622-
if (vm.getBackupOfferingId() != null) {
623+
if (vm.getBackupOfferingId() != null && !BackupEnableAttachDetachVolumes.value()) {
623624
throw new CloudRuntimeException("The selected VM has backups, cannot restore and attach volume to the VM.");
624625
}
625626

@@ -841,7 +842,8 @@ public ConfigKey<?>[] getConfigKeys() {
841842
return new ConfigKey[]{
842843
BackupFrameworkEnabled,
843844
BackupProviderPlugin,
844-
BackupSyncPollingInterval
845+
BackupSyncPollingInterval,
846+
BackupEnableAttachDetachVolumes
845847
};
846848
}
847849

0 commit comments

Comments
 (0)