Skip to content

Commit 42536c1

Browse files
nvazqueznvazquez
committed
[VMware to KVM] Cleanup leftover migrated volumes in case of migration failures (#13151)
1 parent a951ac6 commit 42536c1

6 files changed

Lines changed: 458 additions & 233 deletions

File tree

‎core/src/main/java/com/cloud/agent/api/CleanupConvertedInstanceDisksCommand.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
//
2+
// Licensed to the Apache Software Foundation (ASF) under one
3+
// or more contributor license agreements. See the NOTICE file
4+
// distributed with this work for additional information
5+
// regarding copyright ownership. The ASF licenses this file
6+
// to you under the Apache License, Version 2.0 (the
7+
// "License"); you may not use this file except in compliance
8+
// with the License. You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing,
13+
// software distributed under the License is distributed on an
14+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
// KIND, either express or implied. See the License for the
16+
// specific language governing permissions and limitations
17+
// under the License.
18+
//
19+
20+
package com.cloud.agent.api;
21+
22+
import com.cloud.agent.api.to.DataStoreTO;
23+
24+
/**
25+
* This command is used to cleanup the converted instance disks from the storage pool: vmVolumesStore and the prefix: vmVolumesPrefix.
26+
*/
27+
public class CleanupConvertedInstanceDisksCommand extends Command {
28+
29+
private DataStoreTO vmVolumesStore;
30+
private String vmVolumesPrefix;
31+
32+
public CleanupConvertedInstanceDisksCommand(DataStoreTO vmVolumesStore, String vmVolumesPrefix) {
33+
this.vmVolumesStore = vmVolumesStore;
34+
this.vmVolumesPrefix = vmVolumesPrefix;
35+
}
36+
37+
public DataStoreTO getVmVolumesStore() {
38+
return vmVolumesStore;
39+
}
40+
41+
public void setVmVolumesStore(DataStoreTO vmVolumesStore) {
42+
this.vmVolumesStore = vmVolumesStore;
43+
}
44+
45+
public String getVmVolumesPrefix() {
46+
return vmVolumesPrefix;
47+
}
48+
49+
public void setVmVolumesPrefix(String vmVolumesPrefix) {
50+
this.vmVolumesPrefix = vmVolumesPrefix;
51+
}
52+
53+
@Override
54+
public boolean executeInSequence() {
55+
return false;
56+
}
57+
}

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtBaseConvertCommandWrapper.java‎

Lines changed: 283 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,283 @@
1+
//
2+
// Licensed to the Apache Software Foundation (ASF) under one
3+
// or more contributor license agreements. See the NOTICE file
4+
// distributed with this work for additional information
5+
// regarding copyright ownership. The ASF licenses this file
6+
// to you under the Apache License, Version 2.0 (the
7+
// "License"); you may not use this file except in compliance
8+
// with the License. You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing,
13+
// software distributed under the License is distributed on an
14+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
// KIND, either express or implied. See the License for the
16+
// specific language governing permissions and limitations
17+
// under the License.
18+
//
19+
package com.cloud.hypervisor.kvm.resource.wrapper;
20+
21+
import com.cloud.agent.api.Answer;
22+
import com.cloud.agent.api.Command;
23+
import com.cloud.agent.api.to.DataStoreTO;
24+
import com.cloud.agent.api.to.NfsTO;
25+
import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser;
26+
import com.cloud.hypervisor.kvm.resource.LibvirtVMDef;
27+
import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk;
28+
import com.cloud.hypervisor.kvm.storage.KVMStoragePool;
29+
import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
30+
import com.cloud.resource.CommandWrapper;
31+
import com.cloud.resource.ServerResource;
32+
import com.cloud.storage.Storage;
33+
import com.cloud.utils.FileUtil;
34+
import com.cloud.utils.Pair;
35+
import com.cloud.utils.exception.CloudRuntimeException;
36+
import com.cloud.utils.script.Script;
37+
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
38+
import org.apache.cloudstack.vm.UnmanagedInstanceTO;
39+
import org.apache.commons.collections.CollectionUtils;
40+
import org.apache.commons.io.IOUtils;
41+
import org.apache.commons.lang3.StringUtils;
42+
43+
import java.io.BufferedInputStream;
44+
import java.io.File;
45+
import java.io.FileInputStream;
46+
import java.io.IOException;
47+
import java.io.InputStream;
48+
import java.nio.charset.Charset;
49+
import java.util.ArrayList;
50+
import java.util.List;
51+
import java.util.UUID;
52+
import java.util.stream.Collectors;
53+
54+
public abstract class LibvirtBaseConvertCommandWrapper <T extends Command, A extends Answer, R extends ServerResource> extends CommandWrapper<T, A, R> {
55+
56+
protected KVMStoragePool getTemporaryStoragePool(DataStoreTO conversionTemporaryLocation, KVMStoragePoolManager storagePoolMgr) {
57+
if (conversionTemporaryLocation instanceof NfsTO) {
58+
NfsTO nfsTO = (NfsTO) conversionTemporaryLocation;
59+
return storagePoolMgr.getStoragePoolByURI(nfsTO.getUrl());
60+
} else {
61+
PrimaryDataStoreTO primaryDataStoreTO = (PrimaryDataStoreTO) conversionTemporaryLocation;
62+
return storagePoolMgr.getStoragePool(primaryDataStoreTO.getPoolType(), primaryDataStoreTO.getUuid());
63+
}
64+
}
65+
66+
protected List<KVMPhysicalDisk> getTemporaryDisksFromParsedXml(KVMStoragePool pool, LibvirtDomainXMLParser xmlParser,
67+
String convertedBasePath,
68+
String conversionPoolPath, String vmVolumesPrefix) {
69+
List<LibvirtVMDef.DiskDef> disksDefs = xmlParser.getDisks();
70+
disksDefs = disksDefs.stream().filter(x -> x.getDiskType() == LibvirtVMDef.DiskDef.DiskType.FILE &&
71+
x.getDeviceType() == LibvirtVMDef.DiskDef.DeviceType.DISK).collect(Collectors.toList());
72+
if (CollectionUtils.isEmpty(disksDefs)) {
73+
String err = String.format("Cannot find any disk defined on the converted XML domain %s.xml, " +
74+
"checking disks at: %s with prefix: %s", convertedBasePath, conversionPoolPath, vmVolumesPrefix);
75+
logger.warn(err);
76+
return getTemporaryDisksWithPrefixFromTemporaryPool(pool, conversionPoolPath, vmVolumesPrefix);
77+
}
78+
sanitizeDisksPath(disksDefs);
79+
return getPhysicalDisksFromDefPaths(disksDefs, pool);
80+
}
81+
82+
private List<KVMPhysicalDisk> getPhysicalDisksFromDefPaths(List<LibvirtVMDef.DiskDef> disksDefs, KVMStoragePool pool) {
83+
List<KVMPhysicalDisk> disks = new ArrayList<>();
84+
for (LibvirtVMDef.DiskDef diskDef : disksDefs) {
85+
KVMPhysicalDisk physicalDisk = pool.getPhysicalDisk(diskDef.getDiskPath());
86+
disks.add(physicalDisk);
87+
}
88+
return disks;
89+
}
90+
91+
protected List<KVMPhysicalDisk> getTemporaryDisksWithPrefixFromTemporaryPool(KVMStoragePool pool, String path, String prefix) {
92+
String msg = String.format("Could not parse correctly the converted XML domain, checking for disks on %s with prefix %s", path, prefix);
93+
logger.info(msg);
94+
pool.refresh();
95+
List<KVMPhysicalDisk> disksWithPrefix = pool.listPhysicalDisks()
96+
.stream()
97+
.filter(x -> x.getName().startsWith(prefix) && !x.getName().endsWith(".xml"))
98+
.collect(Collectors.toList());
99+
if (CollectionUtils.isEmpty(disksWithPrefix)) {
100+
msg = String.format("Could not find any converted disk with prefix %s on temporary location %s", prefix, path);
101+
logger.error(msg);
102+
throw new CloudRuntimeException(msg);
103+
}
104+
return disksWithPrefix;
105+
}
106+
107+
protected void cleanupDisksAndDomainFromTemporaryLocation(List<KVMPhysicalDisk> disks,
108+
KVMStoragePool temporaryStoragePool,
109+
String temporaryConvertUuid, boolean xmlExists) {
110+
for (KVMPhysicalDisk disk : disks) {
111+
logger.info(String.format("Cleaning up temporary disk %s after conversion from temporary location", disk.getName()));
112+
temporaryStoragePool.deletePhysicalDisk(disk.getName(), Storage.ImageFormat.QCOW2);
113+
}
114+
if (xmlExists) {
115+
logger.info(String.format("Cleaning up temporary domain %s after conversion from temporary location", temporaryConvertUuid));
116+
FileUtil.deleteFiles(temporaryStoragePool.getLocalPath(), temporaryConvertUuid, ".xml");
117+
}
118+
}
119+
120+
protected void sanitizeDisksPath(List<LibvirtVMDef.DiskDef> disks) {
121+
for (LibvirtVMDef.DiskDef disk : disks) {
122+
String[] diskPathParts = disk.getDiskPath().split("/");
123+
String relativePath = diskPathParts[diskPathParts.length - 1];
124+
disk.setDiskPath(relativePath);
125+
}
126+
}
127+
128+
protected List<KVMPhysicalDisk> moveTemporaryDisksToDestination(List<KVMPhysicalDisk> temporaryDisks,
129+
List<String> destinationStoragePools,
130+
KVMStoragePoolManager storagePoolMgr) {
131+
List<KVMPhysicalDisk> targetDisks = new ArrayList<>();
132+
if (temporaryDisks.size() != destinationStoragePools.size()) {
133+
String warn = String.format("Discrepancy between the converted instance disks (%s) " +
134+
"and the expected number of disks (%s)", temporaryDisks.size(), destinationStoragePools.size());
135+
logger.warn(warn);
136+
}
137+
for (int i = 0; i < temporaryDisks.size(); i++) {
138+
String poolPath = destinationStoragePools.get(i);
139+
KVMStoragePool destinationPool = storagePoolMgr.getStoragePool(Storage.StoragePoolType.NetworkFilesystem, poolPath);
140+
if (destinationPool == null) {
141+
String err = String.format("Could not find a storage pool by URI: %s", poolPath);
142+
logger.error(err);
143+
continue;
144+
}
145+
if (destinationPool.getType() != Storage.StoragePoolType.NetworkFilesystem) {
146+
String err = String.format("Storage pool by URI: %s is not an NFS storage", poolPath);
147+
logger.error(err);
148+
continue;
149+
}
150+
KVMPhysicalDisk sourceDisk = temporaryDisks.get(i);
151+
if (logger.isDebugEnabled()) {
152+
String msg = String.format("Trying to copy converted instance disk number %s from the temporary location %s" +
153+
" to destination storage pool %s", i, sourceDisk.getPool().getLocalPath(), destinationPool.getUuid());
154+
logger.debug(msg);
155+
}
156+
157+
String destinationName = UUID.randomUUID().toString();
158+
159+
try {
160+
if (destinationPool.getAvailable() < sourceDisk.getSize()) {
161+
String msg = String.format("Not enough space on destination pool %s (%s bytes) to copy disk %s (size %s)",
162+
destinationPool.getUuid(), destinationPool.getAvailable(), sourceDisk.getName(), sourceDisk.getSize());
163+
logger.error(msg);
164+
throw new CloudRuntimeException(msg);
165+
}
166+
167+
KVMPhysicalDisk destinationDisk = storagePoolMgr.copyPhysicalDisk(sourceDisk, destinationName, destinationPool, 7200 * 1000);
168+
targetDisks.add(destinationDisk);
169+
} catch (Exception e) {
170+
String err = String.format("Error copying converted instance disk number %s from the temporary location %s" +
171+
" to destination storage pool %s: %s", i, sourceDisk.getPool().getLocalPath(), destinationPool.getUuid(), e.getMessage());
172+
logger.error(err, e);
173+
cleanupMovedDisksOnDestinationPool(targetDisks);
174+
return null;
175+
}
176+
}
177+
return targetDisks;
178+
}
179+
180+
private void cleanupMovedDisksOnDestinationPool(List<KVMPhysicalDisk> targetDisks) {
181+
if (CollectionUtils.isEmpty(targetDisks)) {
182+
return;
183+
}
184+
for (KVMPhysicalDisk disk : targetDisks) {
185+
logger.info(String.format("Cleaning up disk %s from pool %s after conversion", disk.getName(), disk.getPool().getUuid()));
186+
disk.getPool().deletePhysicalDisk(disk.getName(), Storage.ImageFormat.QCOW2);
187+
}
188+
}
189+
190+
protected UnmanagedInstanceTO getConvertedUnmanagedInstance(String baseName,
191+
List<KVMPhysicalDisk> vmDisks,
192+
LibvirtDomainXMLParser xmlParser) {
193+
UnmanagedInstanceTO instanceTO = new UnmanagedInstanceTO();
194+
instanceTO.setName(baseName);
195+
instanceTO.setDisks(getUnmanagedInstanceDisks(vmDisks, xmlParser));
196+
instanceTO.setNics(getUnmanagedInstanceNics(xmlParser));
197+
return instanceTO;
198+
}
199+
200+
private List<UnmanagedInstanceTO.Nic> getUnmanagedInstanceNics(LibvirtDomainXMLParser xmlParser) {
201+
List<UnmanagedInstanceTO.Nic> nics = new ArrayList<>();
202+
if (xmlParser != null) {
203+
List<LibvirtVMDef.InterfaceDef> interfaces = xmlParser.getInterfaces();
204+
for (LibvirtVMDef.InterfaceDef interfaceDef : interfaces) {
205+
UnmanagedInstanceTO.Nic nic = new UnmanagedInstanceTO.Nic();
206+
nic.setMacAddress(interfaceDef.getMacAddress());
207+
nic.setNicId(interfaceDef.getBrName());
208+
nic.setAdapterType(interfaceDef.getModel().toString());
209+
nics.add(nic);
210+
}
211+
}
212+
return nics;
213+
}
214+
215+
protected List<UnmanagedInstanceTO.Disk> getUnmanagedInstanceDisks(List<KVMPhysicalDisk> vmDisks, LibvirtDomainXMLParser xmlParser) {
216+
List<UnmanagedInstanceTO.Disk> instanceDisks = new ArrayList<>();
217+
List<LibvirtVMDef.DiskDef> diskDefs = xmlParser != null ? xmlParser.getDisks() : null;
218+
for (int i = 0; i< vmDisks.size(); i++) {
219+
KVMPhysicalDisk physicalDisk = vmDisks.get(i);
220+
KVMStoragePool storagePool = physicalDisk.getPool();
221+
UnmanagedInstanceTO.Disk disk = new UnmanagedInstanceTO.Disk();
222+
disk.setPosition(i);
223+
Pair<String, String> storagePoolHostAndPath = getNfsStoragePoolHostAndPath(storagePool);
224+
disk.setDatastoreHost(storagePoolHostAndPath.first());
225+
disk.setDatastorePath(storagePoolHostAndPath.second());
226+
disk.setDatastoreName(storagePool.getUuid());
227+
disk.setDatastoreType(storagePool.getType().name());
228+
disk.setCapacity(physicalDisk.getVirtualSize());
229+
disk.setFileBaseName(physicalDisk.getName());
230+
if (CollectionUtils.isNotEmpty(diskDefs)) {
231+
LibvirtVMDef.DiskDef diskDef = diskDefs.get(i);
232+
disk.setController(diskDef.getBusType() != null ? diskDef.getBusType().toString() : LibvirtVMDef.DiskDef.DiskBus.VIRTIO.toString());
233+
} else {
234+
// If the job is finished but we cannot parse the XML, the guest VM can use the virtio driver
235+
disk.setController(LibvirtVMDef.DiskDef.DiskBus.VIRTIO.toString());
236+
}
237+
instanceDisks.add(disk);
238+
}
239+
return instanceDisks;
240+
}
241+
242+
protected Pair<String, String> getNfsStoragePoolHostAndPath(KVMStoragePool storagePool) {
243+
String sourceHostIp = null;
244+
String sourcePath = null;
245+
List<String[]> commands = new ArrayList<>();
246+
commands.add(new String[]{Script.getExecutableAbsolutePath("mount")});
247+
commands.add(new String[]{Script.getExecutableAbsolutePath("grep"), storagePool.getLocalPath()});
248+
String storagePoolMountPoint = Script.executePipedCommands(commands, 0).second();
249+
logger.debug(String.format("NFS Storage pool: %s - local path: %s, mount point: %s", storagePool.getUuid(), storagePool.getLocalPath(), storagePoolMountPoint));
250+
if (StringUtils.isNotEmpty(storagePoolMountPoint)) {
251+
String[] res = storagePoolMountPoint.strip().split(" ");
252+
res = res[0].split(":");
253+
if (res.length > 1) {
254+
sourceHostIp = res[0].strip();
255+
sourcePath = res[1].strip();
256+
}
257+
}
258+
return new Pair<>(sourceHostIp, sourcePath);
259+
}
260+
261+
protected LibvirtDomainXMLParser parseMigratedVMXmlDomain(String installPath) throws IOException {
262+
String xmlPath = String.format("%s.xml", installPath);
263+
if (!new File(xmlPath).exists()) {
264+
String err = String.format("Conversion failed. Unable to find the converted XML domain, expected %s", xmlPath);
265+
logger.error(err);
266+
throw new CloudRuntimeException(err);
267+
}
268+
String xml;
269+
try (InputStream is = new BufferedInputStream(new FileInputStream(xmlPath))) {
270+
xml = IOUtils.toString(is, Charset.defaultCharset());
271+
}
272+
final LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser();
273+
try {
274+
parser.parseDomainXML(xml);
275+
return parser;
276+
} catch (RuntimeException e) {
277+
String err = String.format("Error parsing the converted instance XML domain at %s: %s", xmlPath, e.getMessage());
278+
logger.error(err, e);
279+
logger.debug(xml);
280+
return null;
281+
}
282+
}
283+
}

0 commit comments

Comments
 (0)