Skip to content

Commit b3c22df

Browse files
Forward merge branch '4.11' to master
ConfigDrive fixes: CLOUDSTACK-10288, CLOUDSTACK-10289 (#2566) CLOUDSTACK-9677: Adding storage policy support for swift as secondary storage (#2412)
2 parents 4c42aaf + 8520324 commit b3c22df

25 files changed

Lines changed: 1485 additions & 1183 deletions

File tree

api/src/main/java/com/cloud/agent/api/to/SwiftTO.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,17 +26,19 @@ public class SwiftTO implements DataStoreTO, SwiftUtil.SwiftClientCfg {
2626

2727
String userName;
2828
String key;
29+
String storagePolicy;
2930
private static final String pathSeparator = "/";
3031

3132
public SwiftTO() {
3233
}
3334

34-
public SwiftTO(Long id, String url, String account, String userName, String key) {
35+
public SwiftTO(Long id, String url, String account, String userName, String key, String storagePolicy) {
3536
this.id = id;
3637
this.url = url;
3738
this.account = account;
3839
this.userName = userName;
3940
this.key = key;
41+
this.storagePolicy = storagePolicy;
4042
}
4143

4244
public Long getId() {
@@ -63,6 +65,11 @@ public String getKey() {
6365
return key;
6466
}
6567

68+
@Override
69+
public String getStoragePolicy() {
70+
return this.storagePolicy;
71+
}
72+
6673
@Override
6774
public DataStoreRole getRole() {
6875
return DataStoreRole.Image;

api/src/main/java/com/cloud/network/NetworkModel.java

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ public interface NetworkModel {
5959
String SERVICE_OFFERING_FILE = "service-offering";
6060
String AVAILABILITY_ZONE_FILE = "availability-zone";
6161
String LOCAL_HOSTNAME_FILE = "local-hostname";
62+
String LOCAL_IPV4_FILE = "local-ipv4";
63+
String PUBLIC_HOSTNAME_FILE = "public-hostname";
64+
String PUBLIC_IPV4_FILE = "public-ipv4";
6265
String INSTANCE_ID_FILE = "instance-id";
6366
String VM_ID_FILE = "vm-id";
6467
String PUBLIC_KEYS_FILE = "public-keys";
@@ -309,8 +312,8 @@ public interface NetworkModel {
309312

310313
boolean getNetworkEgressDefaultPolicy(Long networkId);
311314

312-
List<String[]> generateVmData(String userData, String serviceOffering, String zoneName,
313-
String vmName, long vmId, String publicKey, String password, Boolean isWindows);
315+
List<String[]> generateVmData(String userData, String serviceOffering, long datacenterId,
316+
String vmName, long vmId, String vmUuid, String guestIpAddress, String publicKey, String password, Boolean isWindows);
314317

315318
String getValidNetworkCidr(Network guestNetwork);
316319

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -289,6 +289,7 @@ public class ApiConstants {
289289
public static final String STATE = "state";
290290
public static final String STATUS = "status";
291291
public static final String STORAGE_TYPE = "storagetype";
292+
public static final String STORAGE_POLICY = "storagepolicy";
292293
public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled";
293294
public static final String STORAGE_CAPABILITIES = "storagecapabilities";
294295
public static final String SYSTEM_VM_TYPE = "systemvmtype";

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2542,16 +2542,15 @@ private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHo
25422542
if (defaultNic != null) {
25432543
UserVmVO userVm = _userVmDao.findById(vm.getId());
25442544
Map<String, String> details = _vmDetailsDao.listDetailsKeyPairs(vm.getId());
2545-
vm.setDetails(details);
2545+
userVm.setDetails(details);
25462546

25472547
Network network = _networkModel.getNetwork(defaultNic.getNetworkId());
25482548
if (_networkModel.isSharedNetworkWithoutServices(network.getId())) {
25492549
final String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()).getDisplayText();
2550-
final String zoneName = _dcDao.findById(vm.getDataCenterId()).getName();
25512550
boolean isWindows = _guestOSCategoryDao.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
25522551

2553-
vmData = _networkModel.generateVmData(userVm.getUserData(), serviceOffering, zoneName, vm.getInstanceName(), vm.getId(),
2554-
(String) profile.getParameter(VirtualMachineProfile.Param.VmSshPubKey), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows);
2552+
vmData = _networkModel.generateVmData(userVm.getUserData(), serviceOffering, vm.getDataCenterId(), vm.getInstanceName(), vm.getId(),
2553+
vm.getUuid(), defaultNic.getMacAddress(), userVm.getDetail("SSH.PublicKey"), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows);
25552554
String vmName = vm.getInstanceName();
25562555
String configDriveIsoRootFolder = "/tmp";
25572556
String isoFile = configDriveIsoRootFolder + "/" + vmName + "/configDrive/" + vmName + ".iso";

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java

Lines changed: 78 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -18,23 +18,24 @@
1818
*/
1919
package com.cloud.hypervisor.xenserver.resource;
2020

21-
import com.cloud.agent.api.Answer;
22-
import com.cloud.agent.api.to.DataObjectType;
23-
import com.cloud.agent.api.to.DataStoreTO;
24-
import com.cloud.agent.api.to.DataTO;
25-
import com.cloud.agent.api.to.DiskTO;
26-
import com.cloud.agent.api.to.NfsTO;
27-
import com.cloud.agent.api.to.S3TO;
28-
import com.cloud.agent.api.to.SwiftTO;
29-
import com.cloud.exception.InternalErrorException;
30-
import com.cloud.hypervisor.Hypervisor.HypervisorType;
31-
import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase.SRType;
32-
import com.cloud.storage.DataStoreRole;
33-
import com.cloud.storage.Storage;
34-
import com.cloud.storage.Storage.ImageFormat;
35-
import com.cloud.storage.resource.StorageProcessor;
36-
import com.cloud.utils.exception.CloudRuntimeException;
37-
import com.cloud.utils.storage.S3.ClientOptions;
21+
import static com.cloud.utils.ReflectUtil.flattenProperties;
22+
import static com.google.common.collect.Lists.newArrayList;
23+
24+
import java.io.File;
25+
import java.net.URI;
26+
import java.util.ArrayList;
27+
import java.util.Arrays;
28+
import java.util.HashMap;
29+
import java.util.List;
30+
import java.util.Map;
31+
import java.util.Set;
32+
import java.util.UUID;
33+
34+
import org.apache.commons.lang3.BooleanUtils;
35+
import org.apache.log4j.Logger;
36+
import org.apache.xmlrpc.XmlRpcException;
37+
38+
import com.google.common.annotations.VisibleForTesting;
3839
import com.xensource.xenapi.Connection;
3940
import com.xensource.xenapi.SR;
4041
import com.xensource.xenapi.Types;
@@ -44,6 +45,7 @@
4445
import com.xensource.xenapi.VBD;
4546
import com.xensource.xenapi.VDI;
4647
import com.xensource.xenapi.VM;
48+
4749
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
4850
import org.apache.cloudstack.storage.command.AttachAnswer;
4951
import org.apache.cloudstack.storage.command.AttachCommand;
@@ -65,20 +67,24 @@
6567
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
6668
import org.apache.cloudstack.storage.to.TemplateObjectTO;
6769
import org.apache.cloudstack.storage.to.VolumeObjectTO;
68-
import org.apache.log4j.Logger;
69-
import org.apache.xmlrpc.XmlRpcException;
70-
71-
import java.io.File;
72-
import java.net.URI;
73-
import java.util.Arrays;
74-
import java.util.HashMap;
75-
import java.util.List;
76-
import java.util.Map;
77-
import java.util.Set;
78-
import java.util.UUID;
7970

80-
import static com.cloud.utils.ReflectUtil.flattenProperties;
81-
import static com.google.common.collect.Lists.newArrayList;
71+
import com.cloud.agent.api.Answer;
72+
import com.cloud.agent.api.to.DataObjectType;
73+
import com.cloud.agent.api.to.DataStoreTO;
74+
import com.cloud.agent.api.to.DataTO;
75+
import com.cloud.agent.api.to.DiskTO;
76+
import com.cloud.agent.api.to.NfsTO;
77+
import com.cloud.agent.api.to.S3TO;
78+
import com.cloud.agent.api.to.SwiftTO;
79+
import com.cloud.exception.InternalErrorException;
80+
import com.cloud.hypervisor.Hypervisor.HypervisorType;
81+
import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase.SRType;
82+
import com.cloud.storage.DataStoreRole;
83+
import com.cloud.storage.Storage;
84+
import com.cloud.storage.Storage.ImageFormat;
85+
import com.cloud.storage.resource.StorageProcessor;
86+
import com.cloud.utils.exception.CloudRuntimeException;
87+
import com.cloud.utils.storage.S3.ClientOptions;
8288

8389
public class XenServerStorageProcessor implements StorageProcessor {
8490
private static final Logger s_logger = Logger.getLogger(XenServerStorageProcessor.class);
@@ -914,20 +920,55 @@ public Answer copyVolumeFromPrimaryToSecondary(final CopyCommand cmd) {
914920

915921
private boolean swiftUpload(final Connection conn, final SwiftTO swift, final String container, final String ldir, final String lfilename, final Boolean isISCSI,
916922
final int wait) {
917-
String result = null;
923+
924+
List<String> params = getSwiftParams(swift, container, ldir, lfilename, isISCSI);
925+
918926
try {
919-
result =
920-
hypervisorResource.callHostPluginAsync(conn, "swiftxenserver", "swift", wait, "op", "upload", "url", swift.getUrl(), "account", swift.getAccount(), "username",
921-
swift.getUserName(), "key", swift.getKey(), "container", container, "ldir", ldir, "lfilename", lfilename, "isISCSI", isISCSI.toString());
922-
if (result != null && result.equals("true")) {
923-
return true;
924-
}
927+
String result = hypervisorResource.callHostPluginAsync(conn, "swiftxenserver", "swift", wait, params.toArray(new String[params.size()]));
928+
return BooleanUtils.toBoolean(result);
925929
} catch (final Exception e) {
926930
s_logger.warn("swift upload failed due to " + e.toString(), e);
927931
}
928932
return false;
929933
}
930934

935+
@VisibleForTesting
936+
List<String> getSwiftParams(SwiftTO swift, String container, String ldir, String lfilename, Boolean isISCSI) {
937+
// ORDER IS IMPORTANT
938+
List<String> params = new ArrayList<>();
939+
940+
//operation
941+
params.add("op");
942+
params.add("upload");
943+
944+
//auth
945+
params.add("url");
946+
params.add(swift.getUrl());
947+
params.add("account");
948+
params.add(swift.getAccount());
949+
params.add("username");
950+
params.add(swift.getUserName());
951+
params.add("key");
952+
params.add(swift.getKey());
953+
954+
// object info
955+
params.add("container");
956+
params.add(container);
957+
params.add("ldir");
958+
params.add(ldir);
959+
params.add("lfilename");
960+
params.add(lfilename);
961+
params.add("isISCSI");
962+
params.add(isISCSI.toString());
963+
964+
if (swift.getStoragePolicy() != null) {
965+
params.add("storagepolicy");
966+
params.add(swift.getStoragePolicy());
967+
}
968+
969+
return params;
970+
}
971+
931972
protected String deleteSnapshotBackup(final Connection conn, final String localMountPoint, final String path, final String secondaryStorageMountPath, final String backupUUID) {
932973

933974
// If anybody modifies the formatting below again, I'll skin them
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package com.cloud.hypervisor.xenserver.resource;
20+
21+
import static org.mockito.Mockito.when;
22+
23+
import java.util.List;
24+
25+
import org.junit.Assert;
26+
import org.junit.Test;
27+
import org.mockito.Mockito;
28+
29+
import com.cloud.agent.api.to.SwiftTO;
30+
31+
public class XenServerStorageProcessorTest {
32+
@Test
33+
public void testOrderOfSwiftUplodScriptParamsWithoutStoragePolicy() {
34+
CitrixResourceBase resource = Mockito.mock(CitrixResourceBase.class);
35+
XenServerStorageProcessor mock = new XenServerStorageProcessor(resource);
36+
37+
SwiftTO swift = Mockito.mock(SwiftTO.class);
38+
when(swift.getStoragePolicy()).thenReturn(null);
39+
40+
String container = "sample-container-name";
41+
String ldir = "sample-ldir";
42+
String lfilename = "sample-lfilename";
43+
Boolean isISCSI = true;
44+
45+
List<String> params = mock.getSwiftParams(swift, container, ldir, lfilename, isISCSI);
46+
47+
// make sure the params not null and has correct number of items in it
48+
Assert.assertNotNull("params is null", params);
49+
Assert.assertTrue("Expected param list size is 18 but it was" + params.size(), params.size() == 18);
50+
51+
// check the order of params
52+
Assert.assertEquals("unexpected param.", "op", params.get(0));
53+
Assert.assertEquals("unexpected param.", "upload", params.get(1));
54+
Assert.assertEquals("unexpected param.", "url", params.get(2));
55+
Assert.assertEquals("unexpected param.", swift.getUrl(), params.get(3));
56+
Assert.assertEquals("unexpected param.", "account", params.get(4));
57+
Assert.assertEquals("unexpected param.", swift.getAccount(), params.get(5));
58+
Assert.assertEquals("unexpected param.", "username", params.get(6));
59+
Assert.assertEquals("unexpected param.", swift.getUserName(), params.get(7));
60+
Assert.assertEquals("unexpected param.", "key", params.get(8));
61+
Assert.assertEquals("unexpected param.", swift.getKey(), params.get(9));
62+
Assert.assertEquals("unexpected param.", "container", params.get(10));
63+
Assert.assertEquals("unexpected param.", container, params.get(11));
64+
Assert.assertEquals("unexpected param.", "ldir", params.get(12));
65+
Assert.assertEquals("unexpected param.", ldir, params.get(13));
66+
Assert.assertEquals("unexpected param.", "lfilename", params.get(14));
67+
Assert.assertEquals("unexpected param.", lfilename, params.get(15));
68+
Assert.assertEquals("unexpected param.", "isISCSI", params.get(16));
69+
Assert.assertEquals("unexpected param.", isISCSI.toString(), params.get(17));
70+
}
71+
72+
@Test
73+
public void testOrderOfSwiftUplodScriptParamsWithStoragePolicy() {
74+
CitrixResourceBase resource = Mockito.mock(CitrixResourceBase.class);
75+
XenServerStorageProcessor mock = new XenServerStorageProcessor(resource);
76+
77+
SwiftTO swift = Mockito.mock(SwiftTO.class);
78+
when(swift.getStoragePolicy()).thenReturn("sample-storagepolicy");
79+
80+
String container = "sample-container-name";
81+
String ldir = "sample-ldir";
82+
String lfilename = "sample-lfilename";
83+
Boolean isISCSI = true;
84+
85+
List<String> params = mock.getSwiftParams(swift, container, ldir, lfilename, isISCSI);
86+
87+
// make sure the params not null and has correct number of items in it
88+
Assert.assertNotNull("params is null", params);
89+
Assert.assertTrue("Expected param list size is 20 but it was" + params.size(), params.size() == 20);
90+
91+
// check the order of params
92+
Assert.assertEquals("unexpected param.", "op", params.get(0));
93+
Assert.assertEquals("unexpected param.", "upload", params.get(1));
94+
Assert.assertEquals("unexpected param.", "url", params.get(2));
95+
Assert.assertEquals("unexpected param.", swift.getUrl(), params.get(3));
96+
Assert.assertEquals("unexpected param.", "account", params.get(4));
97+
Assert.assertEquals("unexpected param.", swift.getAccount(), params.get(5));
98+
Assert.assertEquals("unexpected param.", "username", params.get(6));
99+
Assert.assertEquals("unexpected param.", swift.getUserName(), params.get(7));
100+
Assert.assertEquals("unexpected param.", "key", params.get(8));
101+
Assert.assertEquals("unexpected param.", swift.getKey(), params.get(9));
102+
Assert.assertEquals("unexpected param.", "container", params.get(10));
103+
Assert.assertEquals("unexpected param.", container, params.get(11));
104+
Assert.assertEquals("unexpected param.", "ldir", params.get(12));
105+
Assert.assertEquals("unexpected param.", ldir, params.get(13));
106+
Assert.assertEquals("unexpected param.", "lfilename", params.get(14));
107+
Assert.assertEquals("unexpected param.", lfilename, params.get(15));
108+
Assert.assertEquals("unexpected param.", "isISCSI", params.get(16));
109+
Assert.assertEquals("unexpected param.", isISCSI.toString(), params.get(17));
110+
Assert.assertEquals("unexpected param.", "storagepolicy", params.get(18));
111+
Assert.assertEquals("unexpected param.", "sample-storagepolicy", params.get(19));
112+
}
113+
}

plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl {
6868
public DataStoreTO getStoreTO(DataStore store) {
6969
ImageStoreImpl imgStore = (ImageStoreImpl)store;
7070
Map<String, String> details = _imageStoreDetailsDao.getDetails(imgStore.getId());
71-
return new SwiftTO(imgStore.getId(), imgStore.getUri(), details.get(ApiConstants.ACCOUNT), details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY));
71+
return new SwiftTO(imgStore.getId(), imgStore.getUri(), details.get(ApiConstants.ACCOUNT), details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY), details.get(ApiConstants.STORAGE_POLICY));
7272
}
7373

7474
@Override

0 commit comments

Comments
 (0)