Skip to content

Commit ae88533

Browse files
nvazqueznvazquez
committed
CLOUDSTACK-10146: Bypass Secondary Storage for KVM templates
1 parent 6724a47 commit ae88533

100 files changed

Lines changed: 2538 additions & 65 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎agent/src/com/cloud/agent/Agent.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
// under the License.
1717
package com.cloud.agent;
1818

19+
import com.cloud.utils.UriUtils;
1920
import java.io.File;
2021
import java.io.IOException;
2122
import java.io.PrintWriter;
@@ -37,6 +38,9 @@
3738

3839
import javax.naming.ConfigurationException;
3940

41+
import org.apache.cloudstack.agent.directdownload.CheckUrlAnswer;
42+
import org.apache.cloudstack.agent.directdownload.CheckUrlCommand;
43+
import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificate;
4044
import org.apache.cloudstack.ca.SetupCertificateAnswer;
4145
import org.apache.cloudstack.ca.SetupCertificateCommand;
4246
import org.apache.cloudstack.ca.SetupKeyStoreCommand;
@@ -551,6 +555,10 @@ protected void processRequest(final Request request, final Link link) {
551555
answer = setupAgentKeystore((SetupKeyStoreCommand) cmd);
552556
} else if (cmd instanceof SetupCertificateCommand && ((SetupCertificateCommand) cmd).isHandleByAgent()) {
553557
answer = setupAgentCertificate((SetupCertificateCommand) cmd);
558+
} else if (cmd instanceof CheckUrlCommand) {
559+
answer = checkUrl((CheckUrlCommand) cmd);
560+
} else if (cmd instanceof SetupDirectDownloadCertificate) {
561+
answer = setupDirectDownloadCertificate((SetupDirectDownloadCertificate) cmd);
554562
} else {
555563
if (cmd instanceof ReadyCommand) {
556564
processReadyCommand(cmd);
@@ -600,6 +608,47 @@ protected void processRequest(final Request request, final Link link) {
600608
}
601609
}
602610

611+
private Answer setupDirectDownloadCertificate(SetupDirectDownloadCertificate cmd) {
612+
String certificate = cmd.getCertificate();
613+
String certificateName = cmd.getCertificateName();
614+
s_logger.debug("Importing certificate " + certificateName + " into keystore");
615+
616+
final File agentFile = PropertiesUtil.findConfigFile("agent.properties");
617+
if (agentFile == null) {
618+
return new Answer(cmd, false, "Failed to find agent.properties file");
619+
}
620+
621+
final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile;
622+
623+
String cerFile = agentFile.getParent() + "/" + certificateName + ".cer";
624+
Script.runSimpleBashScript(String.format("echo %s > %s", certificate, cerFile));
625+
626+
String privatePasswordFormat = "sed -n '/keystore.passphrase/p' '%s' 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null";
627+
String privatePasswordCmd = String.format(privatePasswordFormat, agentFile.getAbsolutePath());
628+
String privatePassword = Script.runSimpleBashScript(privatePasswordCmd);
629+
630+
String importCommandFormat = "keytool -importcert -file %s -keystore %s -alias '%s' -storepass '%s' -noprompt";
631+
String importCmd = String.format(importCommandFormat, cerFile, keyStoreFile, certificateName, privatePassword);
632+
Script.runSimpleBashScript(importCmd);
633+
return new Answer(cmd, true, "Certificate " + certificateName + " imported");
634+
}
635+
636+
private Answer checkUrl(CheckUrlCommand cmd) {
637+
final String url = cmd.getUrl();
638+
s_logger.debug("Checking URL: " + url);
639+
boolean checkResult = true;
640+
Long remoteSize = null;
641+
try {
642+
UriUtils.checkUrlExistence(url);
643+
remoteSize = UriUtils.getRemoteSize(url);
644+
}
645+
catch (IllegalArgumentException e) {
646+
s_logger.warn(e.getMessage());
647+
checkResult = false;
648+
}
649+
return new CheckUrlAnswer(checkResult, remoteSize);
650+
}
651+
603652
public Answer setupAgentKeystore(final SetupKeyStoreCommand cmd) {
604653
final String keyStorePassword = cmd.getKeystorePassword();
605654
final long validityDays = cmd.getValidityDays();

‎agent/src/com/cloud/agent/direct/download/DirectTemplateDownloader.java‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
//
2+
// Licensed to the Apache Software Foundation (ASF) under one
3+
// or more contributor license agreements. See the NOTICE file
4+
// distributed with this work for additional information
5+
// regarding copyright ownership. The ASF licenses this file
6+
// to you under the Apache License, Version 2.0 (the
7+
// "License"); you may not use this file except in compliance
8+
// with the License. You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing,
13+
// software distributed under the License is distributed on an
14+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
// KIND, either express or implied. See the License for the
16+
// specific language governing permissions and limitations
17+
// under the License.
18+
//
19+
20+
package com.cloud.agent.direct.download;
21+
22+
public interface DirectTemplateDownloader {
23+
24+
class DirectTemplateInformation {
25+
private String installPath;
26+
private Long size;
27+
private String checksum;
28+
29+
public DirectTemplateInformation(String installPath, Long size, String checksum) {
30+
this.installPath = installPath;
31+
this.size = size;
32+
this.checksum = checksum;
33+
}
34+
35+
public String getInstallPath() {
36+
return installPath;
37+
}
38+
39+
public Long getSize() {
40+
return size;
41+
}
42+
43+
public String getChecksum() {
44+
return checksum;
45+
}
46+
}
47+
48+
/**
49+
* Perform template download to pool specified on downloader creation
50+
* @return true if successful, false if not
51+
*/
52+
boolean downloadTemplate();
53+
54+
/**
55+
* Perform extraction (if necessary) and installation of previously downloaded template
56+
* @return true if successful, false if not
57+
*/
58+
boolean extractAndInstallDownloadedTemplate();
59+
60+
/**
61+
* Get template information after it is properly installed on pool
62+
* @return template information
63+
*/
64+
DirectTemplateInformation getTemplateInformation();
65+
66+
/**
67+
* Perform checksum validation of previously downloadeed template
68+
* @return true if successful, false if not
69+
*/
70+
boolean validateChecksum();
71+
}

‎agent/src/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
//
2+
// Licensed to the Apache Software Foundation (ASF) under one
3+
// or more contributor license agreements. See the NOTICE file
4+
// distributed with this work for additional information
5+
// regarding copyright ownership. The ASF licenses this file
6+
// to you under the Apache License, Version 2.0 (the
7+
// "License"); you may not use this file except in compliance
8+
// with the License. You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing,
13+
// software distributed under the License is distributed on an
14+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
// KIND, either express or implied. See the License for the
16+
// specific language governing permissions and limitations
17+
// under the License.
18+
//
19+
package com.cloud.agent.direct.download;
20+
21+
import com.cloud.utils.exception.CloudRuntimeException;
22+
import com.cloud.utils.script.Script;
23+
import org.apache.cloudstack.utils.security.ChecksumValue;
24+
import org.apache.commons.lang.StringUtils;
25+
26+
import java.io.File;
27+
import java.util.UUID;
28+
29+
public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDownloader {
30+
31+
private String url;
32+
private String destPoolPath;
33+
private Long templateId;
34+
private String downloadedFilePath;
35+
private String installPath;
36+
private String checksum;
37+
38+
protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId, final String checksum) {
39+
this.url = url;
40+
this.destPoolPath = destPoolPath;
41+
this.templateId = templateId;
42+
this.checksum = checksum;
43+
}
44+
45+
private static String directDownloadDir = "template";
46+
47+
/**
48+
* Return direct download temporary path to download template
49+
* @param templateId
50+
* @return
51+
*/
52+
protected static String getDirectDownloadTempPath(Long templateId) {
53+
String templateIdAsString = String.valueOf(templateId);
54+
return directDownloadDir + File.separator + templateIdAsString.substring(0,1) +
55+
File.separator + templateIdAsString;
56+
}
57+
58+
/**
59+
* Create folder on path if it does not exist
60+
* @param path
61+
*/
62+
protected void createFolder(String path) {
63+
File dir = new File(path);
64+
if (!dir.exists()) {
65+
dir.mkdirs();
66+
}
67+
}
68+
69+
public String getUrl() {
70+
return url;
71+
}
72+
73+
public String getDestPoolPath() {
74+
return destPoolPath;
75+
}
76+
77+
public Long getTemplateId() {
78+
return templateId;
79+
}
80+
81+
public String getDownloadedFilePath() {
82+
return downloadedFilePath;
83+
}
84+
85+
public void setDownloadedFilePath(String filePath) {
86+
this.downloadedFilePath = filePath;
87+
}
88+
89+
/**
90+
* Return filename from url
91+
* @return
92+
*/
93+
public String getFileNameFromUrl() {
94+
String[] urlParts = url.split("/");
95+
return urlParts[urlParts.length - 1];
96+
}
97+
98+
/**
99+
* Checks if downloaded template is extractable
100+
* @return true if it should be extracted, false if not
101+
*/
102+
private boolean isTemplateExtractable() {
103+
String type = Script.runSimpleBashScript("file " + downloadedFilePath + " | awk -F' ' '{print $2}'");
104+
return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip");
105+
}
106+
107+
@Override
108+
public boolean extractAndInstallDownloadedTemplate() {
109+
installPath = UUID.randomUUID().toString();
110+
if (isTemplateExtractable()) {
111+
extractDownloadedTemplate();
112+
} else {
113+
Script.runSimpleBashScript("mv " + downloadedFilePath + " " + getInstallFullPath());
114+
}
115+
return true;
116+
}
117+
118+
/**
119+
* Return install full path
120+
* @return
121+
*/
122+
private String getInstallFullPath() {
123+
return destPoolPath + File.separator + installPath;
124+
}
125+
126+
/**
127+
* Return extract command to execute given downloaded file
128+
* @return
129+
*/
130+
private String getExtractCommandForDownloadedFile() {
131+
if (downloadedFilePath.endsWith(".zip")) {
132+
return "unzip -p " + downloadedFilePath + " | cat > " + getInstallFullPath();
133+
} else if (downloadedFilePath.endsWith(".bz2")) {
134+
return "bunzip2 -c " + downloadedFilePath + " > " + getInstallFullPath();
135+
} else if (downloadedFilePath.endsWith(".gz")) {
136+
return "gunzip -c " + downloadedFilePath + " > " + getInstallFullPath();
137+
} else {
138+
throw new CloudRuntimeException("Unable to extract template " + templateId + " on " + downloadedFilePath);
139+
}
140+
}
141+
142+
/**
143+
* Extract downloaded template into installPath, remove compressed file
144+
*/
145+
private void extractDownloadedTemplate() {
146+
String extractCommand = getExtractCommandForDownloadedFile();
147+
Script.runSimpleBashScript(extractCommand);
148+
Script.runSimpleBashScript("rm -f " + downloadedFilePath);
149+
}
150+
151+
@Override
152+
public DirectTemplateInformation getTemplateInformation() {
153+
String sizeResult = Script.runSimpleBashScript("ls -als " + getInstallFullPath() + " | awk '{print $1}'");
154+
long size = Long.parseLong(sizeResult);
155+
return new DirectTemplateInformation(installPath, size, checksum);
156+
}
157+
158+
/**
159+
* Return checksum command from algorithm
160+
* @param algorithm
161+
* @return
162+
*/
163+
private String getChecksumCommandFromAlgorithm(String algorithm) {
164+
if (algorithm.equalsIgnoreCase("MD5")) {
165+
return "md5sum";
166+
} else if (algorithm.equalsIgnoreCase("SHA-1")) {
167+
return "sha1sum";
168+
} else if (algorithm.equalsIgnoreCase("SHA-224")) {
169+
return "sha224sum";
170+
} else if (algorithm.equalsIgnoreCase("SHA-256")) {
171+
return "sha256sum";
172+
} else if (algorithm.equalsIgnoreCase("SHA-384")) {
173+
return "sha384sum";
174+
} else if (algorithm.equalsIgnoreCase("SHA-512")) {
175+
return "sha512sum";
176+
} else {
177+
throw new CloudRuntimeException("Unknown checksum algorithm: " + algorithm);
178+
}
179+
}
180+
181+
@Override
182+
public boolean validateChecksum() {
183+
if (StringUtils.isNotBlank(checksum)) {
184+
ChecksumValue providedChecksum = new ChecksumValue(checksum);
185+
String algorithm = providedChecksum.getAlgorithm();
186+
String checksumCommand = "echo '%s %s' | %s -c --quiet";
187+
String cmd = String.format(checksumCommand, providedChecksum.getChecksum(), downloadedFilePath, getChecksumCommandFromAlgorithm(algorithm));
188+
int result = Script.runSimpleBashScriptForExitValue(cmd);
189+
return result == 0;
190+
}
191+
return true;
192+
}
193+
}

0 commit comments

Comments
 (0)