CKS Enhancements and SystemVM template upgrade improvements

.gitignore

@@ -48,6 +48,7 @@ tools/cli/cloudmonkey/precache.py
 tools/marvin/marvin/cloudstackAPI/
 tools/marvin/build/
 tools/cli/build/
+tools/appliance/systemvmtemplate/packer_cache/
 *.jar
 *.war
 *.mar

agent/src/main/java/com/cloud/agent/AgentShell.java

@@ -16,6 +16,26 @@
 // under the License.
 package com.cloud.agent;
 
+import com.cloud.agent.Agent.ExitStatus;
+import com.cloud.agent.dao.StorageComponent;
+import com.cloud.agent.dao.impl.PropertiesStorage;
+import com.cloud.resource.ServerResource;
+import com.cloud.utils.LogUtils;
+import com.cloud.utils.NumbersUtil;
+import com.cloud.utils.ProcessUtil;
+import com.cloud.utils.PropertiesUtil;
+import com.cloud.utils.backoff.BackoffAlgorithm;
+import com.cloud.utils.backoff.impl.ConstantTimeBackoff;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.google.common.base.Strings;
+import org.apache.commons.daemon.Daemon;
+import org.apache.commons.daemon.DaemonContext;
+import org.apache.commons.daemon.DaemonInitException;
+import org.apache.commons.lang.math.NumberUtils;
+import org.apache.log4j.Logger;
+import org.apache.log4j.xml.DOMConfigurator;
+
+import javax.naming.ConfigurationException;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -30,28 +50,6 @@
 import java.util.Properties;
 import java.util.UUID;
 
-import javax.naming.ConfigurationException;
-
-import org.apache.commons.daemon.Daemon;
-import org.apache.commons.daemon.DaemonContext;
-import org.apache.commons.daemon.DaemonInitException;
-import org.apache.commons.lang.math.NumberUtils;
-import org.apache.log4j.Logger;
-import org.apache.log4j.xml.DOMConfigurator;
-
-import com.cloud.agent.Agent.ExitStatus;
-import com.cloud.agent.dao.StorageComponent;
-import com.cloud.agent.dao.impl.PropertiesStorage;
-import com.cloud.resource.ServerResource;
-import com.cloud.utils.LogUtils;
-import com.cloud.utils.NumbersUtil;
-import com.cloud.utils.ProcessUtil;
-import com.cloud.utils.PropertiesUtil;
-import com.cloud.utils.backoff.BackoffAlgorithm;
-import com.cloud.utils.backoff.impl.ConstantTimeBackoff;
-import com.cloud.utils.exception.CloudRuntimeException;
-import com.google.common.base.Strings;
-
 public class AgentShell implements IAgentShell, Daemon {
     private static final Logger s_logger = Logger.getLogger(AgentShell.class.getName());
 
@@ -423,13 +421,13 @@ private void launchAgentFromClassInfo(String resourceClassNames) throws Configur
             } catch (final ClassNotFoundException e) {
                 throw new ConfigurationException("Resource class not found: " + name + " due to: " + e.toString());
             } catch (final SecurityException e) {
-                throw new ConfigurationException("Security excetion when loading resource: " + name + " due to: " + e.toString());
+                throw new ConfigurationException("Security exception when loading resource: " + name + " due to: " + e.toString());
             } catch (final NoSuchMethodException e) {
-                throw new ConfigurationException("Method not found excetion when loading resource: " + name + " due to: " + e.toString());
+                throw new ConfigurationException("Method not found exception when loading resource: " + name + " due to: " + e.toString());
             } catch (final IllegalArgumentException e) {
-                throw new ConfigurationException("Illegal argument excetion when loading resource: " + name + " due to: " + e.toString());
+                throw new ConfigurationException("Illegal argument exception when loading resource: " + name + " due to: " + e.toString());
             } catch (final InstantiationException e) {
-                throw new ConfigurationException("Instantiation excetion when loading resource: " + name + " due to: " + e.toString());
+                throw new ConfigurationException("Instantiation exception when loading resource: " + name + " due to: " + e.toString());
             } catch (final IllegalAccessException e) {
                 throw new ConfigurationException("Illegal access exception when loading resource: " + name + " due to: " + e.toString());
             } catch (final InvocationTargetException e) {

api/src/main/java/com/cloud/vm/VmDetailConstants.java

@@ -73,4 +73,5 @@ public interface VmDetailConstants {
     String DISK_OFFERING = "diskOffering";
 
     String DEPLOY_AS_IS_CONFIGURATION = "configurationId";
+    String CKS_CONTROL_NODE_LOGIN_USER = "controlNodeLoginUser";
 }

engine/schema/pom.xml

@@ -73,10 +73,10 @@
                         </goals>
                         <configuration>
                             <source>
-                                def projectVersion = project.version
+                                def projectVersion = project.properties['project.systemvm.template.version']
                                 String[] versionParts =  projectVersion.tokenize('.')
-                                pom.properties['cs.version'] = "4.16"
-                                pom.properties['patch.version'] = "0"
+                                pom.properties['cs.version'] = versionParts[0] + "." + versionParts[1]
+                                pom.properties['patch.version'] = versionParts[2]
                             </source>
                         </configuration>
                     </execution>
@@ -146,7 +146,7 @@
                             <executable>bash</executable>
                             <arguments>
                                 <argument>templateConfig.sh</argument>
-                                <armument>${project.version}</armument>
+                                <argument>${project.systemvm.template.version}</argument>
                             </arguments>
                         </configuration>
                     </execution>

engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java

@@ -367,10 +367,11 @@ public void check() {
                     return;
                 }
 
-                SystemVmTemplateRegistration.parseMetadataFile();
-                final CloudStackVersion currentVersion = CloudStackVersion.parse(currentVersionValue);
-                SystemVmTemplateRegistration.CS_MAJOR_VERSION  = String.valueOf(currentVersion.getMajorRelease()) + "." + String.valueOf(currentVersion.getMinorRelease());
-                SystemVmTemplateRegistration.CS_TINY_VERSION = String.valueOf(currentVersion.getPatchRelease());
+                String csVersion = SystemVmTemplateRegistration.parseMetadataFile();
+                final CloudStackVersion sysVmVersion = CloudStackVersion.parse(csVersion);
+                final  CloudStackVersion currentVersion = CloudStackVersion.parse(currentVersionValue);
+                SystemVmTemplateRegistration.CS_MAJOR_VERSION  = String.valueOf(sysVmVersion.getMajorRelease()) + "." + String.valueOf(sysVmVersion.getMinorRelease());
+                SystemVmTemplateRegistration.CS_TINY_VERSION = String.valueOf(sysVmVersion.getPatchRelease());
 
                 s_logger.info("DB version = " + dbVersion + " Code Version = " + currentVersion);
 

engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java

@@ -54,7 +54,7 @@
 import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
-import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.cloudstack.utils.security.DigestHelper;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Logger;
 import org.ini4j.Ini;
@@ -64,7 +64,6 @@
 import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
-import java.io.InputStream;
 import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -351,16 +350,6 @@ public static boolean validateIfSeeded(String url, String path) {
         }
     }
 
-    private String calculateChecksum(File file) {
-        try (InputStream is = Files.newInputStream(Paths.get(file.getPath()))) {
-            return DigestUtils.md5Hex(is);
-        } catch (IOException e) {
-            String errMsg = "Failed to calculate template checksum";
-            LOGGER.error(errMsg, e);
-            throw new CloudRuntimeException(errMsg, e);
-        }
-    }
-
     public Long getRegisteredTemplateId(Pair<Hypervisor.HypervisorType, String> hypervisorAndTemplateName) {
         VMTemplateVO vmTemplate = vmTemplateDao.findLatestTemplateByName(hypervisorAndTemplateName.second());
         Long templateId = null;
@@ -371,24 +360,24 @@ public Long getRegisteredTemplateId(Pair<Hypervisor.HypervisorType, String> hype
     }
 
     private static String fetchTemplatesPath() {
-            String filePath = RELATIVE_TEMPLATE_PATH + METADATA_FILE_NAME;
+        String filePath = RELATIVE_TEMPLATE_PATH + METADATA_FILE_NAME;
+        LOGGER.debug(String.format("Looking for file [ %s ] in the classpath.", filePath));
+        File metaFile = new File(filePath);
+        String templatePath = null;
+        if (metaFile.exists()) {
+            templatePath = RELATIVE_TEMPLATE_PATH;
+        }
+        if (templatePath == null) {
+            filePath = ABSOLUTE_TEMPLATE_PATH + METADATA_FILE_NAME;
+            metaFile = new File(filePath);
+            templatePath = ABSOLUTE_TEMPLATE_PATH;
             LOGGER.debug(String.format("Looking for file [ %s ] in the classpath.", filePath));
-            File metaFile = new File(filePath);
-            String templatePath = null;
-            if (metaFile.exists()) {
-                templatePath = RELATIVE_TEMPLATE_PATH;
-            }
-            if (templatePath == null) {
-                filePath = ABSOLUTE_TEMPLATE_PATH + METADATA_FILE_NAME;
-                metaFile = new File(filePath);
-                templatePath = ABSOLUTE_TEMPLATE_PATH;
-                LOGGER.debug(String.format("Looking for file [ %s ] in the classpath.", filePath));
-                if (!metaFile.exists()) {
-                    String errMsg = String.format("Unable to locate metadata file in your setup at %s", filePath.toString());
-                    LOGGER.error(errMsg);
-                    throw new CloudRuntimeException(errMsg);
-                }
+            if (!metaFile.exists()) {
+                String errMsg = String.format("Unable to locate metadata file in your setup at %s", filePath.toString());
+                LOGGER.error(errMsg);
+                throw new CloudRuntimeException(errMsg);
             }
+        }
         return templatePath;
     }
 
@@ -481,7 +470,7 @@ private VMTemplateVO createTemplateObjectInDB(SystemVMTemplateDetails details) {
         template.setCrossZones(true);
         template.setHypervisorType(details.getHypervisorType());
         template.setState(VirtualMachineTemplate.State.Inactive);
-        template.setDeployAsIs(Hypervisor.HypervisorType.VMware.equals(details.getHypervisorType()));
+        template.setDeployAsIs(false);
         template = vmTemplateDao.persist(template);
         return template;
     }
@@ -577,12 +566,12 @@ private static void readTemplateProperties(String path, SystemVMTemplateDetails
     }
 
     private void updateTemplateTablesOnFailure(long templateId) {
-            VMTemplateVO template = vmTemplateDao.createForUpdate(templateId);
-            template.setState(VirtualMachineTemplate.State.Inactive);
-            vmTemplateDao.update(template.getId(), template);
-            vmTemplateDao.remove(templateId);
-            TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByTemplate(template.getId(), DataStoreRole.Image);
-            templateDataStoreDao.remove(templateDataStoreVO.getId());
+        VMTemplateVO template = vmTemplateDao.createForUpdate(templateId);
+        template.setState(VirtualMachineTemplate.State.Inactive);
+        vmTemplateDao.update(template.getId(), template);
+        vmTemplateDao.remove(templateId);
+        TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByTemplate(template.getId(), DataStoreRole.Image);
+        templateDataStoreDao.remove(templateDataStoreVO.getId());
     }
 
     public static void unmountStore(String filePath) {
@@ -603,7 +592,7 @@ public static void unmountStore(String filePath) {
     }
 
     private void setupTemplate(String templateName, Pair<Hypervisor.HypervisorType, String> hypervisorAndTemplateName,
-        String destTempFolder) throws CloudRuntimeException {
+                               String destTempFolder) throws CloudRuntimeException {
         String setupTmpltScript = Script.findScript(storageScriptsDir, "setup-sysvm-tmplt");
         if (setupTmpltScript == null) {
             throw new CloudRuntimeException("Unable to find the createtmplt.sh");
@@ -651,7 +640,7 @@ private Long performTemplateRegistrationOperations(Pair<Hypervisor.HypervisorTyp
     }
 
     public void registerTemplate(Pair<Hypervisor.HypervisorType, String> hypervisorAndTemplateName,
-                                        Pair<String, Long> storeUrlAndId, VMTemplateVO templateVO, String filePath) {
+                                 Pair<String, Long> storeUrlAndId, VMTemplateVO templateVO, String filePath) {
         Long templateId = null;
         try {
             templateId = templateVO.getId();
@@ -690,7 +679,13 @@ public void registerTemplate(Pair<Hypervisor.HypervisorType, String> hypervisorA
         }
     }
 
-    public static void parseMetadataFile() {
+    /**
+     * This method parses the metadata file consisting of the systemVM templates information
+     * @return the version of the systemvm template that is to be used. This is done to in order
+     * to fallback on the latest available version of the systemVM template when there does not
+     * exist a template corresponding to the current code version.
+     */
+    public static String parseMetadataFile() {
         try {
             Ini ini = new Ini();
             ini.load(new FileReader(METADATA_FILE));
@@ -702,6 +697,8 @@ public static void parseMetadataFile() {
                 NewTemplateChecksum.put(hypervisorType, section.get("checksum"));
                 NewTemplateUrl.put(hypervisorType, section.get("downloadurl"));
             }
+            Ini.Section section = ini.get("default");
+            return section.get("version");
         } catch (Exception e) {
             String errMsg = String.format("Failed to parse systemVM template metadata file: %s", METADATA_FILE);
             LOGGER.error(errMsg, e);
@@ -735,7 +732,7 @@ private void validateTemplates(Set<Hypervisor.HypervisorType> hypervisorsInUse)
             }
 
             File tempFile = new File(TEMPLATES_PATH + matchedTemplate);
-            String templateChecksum = calculateChecksum(tempFile);
+            String templateChecksum = DigestHelper.calculateChecksum(tempFile);
             if (!templateChecksum.equals(NewTemplateChecksum.get(getHypervisorType(hypervisor)))) {
                 LOGGER.error(String.format("Checksum mismatch: %s != %s ", templateChecksum, NewTemplateChecksum.get(getHypervisorType(hypervisor))));
                 templatesFound = false;
@@ -812,9 +809,6 @@ public void doInTransactionWithoutResult(final TransactionStatus status) {
     private void updateRegisteredTemplateDetails(Long templateId, Map.Entry<Hypervisor.HypervisorType, String> hypervisorAndTemplateName) {
         VMTemplateVO templateVO = vmTemplateDao.findById(templateId);
         templateVO.setTemplateType(Storage.TemplateType.SYSTEM);
-        if (Hypervisor.HypervisorType.VMware == templateVO.getHypervisorType()) {
-            templateVO.setDeployAsIs(true);
-        }
         boolean updated = vmTemplateDao.update(templateVO.getId(), templateVO);
         if (!updated) {
             String errMsg = String.format("updateSystemVmTemplates:Exception while updating template with id %s to be marked as 'system'", templateId);
@@ -834,9 +828,6 @@ private void updateRegisteredTemplateDetails(Long templateId, Map.Entry<Hypervis
     private void updateTemplateUrlAndChecksum(VMTemplateVO templateVO, Map.Entry<Hypervisor.HypervisorType, String> hypervisorAndTemplateName) {
         templateVO.setUrl(NewTemplateUrl.get(hypervisorAndTemplateName.getKey()));
         templateVO.setChecksum(NewTemplateChecksum.get(hypervisorAndTemplateName.getKey()));
-        if (Hypervisor.HypervisorType.VMware == templateVO.getHypervisorType()) {
-            templateVO.setDeployAsIs(true);
-        }
         boolean updated = vmTemplateDao.update(templateVO.getId(), templateVO);
         if (!updated) {
             String errMsg = String.format("updateSystemVmTemplates:Exception while updating 'url' and 'checksum' for hypervisor type %s", hypervisorAndTemplateName.getKey().name());

engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql

@@ -128,6 +128,29 @@ CALL `cloud_usage`.`IDEMPOTENT_ADD_UNIQUE_INDEX`('cloud_usage.usage_volume', 'id
 -- Add PK to cloud_usage.usage_vpn_user
 CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_vpn_user', 'id', 'BIGINT(20) NOT NULL AUTO_INCREMENT FIRST, ADD PRIMARY KEY (`id`)');
 
+UPDATE `cloud`.`vm_template` SET deploy_as_is = 0 WHERE id = 8;
+
+CREATE PROCEDURE `cloud`.`UPDATE_KUBERNETES_NODE_DETAILS`()
+BEGIN
+  DECLARE vmid BIGINT
+; DECLARE done TINYINT DEFAULT FALSE
+; DECLARE vmidcursor CURSOR FOR SELECT DISTINCT(vm_id) FROM `cloud`.`kubernetes_cluster_vm_map`
+; DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE
+; OPEN vmidcursor
+; vmid_loop:LOOP
+    FETCH NEXT FROM vmidcursor INTO vmid
+;   IF done THEN
+      LEAVE vmid_loop
+;   ELSE
+      INSERT `cloud`.`user_vm_details` (vm_id, name, value, display) VALUES (vmid, 'controlNodeLoginUser', 'core', 1)
+;   END IF
+; END LOOP
+; CLOSE vmidcursor
+; END;
+
+CALL `cloud`.`UPDATE_KUBERNETES_NODE_DETAILS`();
+DROP PROCEDURE IF EXISTS `cloud`.`UPDATE_KUBERNETES_NODE_DETAILS`;
+
 -- Add support for VMware 7.0.2.0
 INSERT IGNORE INTO `cloud`.`hypervisor_capabilities` (uuid, hypervisor_type, hypervisor_version, max_guests_limit, security_group_enabled, max_data_volumes_limit, max_hosts_per_cluster, storage_motion_supported, vm_snapshot_enabled) values (UUID(), 'VMware', '7.0.2.0', 1024, 0, 59, 64, 1, 1);
 -- Copy VMware 7.0.1.0 hypervisor guest OS mappings to VMware 7.0.2.0