Merge branch '4.22' of https://github.com/apache/cloudstack into improve-listtemplates-perf

Author: Pearl Dsilva

.github/workflows/ci.yml

@@ -282,7 +282,7 @@ jobs:
           # https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2004-Readme.md#mysql
           sudo apt-get install -y mysql-server
           sudo systemctl start mysql
-          sudo mysql -uroot -proot -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY ''; FLUSH PRIVILEGES;"
+          sudo mysql -uroot -proot -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH caching_sha2_password BY ''; FLUSH PRIVILEGES;"
           sudo systemctl restart mysql
           sudo mysql -uroot -e "SELECT VERSION();"
 

.github/workflows/merge-conflict-checker.yml

@@ -18,8 +18,8 @@
 name: "PR Merge Conflict Check"
 on:
   push:
-  pull_request_target:
-    types: [synchronize]
+  pull_request:
+    types: [opened, synchronize, reopened]
 
 permissions:  # added using https://github.com/step-security/secure-workflows
   contents: read

agent/conf/agent.properties

@@ -457,3 +457,18 @@ iscsi.session.cleanup.enabled=false
 
 # Instance conversion VIRT_V2V_TMPDIR env var
 #convert.instance.env.virtv2v.tmpdir=
+
+# Time, in seconds, to wait before retrying to rebase during the incremental snapshot process.
+# incremental.snapshot.retry.rebase.wait=60
+
+# Path to the VDDK library directory for VMware to KVM conversion via VDDK,
+# passed to virt-v2v as -io vddk-libdir=<path>
+#vddk.lib.dir=
+
+# Ordered VDDK transport preference for VMware to KVM conversion via VDDK, passed as
+# -io vddk-transports=<value> to virt-v2v. Example: nbd:nbdssl
+#vddk.transports=
+
+# Optional vCenter SHA1 thumbprint for VMware to KVM conversion via VDDK, passed as
+# -io vddk-thumbprint=<value>. If unset, CloudStack computes it on the KVM host via openssl.
+#vddk.thumbprint=

agent/pom.xml

@@ -24,7 +24,7 @@
     <parent>
         <groupId>org.apache.cloudstack</groupId>
         <artifactId>cloudstack</artifactId>
-        <version>4.22.1.0-SNAPSHOT</version>
+        <version>4.22.2.0-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -808,6 +808,30 @@ public Property<Integer> getWorkers() {
      */
     public static final Property<String> CONVERT_ENV_VIRTV2V_TMPDIR = new Property<>("convert.instance.env.virtv2v.tmpdir", null, String.class);
 
+    /**
+     * Path to the VDDK library directory on the KVM conversion host, used when converting VMs from VMware to KVM via VDDK.
+     * This directory is passed to virt-v2v as <code>-io vddk-libdir=&lt;path&gt;</code>.
+     * Data type: String.<br>
+     * Default value: <code>null</code>
+     */
+    public static final Property<String> VDDK_LIB_DIR = new Property<>("vddk.lib.dir", null, String.class);
+
+    /**
+     * Ordered list of VDDK transports for virt-v2v, passed as <code>-io vddk-transports=&lt;value&gt;</code>.
+     * Example: <code>nbd:nbdssl</code>.
+     * Data type: String.<br>
+     * Default value: <code>null</code>
+     */
+    public static final Property<String> VDDK_TRANSPORTS = new Property<>("vddk.transports", null, String.class);
+
+    /**
+     * vCenter TLS certificate thumbprint used by virt-v2v VDDK mode, passed as <code>-io vddk-thumbprint=&lt;value&gt;</code>.
+     * If unset, the KVM host computes it at runtime from the vCenter endpoint.
+     * Data type: String.<br>
+     * Default value: <code>null</code>
+     */
+    public static final Property<String> VDDK_THUMBPRINT = new Property<>("vddk.thumbprint", null, String.class);
+
     /**
      * BGP controll CIDR
      * Data type: String.<br>
@@ -885,6 +909,11 @@ public Property<Integer> getWorkers() {
      */
     public static final Property<Boolean> CREATE_FULL_CLONE = new Property<>("create.full.clone", false);
 
+    /**
+     * Time, in seconds, to wait before retrying to rebase during the incremental snapshot process.
+     * */
+    public static final Property<Integer> INCREMENTAL_SNAPSHOT_RETRY_REBASE_WAIT = new Property<>("incremental.snapshot.retry.rebase.wait", 60);
+
 
     public static class Property <T>{
         private String name;

agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java

@@ -175,12 +175,12 @@ private Answer executeProxyLoadScan(final Command cmd, final long proxyVmId, fin
                 try {
                     is.close();
                 } catch (final IOException e) {
-                    logger.warn("Exception when closing , console proxy address : {}", proxyManagementIp);
+                    logger.warn("Exception when closing , console proxy address: {}", proxyManagementIp);
                     success = false;
                 }
             }
         } catch (final IOException e) {
-            logger.warn("Unable to open console proxy command port url, console proxy address : {}", proxyManagementIp);
+            logger.warn("Unable to open console proxy command port url, console proxy address: {}", proxyManagementIp);
             success = false;
         }
 
@@ -331,7 +331,7 @@ private void launchConsoleProxy(final byte[] ksBits, final String ksPassword, fi
         final Object resource = this;
         logger.info("Building class loader for com.cloud.consoleproxy.ConsoleProxy");
         if (consoleProxyMain == null) {
-            logger.info("Running com.cloud.consoleproxy.ConsoleProxy with encryptor password={}", encryptorPassword);
+            logger.info("Running com.cloud.consoleproxy.ConsoleProxy");
             consoleProxyMain = new Thread(new ManagedContextRunnable() {
                 @Override
                 protected void runInContext() {

api/pom.xml

@@ -24,7 +24,7 @@
     <parent>
         <groupId>org.apache.cloudstack</groupId>
         <artifactId>cloudstack</artifactId>
-        <version>4.22.1.0-SNAPSHOT</version>
+        <version>4.22.2.0-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>

api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java

@@ -278,8 +278,8 @@ private DatadiskTO generateDiskTO(OVFFile file, OVFDisk disk, String ovfParentPa
         if (StringUtils.isNotBlank(path)) {
             File f = new File(path);
             if (!f.exists() || f.isDirectory()) {
-                logger.error("One of the attached disk or iso does not exists " + path);
-                throw new InternalErrorException("One of the attached disk or iso as stated on OVF does not exists " + path);
+                logger.error("One of the attached disk or ISO does not exists " + path);
+                throw new InternalErrorException("One of the attached disk or ISO as stated on OVF does not exists " + path);
             }
         }
         Long capacity = disk != null ? disk._capacity : file._size;
@@ -335,7 +335,7 @@ protected List<OVFDisk> extractDisksFromOvfDocumentTree(Document doc) {
             vd.add(od);
         }
         if (logger.isTraceEnabled()) {
-            logger.trace(String.format("found %d disk definitions",vd.size()));
+            logger.trace(String.format("Found %d disk definitions", vd.size()));
         }
         return vd;
     }
@@ -367,7 +367,7 @@ protected List<OVFFile> extractFilesFromOvfDocumentTree(File ovfFile, Document d
             }
         }
         if (logger.isTraceEnabled()) {
-            logger.trace(String.format("found %d file definitions in %s",vf.size(), ovfFile.getPath()));
+            logger.trace(String.format("Found %d file definitions in %s", vf.size(), ovfFile.getPath()));
         }
         return vf;
     }
@@ -523,7 +523,7 @@ OVFFile getFileDefinitionFromDiskDefinition(String fileRef, List<OVFFile> files)
     public List<OVFNetworkTO> getNetPrerequisitesFromDocument(Document doc) throws InternalErrorException {
         if (doc == null) {
             if (logger.isTraceEnabled()) {
-                logger.trace("no document to parse; returning no prerequisite networks");
+                logger.trace("No document to parse; returning no prerequisite Networks");
             }
             return Collections.emptyList();
         }
@@ -541,7 +541,7 @@ private void matchNicsToNets(Map<String, OVFNetworkTO> nets, Node systemElement)
         final DocumentTraversal traversal = (DocumentTraversal) systemElement;
         final NodeIterator iterator = traversal.createNodeIterator(systemElement, NodeFilter.SHOW_ELEMENT, null, true);
         if (logger.isTraceEnabled()) {
-            logger.trace(String.format("starting out with %d network-prerequisites, parsing hardware",nets.size()));
+            logger.trace(String.format("Starting out with %d network-prerequisites, parsing hardware",nets.size()));
         }
         int nicCount = 0;
         for (Node n = iterator.nextNode(); n != null; n = iterator.nextNode()) {
@@ -551,7 +551,7 @@ private void matchNicsToNets(Map<String, OVFNetworkTO> nets, Node systemElement)
                 String name = e.getTextContent(); // should be in our nets
                 if(nets.get(name) == null) {
                     if(logger.isInfoEnabled()) {
-                        logger.info(String.format("found a nic definition without a network definition byname %s, adding it to the list.", name));
+                        logger.info(String.format("Found a NIC definition without a Network definition by name %s, adding it to the list.", name));
                     }
                     nets.put(name, new OVFNetworkTO());
                 }
@@ -562,7 +562,7 @@ private void matchNicsToNets(Map<String, OVFNetworkTO> nets, Node systemElement)
             }
         }
         if (logger.isTraceEnabled()) {
-            logger.trace(String.format("ending up with %d network-prerequisites, parsed %d nics", nets.size(), nicCount));
+            logger.trace(String.format("Ending up with %d network-prerequisites, parsed %d NICs", nets.size(), nicCount));
         }
     }
 
@@ -631,7 +631,7 @@ private Map<String, OVFNetworkTO> getNetworksFromDocumentTree(Document doc) {
             nets.put(networkName,network);
         }
         if (logger.isTraceEnabled()) {
-            logger.trace(String.format("found %d networks in template", nets.size()));
+            logger.trace(String.format("Found %d Networks in Template", nets.size()));
         }
         return nets;
     }

api/src/main/java/com/cloud/agent/api/to/BucketTO.java

@@ -26,10 +26,13 @@ public final class BucketTO {
 
     private String secretKey;
 
+    private long accountId;
+
     public BucketTO(Bucket bucket) {
         this.name = bucket.getName();
         this.accessKey = bucket.getAccessKey();
         this.secretKey = bucket.getSecretKey();
+        this.accountId = bucket.getAccountId();
     }
 
     public BucketTO(String name) {
@@ -47,4 +50,8 @@ public String getAccessKey() {
     public String getSecretKey() {
         return this.secretKey;
     }
+
+    public long getAccountId() {
+        return this.accountId;
+    }
 }

api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java

@@ -36,13 +36,17 @@ public class RemoteInstanceTO implements Serializable {
     private String vcenterPassword;
     private String vcenterHost;
     private String datacenterName;
+    private String clusterName;
+    private String hostName;
 
     public RemoteInstanceTO() {
     }
 
-    public RemoteInstanceTO(String instanceName) {
+    public RemoteInstanceTO(String instanceName, String clusterName, String hostName) {
         this.hypervisorType = Hypervisor.HypervisorType.VMware;
         this.instanceName = instanceName;
+        this.clusterName = clusterName;
+        this.hostName = hostName;
     }
 
     public RemoteInstanceTO(String instanceName, String instancePath, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName) {
@@ -55,6 +59,12 @@ public RemoteInstanceTO(String instanceName, String instancePath, String vcenter
         this.datacenterName = datacenterName;
     }
 
+    public RemoteInstanceTO(String instanceName, String instancePath, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName, String clusterName, String hostName) {
+        this(instanceName, instancePath, vcenterHost, vcenterUsername, vcenterPassword, datacenterName);
+        this.clusterName = clusterName;
+        this.hostName = hostName;
+    }
+
     public Hypervisor.HypervisorType getHypervisorType() {
         return this.hypervisorType;
     }
@@ -82,4 +92,12 @@ public String getVcenterHost() {
     public String getDatacenterName() {
         return datacenterName;
     }
+
+    public String getClusterName() {
+        return clusterName;
+    }
+
+    public String getHostName() {
+        return hostName;
+    }
 }