Multiple SSH Keys support

api/src/main/java/com/cloud/vm/UserVmService.java

@@ -215,7 +215,7 @@ UserVm startVirtualMachine(StartVMCmd cmd) throws StorageUnavailableException, E
      */
     UserVm createBasicSecurityGroupVirtualMachine(DataCenter zone, ServiceOffering serviceOffering, VirtualMachineTemplate template, List<Long> securityGroupIdList,
         Account owner, String hostName, String displayName, Long diskOfferingId, Long diskSize, String group, HypervisorType hypervisor, HTTPMethod httpmethod,
-        String userData, String sshKeyPair, Map<Long, IpAddresses> requestedIps, IpAddresses defaultIp, Boolean displayVm, String keyboard,
+        String userData, List<String> sshKeyPairs, Map<Long, IpAddresses> requestedIps, IpAddresses defaultIp, Boolean displayVm, String keyboard,
         List<Long> affinityGroupIdList, Map<String, String> customParameter, String customId, Map<String, Map<Integer, String>> dhcpOptionMap,
         Map<Long, DiskOffering> dataDiskTemplateToDiskOfferingMap,
         Map<String, String> userVmOVFProperties, boolean dynamicScalingEnabled, Long overrideDiskOfferingId) throws InsufficientCapacityException,
@@ -297,7 +297,7 @@ UserVm createBasicSecurityGroupVirtualMachine(DataCenter zone, ServiceOffering s
      */
     UserVm createAdvancedSecurityGroupVirtualMachine(DataCenter zone, ServiceOffering serviceOffering, VirtualMachineTemplate template, List<Long> networkIdList,
         List<Long> securityGroupIdList, Account owner, String hostName, String displayName, Long diskOfferingId, Long diskSize, String group, HypervisorType hypervisor,
-        HTTPMethod httpmethod, String userData, String sshKeyPair, Map<Long, IpAddresses> requestedIps, IpAddresses defaultIps, Boolean displayVm, String keyboard,
+        HTTPMethod httpmethod, String userData, List<String> sshKeyPairs, Map<Long, IpAddresses> requestedIps, IpAddresses defaultIps, Boolean displayVm, String keyboard,
         List<Long> affinityGroupIdList, Map<String, String> customParameters, String customId, Map<String, Map<Integer, String>> dhcpOptionMap,
         Map<Long, DiskOffering> dataDiskTemplateToDiskOfferingMap,
         Map<String, String> userVmOVFProperties, boolean dynamicScalingEnabled, Long overrideDiskOfferingId) throws InsufficientCapacityException,
@@ -377,7 +377,7 @@ UserVm createAdvancedSecurityGroupVirtualMachine(DataCenter zone, ServiceOfferin
      */
     UserVm createAdvancedVirtualMachine(DataCenter zone, ServiceOffering serviceOffering, VirtualMachineTemplate template, List<Long> networkIdList, Account owner,
         String hostName, String displayName, Long diskOfferingId, Long diskSize, String group, HypervisorType hypervisor, HTTPMethod httpmethod, String userData,
-        String sshKeyPair, Map<Long, IpAddresses> requestedIps, IpAddresses defaultIps, Boolean displayVm, String keyboard, List<Long> affinityGroupIdList,
+        List<String> sshKeyPairs, Map<Long, IpAddresses> requestedIps, IpAddresses defaultIps, Boolean displayVm, String keyboard, List<Long> affinityGroupIdList,
         Map<String, String> customParameters, String customId, Map<String, Map<Integer, String>> dhcpOptionMap, Map<Long, DiskOffering> dataDiskTemplateToDiskOfferingMap,
         Map<String, String> templateOvfPropertiesMap, boolean dynamicScalingEnabled, String type, Long overrideDiskOfferingId)
 

api/src/main/java/com/cloud/vm/VmDetailConstants.java

@@ -59,6 +59,7 @@ public interface VmDetailConstants {
     String MESSAGE_RESERVED_CAPACITY_FREED_FLAG = "Message.ReservedCapacityFreed.Flag";
     String DEPLOY_VM = "deployvm";
     String SSH_PUBLIC_KEY = "SSH.PublicKey";
+    String SSH_KEY_PAIR_NAMES = "SSH.KeyPairNames";
     String PASSWORD = "password";
     String ENCRYPTED_PASSWORD = "Encrypted.Password";
 
@@ -73,5 +74,6 @@ public interface VmDetailConstants {
     String DISK_OFFERING = "diskOffering";
 
     String DEPLOY_AS_IS_CONFIGURATION = "configurationId";
+    String KEY_PAIR_NAMES = "keypairnames";
     String CKS_CONTROL_NODE_LOGIN_USER = "controlNodeLoginUser";
 }

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -441,6 +441,7 @@ public class ApiConstants {
     public static final String NETWORKRATE = "networkrate";
     public static final String HOST_TAGS = "hosttags";
     public static final String SSH_KEYPAIR = "keypair";
+    public static final String SSH_KEYPAIRS = "keypairs";
     public static final String HTTPMETHOD = "httpmethod";
     public static final String HOST_CPU_CAPACITY = "hostcpucapacity";
     public static final String HOST_CPU_NUM = "hostcpunum";

api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java

@@ -153,9 +153,13 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG
             length = 1048576)
     private String userData;
 
+    @Deprecated
     @Parameter(name = ApiConstants.SSH_KEYPAIR, type = CommandType.STRING, description = "name of the ssh key pair used to login to the virtual machine")
     private String sshKeyPairName;
 
+    @Parameter(name = ApiConstants.SSH_KEYPAIRS, type = CommandType.LIST, collectionType = CommandType.STRING, since="4.17", description = "names of the ssh key pairs used to login to the virtual machine")
+    private List<String> sshKeyPairNames;
+
     @Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, description = "destination Host ID to deploy the VM to - parameter available for root admin only")
     private Long hostId;
 
@@ -444,8 +448,15 @@ public String getName() {
         return name;
     }
 
-    public String getSSHKeyPairName() {
-        return sshKeyPairName;
+    public List<String> getSSHKeyPairNames() {
+        List<String> sshKeyPairs = new ArrayList<String>();
+        if(sshKeyPairNames != null) {
+            sshKeyPairs = sshKeyPairNames;
+        }
+        if(sshKeyPairName != null && !sshKeyPairName.isEmpty()) {
+            sshKeyPairs.add(sshKeyPairName);
+        }
+        return sshKeyPairs;
     }
 
     public Long getHostId() {

api/src/main/java/org/apache/cloudstack/api/command/user/vm/ResetVMSSHKeyCmd.java

@@ -42,6 +42,9 @@
 import com.cloud.uservm.UserVm;
 import com.cloud.vm.VirtualMachine;
 
+import java.util.ArrayList;
+import java.util.List;
+
 @APICommand(name = "resetSSHKeyForVirtualMachine", responseObject = UserVmResponse.class, description = "Resets the SSH Key for virtual machine. " +
         "The virtual machine must be in a \"Stopped\" state. [async]", responseView = ResponseView.Restricted, entityType = {VirtualMachine.class},
     requestHasSensitiveInfo = false, responseHasSensitiveInfo = true)
@@ -58,8 +61,12 @@ public class ResetVMSSHKeyCmd extends BaseAsyncCmd implements UserCmd {
     @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = UserVmResponse.class, required = true, description = "The ID of the virtual machine")
     private Long id;
 
-    @Parameter(name = ApiConstants.SSH_KEYPAIR, type = CommandType.STRING, required = true, description = "name of the ssh key pair used to login to the virtual machine")
-    private String name;
+    @Deprecated
+    @Parameter(name = ApiConstants.SSH_KEYPAIR, type = CommandType.STRING ,description = "name of the ssh key pair used to login to the virtual machine")
+    String name;
+
+    @Parameter(name = ApiConstants.SSH_KEYPAIRS, type = CommandType.LIST, collectionType = CommandType.STRING, since="4.17", description = "names of the ssh key pairs to be used to login to the virtual machine")
+    List<String> names;
 
     //Owner information
     @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the ssh key. Must be used with domainId.")
@@ -78,8 +85,15 @@ public class ResetVMSSHKeyCmd extends BaseAsyncCmd implements UserCmd {
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
 
-    public String getName() {
-        return name;
+    public List<String> getNames() {
+        List<String> keypairnames = new ArrayList<String>();
+        if (names != null) {
+            keypairnames = names;
+        }
+        if (name != null && !name.isEmpty()) {
+            keypairnames.add(name);
+        }
+        return keypairnames;
     }
 
     public Long getId() {

api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java

@@ -280,9 +280,9 @@ public class UserVmResponse extends BaseResponseWithTagInformation implements Co
     @Param(description = "List of read-only Vm details as comma separated string.", since = "4.16.0")
     private String readOnlyDetails;
 
-    @SerializedName(ApiConstants.SSH_KEYPAIR)
-    @Param(description = "ssh key-pair")
-    private String keyPairName;
+    @SerializedName(ApiConstants.SSH_KEYPAIRS)
+    @Param(description = "ssh key-pairs")
+    private String keyPairNames;
 
     @SerializedName("affinitygroup")
     @Param(description = "list of affinity groups associated with the virtual machine", responseObject = AffinityGroupResponse.class)
@@ -588,8 +588,8 @@ public String getInstanceName() {
         return instanceName;
     }
 
-    public String getKeyPairName() {
-        return keyPairName;
+    public String getKeyPairNames() {
+        return keyPairNames;
     }
 
     public Set<AffinityGroupResponse> getAffinityGroupList() {
@@ -848,8 +848,8 @@ public void setTags(Set<ResourceTagResponse> tags) {
         this.tags = tags;
     }
 
-    public void setKeyPairName(String keyPairName) {
-        this.keyPairName = keyPairName;
+    public void setKeyPairNames(String keyPairNames) {
+        this.keyPairNames = keyPairNames;
     }
 
     public void setAffinityGroupList(Set<AffinityGroupResponse> affinityGroups) {

engine/schema/src/main/java/com/cloud/user/dao/SSHKeyPairDao.java

@@ -37,4 +37,6 @@ public interface SSHKeyPairDao extends GenericDao<SSHKeyPairVO, Long> {
 
     public SSHKeyPairVO findByPublicKey(long accountId, long domainId, String publicKey);
 
+    public List<SSHKeyPairVO> findByNames(long accountId, long domainId, List<String> names);
+
 }

engine/schema/src/main/java/com/cloud/user/dao/SSHKeyPairDaoImpl.java

@@ -19,6 +19,7 @@
 import java.util.List;
 
 
+import com.cloud.utils.db.Filter;
 import org.springframework.stereotype.Component;
 
 import com.cloud.user.SSHKeyPairVO;
@@ -63,6 +64,16 @@ public SSHKeyPairVO findByName(long accountId, long domainId, String name) {
         return findOneBy(sc);
     }
 
+    @Override
+    public List<SSHKeyPairVO> findByNames(long accountId, long domainId, List<String> names) {
+        SearchCriteria<SSHKeyPairVO> sc = createSearchCriteria();
+        final Filter filter = new Filter(SSHKeyPairVO.class,"name",false, null, null);
+        sc.addAnd("accountId", SearchCriteria.Op.EQ, accountId);
+        sc.addAnd("domainId", SearchCriteria.Op.EQ, domainId);
+        sc.addAnd("name", SearchCriteria.Op.IN, names.toArray());
+        return this.search(sc, filter);
+    }
+
     @Override
     public SSHKeyPairVO findByPublicKey(String publicKey) {
         SearchCriteria<SSHKeyPairVO> sc = createSearchCriteria();

engine/schema/src/main/java/com/cloud/vm/VMInstanceVO.java

@@ -488,8 +488,7 @@ public Map<String, String> getDetails() {
 
     public void setDetail(String name, String value) {
         assert (details != null) : "Did you forget to load the details?";
-
-        details.put(name, value);
+        this.details.put(name, value);
     }
 
     public void setDetails(Map<String, String> details) {

engine/schema/src/main/resources/META-INF/db/schema-41610to41700.sql

@@ -471,7 +471,7 @@ SELECT
     `user_ip_address`.`id` AS `public_ip_id`,
     `user_ip_address`.`uuid` AS `public_ip_uuid`,
     `user_ip_address`.`public_ip_address` AS `public_ip_address`,
-    `ssh_keypairs`.`keypair_name` AS `keypair_name`,
+    `ssh_details`.`value` AS `keypair_names`,
     `resource_tags`.`id` AS `tag_id`,
     `resource_tags`.`uuid` AS `tag_uuid`,
     `resource_tags`.`key` AS `tag_key`,
@@ -495,7 +495,7 @@ SELECT
     `affinity_group`.`description` AS `affinity_group_description`,
     `vm_instance`.`dynamically_scalable` AS `dynamically_scalable`
 FROM
-    (((((((((((((((((((((((((((((((((`user_vm`
+    ((((((((((((((((((((((((((((((((`user_vm`
         JOIN `vm_instance` ON (((`vm_instance`.`id` = `user_vm`.`id`)
             AND ISNULL(`vm_instance`.`removed`))))
         JOIN `account` ON ((`vm_instance`.`account_id` = `account`.`id`)))
@@ -524,9 +524,7 @@ FROM
             AND ISNULL(`vpc`.`removed`))))
         LEFT JOIN `user_ip_address` ON ((`user_ip_address`.`vm_id` = `vm_instance`.`id`)))
         LEFT JOIN `user_vm_details` `ssh_details` ON (((`ssh_details`.`vm_id` = `vm_instance`.`id`)
-            AND (`ssh_details`.`name` = 'SSH.PublicKey'))))
-        LEFT JOIN `ssh_keypairs` ON (((`ssh_keypairs`.`public_key` = `ssh_details`.`value`)
-            AND (`ssh_keypairs`.`account_id` = `account`.`id`))))
+            AND (`ssh_details`.`name` = 'SSH.KeyPairNames'))))
         LEFT JOIN `resource_tags` ON (((`resource_tags`.`resource_id` = `vm_instance`.`id`)
             AND (`resource_tags`.`resource_type` = 'UserVm'))))
         LEFT JOIN `async_job` ON (((`async_job`.`instance_id` = `vm_instance`.`id`)
@@ -646,3 +644,11 @@ CREATE VIEW `cloud`.`domain_router_view` AS
 
 INSERT INTO `cloud`.`role_permissions` (`uuid`, `role_id`, `rule`, `permission`, `sort_order`) SELECT UUID(), 3, 'listConfigurations', 'ALLOW', (SELECT MAX(`sort_order`)+1 FROM `cloud`.`role_permissions`) ON DUPLICATE KEY UPDATE rule=rule;
 INSERT INTO `cloud`.`role_permissions` (`uuid`, `role_id`, `rule`, `permission`, `sort_order`) SELECT UUID(), 3, 'updateConfiguration', 'ALLOW', (SELECT MAX(`sort_order`)+1 FROM `cloud`.`role_permissions`) ON DUPLICATE KEY UPDATE rule=rule;
+
+INSERT INTO `cloud`.`user_vm_details`(`vm_id`, `name`, `value`)
+    SELECT `user_vm_details`.`vm_id`, 'SSH.KeyPairNames', `ssh_keypairs`.`keypair_name`
+        FROM `cloud`.`user_vm_details`
+        INNER JOIN `cloud`.`ssh_keypairs` ON ssh_keypairs.public_key = user_vm_details.value
+        INNER JOIN `cloud`.`vm_instance` ON vm_instance.id = user_vm_details.vm_id
+        WHERE ssh_keypairs.account_id = vm_instance.account_id;
+

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java

@@ -376,9 +376,13 @@ protected UserVm createKubernetesNode(String joinIp) throws ManagementServerExce
             logAndThrow(Level.ERROR, "Failed to read Kubernetes node configuration file", e);
         }
         String base64UserData = Base64.encodeBase64String(k8sNodeConfig.getBytes(com.cloud.utils.StringUtils.getPreferredCharset()));
+        List<String> keypairs = new ArrayList<String>();
+        if (StringUtils.isNotBlank(kubernetesCluster.getKeyPair())) {
+            keypairs.add(kubernetesCluster.getKeyPair());
+        }
         nodeVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, clusterTemplate, networkIds, owner,
                 hostName, hostName, null, null, null,
-                Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(),
+                Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, keypairs,
                 null, addrs, null, null, null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null);
         if (LOGGER.isInfoEnabled()) {
             LOGGER.info(String.format("Created node VM : %s, %s in the Kubernetes cluster : %s", hostName, nodeVm.getUuid(), kubernetesCluster.getName()));

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java

@@ -213,9 +213,13 @@ private UserVm createKubernetesControlNode(final Network network, String serverI
             logAndThrow(Level.ERROR, "Failed to read Kubernetes control node configuration file", e);
         }
         String base64UserData = Base64.encodeBase64String(k8sControlNodeConfig.getBytes(com.cloud.utils.StringUtils.getPreferredCharset()));
+        List<String> keypairs = new ArrayList<String>();
+        if (StringUtils.isNotBlank(kubernetesCluster.getKeyPair())) {
+            keypairs.add(kubernetesCluster.getKeyPair());
+        }
         controlVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, clusterTemplate, networkIds, owner,
                 hostName, hostName, null, null, null,
-                Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(),
+                Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, keypairs,
                 requestedIps, addrs, null, null, null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null);
         if (LOGGER.isInfoEnabled()) {
             LOGGER.info(String.format("Created control VM ID: %s, %s in the Kubernetes cluster : %s", controlVm.getUuid(), hostName, kubernetesCluster.getName()));
@@ -273,9 +277,13 @@ private UserVm createKubernetesAdditionalControlNode(final String joinIp, final
             logAndThrow(Level.ERROR, "Failed to read Kubernetes control configuration file", e);
         }
         String base64UserData = Base64.encodeBase64String(k8sControlNodeConfig.getBytes(com.cloud.utils.StringUtils.getPreferredCharset()));
+        List<String> keypairs = new ArrayList<String>();
+        if (StringUtils.isNotBlank(kubernetesCluster.getKeyPair())) {
+            keypairs.add(kubernetesCluster.getKeyPair());
+        }
         additionalControlVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, clusterTemplate, networkIds, owner,
                 hostName, hostName, null, null, null,
-                Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(),
+                Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, keypairs,
                 null, addrs, null, null, null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null);
         if (LOGGER.isInfoEnabled()) {
             LOGGER.info(String.format("Created control VM ID : %s, %s in the Kubernetes cluster : %s", additionalControlVm.getUuid(), hostName, kubernetesCluster.getName()));

server/src/main/java/com/cloud/api/query/QueryManagerImpl.java

@@ -1072,7 +1072,7 @@ private Pair<List<UserVmJoinVO>, Integer> searchForUserVMsInternal(ListVMsCmd cm
         }
 
         if (keyPairName != null) {
-            sb.and("keyPairName", sb.entity().getKeypairName(), SearchCriteria.Op.EQ);
+            sb.and("keyPairName", sb.entity().getKeypairNames(), SearchCriteria.Op.FIND_IN_SET);
         }
 
         if (!isRootAdmin) {

server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java

@@ -219,7 +219,7 @@ public UserVmResponse newUserVmResponse(ResponseView view, String objectName, Us
 
         userVmResponse.setPublicIpId(userVm.getPublicIpUuid());
         userVmResponse.setPublicIp(userVm.getPublicIpAddress());
-        userVmResponse.setKeyPairName(userVm.getKeypairName());
+        userVmResponse.setKeyPairNames(userVm.getKeypairNames());
         userVmResponse.setOsTypeId(userVm.getGuestOsUuid());
         GuestOS guestOS = ApiDBUtils.findGuestOSById(userVm.getGuestOsId());
         if (guestOS != null) {

server/src/main/java/com/cloud/api/query/vo/UserVmJoinVO.java

@@ -353,8 +353,8 @@ public class UserVmJoinVO extends BaseViewWithTagInformationVO implements Contro
     @Column(name = "project_name")
     private String projectName;
 
-    @Column(name = "keypair_name")
-    private String keypairName;
+    @Column(name = "keypair_names")
+    private String keypairNames;
 
     @Column(name = "job_id")
     private Long jobId;
@@ -779,8 +779,8 @@ public String getProjectName() {
         return projectName;
     }
 
-    public String getKeypairName() {
-        return keypairName;
+    public String getKeypairNames() {
+        return keypairNames;
     }
 
     public boolean isLimitCpuUse() {