From c3582eb2d56f1898b4fd30559f30a98eae6b5257 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 8 Jun 2023 12:30:54 +0530 Subject: [PATCH 1/4] vm-import: fix stopped vms listing in unmanaged instances Signed-off-by: Abhishek Kumar --- .../java/com/cloud/vm/dao/VMInstanceDao.java | 2 +- .../com/cloud/vm/dao/VMInstanceDaoImpl.java | 8 +- .../vm/UnmanagedVMsManagerImpl.java | 132 ++++++++---------- 3 files changed, 67 insertions(+), 75 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java index bdb2534b62d0..42c00231aac1 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java @@ -164,5 +164,5 @@ public interface VMInstanceDao extends GenericDao, StateDao< void updateSystemVmTemplateId(long templateId, Hypervisor.HypervisorType hypervisorType); - List listByHostOrLastHostOrHostPod(long hostId, long podId); + List listByHostOrLastHostOrHostPod(List hostIds, long podId); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index 0e3d4bdde8f1..25a9df2fa064 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -989,9 +989,9 @@ public void updateSystemVmTemplateId(long templateId, Hypervisor.HypervisorType } @Override - public List listByHostOrLastHostOrHostPod(long hostId, long podId) { + public List listByHostOrLastHostOrHostPod(List hostIds, long podId) { SearchBuilder sb = createSearchBuilder(); - sb.or().op("hostId", sb.entity().getHostId(), Op.EQ); + sb.or().op("hostId", sb.entity().getHostId(), Op.IN); sb.or("lastHostId", sb.entity().getLastHostId(), Op.EQ); sb.and().op("hostIdNull", sb.entity().getHostId(), SearchCriteria.Op.NULL); sb.and("lastHostIdNull", sb.entity().getHostId(), SearchCriteria.Op.NULL); @@ -1000,8 +1000,8 @@ public List listByHostOrLastHostOrHostPod(long hostId, long podId) sb.cp(); sb.done(); SearchCriteria sc = sb.create(); - sc.setParameters("hostId", String.valueOf(hostId)); - sc.setParameters("lastHostId", String.valueOf(hostId)); + sc.setParameters("hostId", hostIds.toArray()); + sc.setParameters("lastHostId", hostIds.toArray()); sc.setParameters("podId", String.valueOf(podId)); return listBy(sc); } diff --git a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java index 72ccf621e3a3..9126b89c3bb9 100644 --- a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java @@ -363,8 +363,11 @@ private List getAdditionalNameFilters(Cluster cluster) { return additionalNameFilter; } - private List getHostManagedVms(Host host) { - List instances = vmDao.listByHostOrLastHostOrHostPod(host.getId(), host.getPodId()); + private List getHostsManagedVms(List hosts) { + if (CollectionUtils.isEmpty(hosts)) { + return new ArrayList<>(); + } + List instances = vmDao.listByHostOrLastHostOrHostPod(hosts.stream().map(HostVO::getId).collect(Collectors.toList()), hosts.get(0).getPodId()); List managedVms = instances.stream().map(VMInstanceVO::getInstanceName).collect(Collectors.toList()); return managedVms; } @@ -1035,6 +1038,24 @@ private UserVm importVirtualMachineInternal(final UnmanagedInstanceTO unmanagedI return userVm; } + private HashMap getUnmanagedInstancesForHost(HostVO host, String instanceName, List managedVms) { + HashMap unmanagedInstances = new HashMap<>(); + if (host.isInMaintenanceStates()) { + return unmanagedInstances; + } + + GetUnmanagedInstancesCommand command = new GetUnmanagedInstancesCommand(); + command.setInstanceName(instanceName); + command.setManagedInstancesNames(managedVms); + Answer answer = agentManager.easySend(host.getId(), command); + if (!(answer instanceof GetUnmanagedInstancesAnswer)) { + return unmanagedInstances; + } + GetUnmanagedInstancesAnswer unmanagedInstancesAnswer = (GetUnmanagedInstancesAnswer) answer; + unmanagedInstances = unmanagedInstancesAnswer.getUnmanagedInstances(); + return unmanagedInstances; + } + @Override public ListResponse listUnmanagedInstances(ListUnmanagedInstancesCmd cmd) { final Account caller = CallContext.current().getCallingAccount(); @@ -1058,24 +1079,11 @@ public ListResponse listUnmanagedInstances(ListUnmana } List hosts = resourceManager.listHostsInClusterByStatus(clusterId, Status.Up); List additionalNameFilters = getAdditionalNameFilters(cluster); + List managedVms = new ArrayList<>(additionalNameFilters); + managedVms.addAll(getHostsManagedVms(hosts)); List responses = new ArrayList<>(); for (HostVO host : hosts) { - if (host.isInMaintenanceStates()) { - continue; - } - List managedVms = new ArrayList<>(); - managedVms.addAll(additionalNameFilters); - managedVms.addAll(getHostManagedVms(host)); - - GetUnmanagedInstancesCommand command = new GetUnmanagedInstancesCommand(); - command.setInstanceName(cmd.getName()); - command.setManagedInstancesNames(managedVms); - Answer answer = agentManager.easySend(host.getId(), command); - if (!(answer instanceof GetUnmanagedInstancesAnswer)) { - continue; - } - GetUnmanagedInstancesAnswer unmanagedInstancesAnswer = (GetUnmanagedInstancesAnswer) answer; - HashMap unmanagedInstances = new HashMap<>(unmanagedInstancesAnswer.getUnmanagedInstances()); + HashMap unmanagedInstances = getUnmanagedInstancesForHost(host, cmd.getName(), managedVms); Set keys = unmanagedInstances.keySet(); for (String key : keys) { UnmanagedInstanceTO instance = unmanagedInstances.get(key); @@ -1098,9 +1106,6 @@ public UserVmResponse importUnmanagedInstance(ImportUnmanagedInstanceCmd cmd) { throw new PermissionDeniedException(String.format("Cannot perform this operation, Calling account is not root admin: %s", caller.getUuid())); } final Long clusterId = cmd.getClusterId(); - if (clusterId == null) { - throw new InvalidParameterValueException(String.format("Cluster ID cannot be null")); - } final Cluster cluster = clusterDao.findById(clusterId); if (cluster == null) { throw new InvalidParameterValueException(String.format("Cluster ID: %d cannot be found", clusterId)); @@ -1111,10 +1116,10 @@ public UserVmResponse importUnmanagedInstance(ImportUnmanagedInstanceCmd cmd) { final DataCenter zone = dataCenterDao.findById(cluster.getDataCenterId()); final String instanceName = cmd.getName(); if (StringUtils.isEmpty(instanceName)) { - throw new InvalidParameterValueException(String.format("Instance name cannot be empty")); + throw new InvalidParameterValueException("Instance name cannot be empty"); } if (cmd.getDomainId() != null && StringUtils.isEmpty(cmd.getAccountName())) { - throw new InvalidParameterValueException("domainid parameter must be specified with account parameter"); + throw new InvalidParameterValueException(String.format("%s parameter must be specified with %s parameter", ApiConstants.DOMAIN_ID, ApiConstants.ACCOUNT)); } final Account owner = accountService.getActiveAccountById(cmd.getEntityOwnerId()); long userId = CallContext.current().getCallingUserId(); @@ -1122,7 +1127,7 @@ public UserVmResponse importUnmanagedInstance(ImportUnmanagedInstanceCmd cmd) { if (CollectionUtils.isNotEmpty(userVOs)) { userId = userVOs.get(0).getId(); } - VMTemplateVO template = null; + VMTemplateVO template; final Long templateId = cmd.getTemplateId(); if (templateId == null) { template = templateDao.findByName(VM_IMPORT_DEFAULT_TEMPLATE_NAME); @@ -1139,9 +1144,6 @@ public UserVmResponse importUnmanagedInstance(ImportUnmanagedInstanceCmd cmd) { throw new InvalidParameterValueException(String.format("Template ID: %d cannot be found", templateId)); } final Long serviceOfferingId = cmd.getServiceOfferingId(); - if (serviceOfferingId == null) { - throw new InvalidParameterValueException(String.format("Service offering ID cannot be null")); - } final ServiceOfferingVO serviceOffering = serviceOfferingDao.findById(serviceOfferingId); if (serviceOffering == null) { throw new InvalidParameterValueException(String.format("Service offering ID: %d cannot be found", serviceOfferingId)); @@ -1160,7 +1162,7 @@ public UserVmResponse importUnmanagedInstance(ImportUnmanagedInstanceCmd cmd) { String hostName = cmd.getHostName(); if (StringUtils.isEmpty(hostName)) { if (!NetUtils.verifyDomainNameLabel(instanceName, true)) { - throw new InvalidParameterValueException(String.format("Please provide hostname for the VM. VM name contains unsupported characters for it to be used as hostname")); + throw new InvalidParameterValueException("Please provide hostname for the VM. VM name contains unsupported characters for it to be used as hostname"); } hostName = instanceName; } @@ -1185,59 +1187,49 @@ public UserVmResponse importUnmanagedInstance(ImportUnmanagedInstanceCmd cmd) { List hosts = resourceManager.listHostsInClusterByStatus(clusterId, Status.Up); UserVm userVm = null; List additionalNameFilters = getAdditionalNameFilters(cluster); + List managedVms = new ArrayList<>(additionalNameFilters); + managedVms.addAll(getHostsManagedVms(hosts)); for (HostVO host : hosts) { - if (host.isInMaintenanceStates()) { - continue; - } - List managedVms = new ArrayList<>(); - managedVms.addAll(additionalNameFilters); - managedVms.addAll(getHostManagedVms(host)); - GetUnmanagedInstancesCommand command = new GetUnmanagedInstancesCommand(instanceName); - command.setManagedInstancesNames(managedVms); - Answer answer = agentManager.easySend(host.getId(), command); - if (!(answer instanceof GetUnmanagedInstancesAnswer)) { - continue; - } - GetUnmanagedInstancesAnswer unmanagedInstancesAnswer = (GetUnmanagedInstancesAnswer) answer; - HashMap unmanagedInstances = unmanagedInstancesAnswer.getUnmanagedInstances(); + HashMap unmanagedInstances = getUnmanagedInstancesForHost(host, cmd.getName(), managedVms); if (MapUtils.isEmpty(unmanagedInstances)) { continue; } Set names = unmanagedInstances.keySet(); for (String name : names) { - if (instanceName.equals(name)) { - UnmanagedInstanceTO unmanagedInstance = unmanagedInstances.get(name); - if (unmanagedInstance == null) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to retrieve details for unmanaged VM: %s", name)); + if (!instanceName.equals(name)) { + continue; + } + UnmanagedInstanceTO unmanagedInstance = unmanagedInstances.get(name); + if (unmanagedInstance == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to retrieve details for unmanaged VM: %s", name)); + } + if (template.getName().equals(VM_IMPORT_DEFAULT_TEMPLATE_NAME)) { + String osName = unmanagedInstance.getOperatingSystem(); + GuestOS guestOS = null; + if (StringUtils.isNotEmpty(osName)) { + guestOS = guestOSDao.listByDisplayName(osName); } - if (template.getName().equals(VM_IMPORT_DEFAULT_TEMPLATE_NAME)) { - String osName = unmanagedInstance.getOperatingSystem(); - GuestOS guestOS = null; - if (StringUtils.isNotEmpty(osName)) { - guestOS = guestOSDao.listByDisplayName(osName); - } - GuestOSHypervisor guestOSHypervisor = null; + GuestOSHypervisor guestOSHypervisor = null; + if (guestOS != null) { + guestOSHypervisor = guestOSHypervisorDao.findByOsIdAndHypervisor(guestOS.getId(), host.getHypervisorType().toString(), host.getHypervisorVersion()); + } + if (guestOSHypervisor == null && StringUtils.isNotEmpty(unmanagedInstance.getOperatingSystemId())) { + guestOSHypervisor = guestOSHypervisorDao.findByOsNameAndHypervisor(unmanagedInstance.getOperatingSystemId(), host.getHypervisorType().toString(), host.getHypervisorVersion()); + } + if (guestOSHypervisor == null) { if (guestOS != null) { - guestOSHypervisor = guestOSHypervisorDao.findByOsIdAndHypervisor(guestOS.getId(), host.getHypervisorType().toString(), host.getHypervisorVersion()); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to find hypervisor guest OS ID: %s details for unmanaged VM: %s for hypervisor: %s version: %s. templateid parameter can be used to assign template for VM", guestOS.getUuid(), name, host.getHypervisorType().toString(), host.getHypervisorVersion())); } - if (guestOSHypervisor == null && StringUtils.isNotEmpty(unmanagedInstance.getOperatingSystemId())) { - guestOSHypervisor = guestOSHypervisorDao.findByOsNameAndHypervisor(unmanagedInstance.getOperatingSystemId(), host.getHypervisorType().toString(), host.getHypervisorVersion()); - } - if (guestOSHypervisor == null) { - if (guestOS != null) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to find hypervisor guest OS ID: %s details for unmanaged VM: %s for hypervisor: %s version: %s. templateid parameter can be used to assign template for VM", guestOS.getUuid(), name, host.getHypervisorType().toString(), host.getHypervisorVersion())); - } - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to retrieve guest OS details for unmanaged VM: %s with OS name: %s, OS ID: %s for hypervisor: %s version: %s. templateid parameter can be used to assign template for VM", name, osName, unmanagedInstance.getOperatingSystemId(), host.getHypervisorType().toString(), host.getHypervisorVersion())); - } - template.setGuestOSId(guestOSHypervisor.getGuestOsId()); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to retrieve guest OS details for unmanaged VM: %s with OS name: %s, OS ID: %s for hypervisor: %s version: %s. templateid parameter can be used to assign template for VM", name, osName, unmanagedInstance.getOperatingSystemId(), host.getHypervisorType().toString(), host.getHypervisorVersion())); } - userVm = importVirtualMachineInternal(unmanagedInstance, instanceName, zone, cluster, host, - template, displayName, hostName, caller, owner, userId, - serviceOffering, dataDiskOfferingMap, - nicNetworkMap, nicIpAddressMap, - details, cmd.getMigrateAllowed(), forced); - break; + template.setGuestOSId(guestOSHypervisor.getGuestOsId()); } + userVm = importVirtualMachineInternal(unmanagedInstance, instanceName, zone, cluster, host, + template, displayName, hostName, caller, owner, userId, + serviceOffering, dataDiskOfferingMap, + nicNetworkMap, nicIpAddressMap, + details, cmd.getMigrateAllowed(), forced); + break; } if (userVm != null) { break; From 6aff5d4f1746c2764eadd9dbd8166d91919e5a41 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 8 Jun 2023 12:50:29 +0530 Subject: [PATCH 2/4] fix listing vms for cluster Signed-off-by: Abhishek Kumar --- .../main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java | 2 +- .../java/com/cloud/api/query/QueryManagerImpl.java | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index 25a9df2fa064..72a443f2da6b 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -992,7 +992,7 @@ public void updateSystemVmTemplateId(long templateId, Hypervisor.HypervisorType public List listByHostOrLastHostOrHostPod(List hostIds, long podId) { SearchBuilder sb = createSearchBuilder(); sb.or().op("hostId", sb.entity().getHostId(), Op.IN); - sb.or("lastHostId", sb.entity().getLastHostId(), Op.EQ); + sb.or("lastHostId", sb.entity().getLastHostId(), Op.IN); sb.and().op("hostIdNull", sb.entity().getHostId(), SearchCriteria.Op.NULL); sb.and("lastHostIdNull", sb.entity().getHostId(), SearchCriteria.Op.NULL); sb.and("podId", sb.entity().getPodIdToDeployIn(), Op.EQ); diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 0968af0e7c0e..e37521dab847 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -195,6 +195,7 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; import com.cloud.ha.HighAvailabilityManager; +import com.cloud.host.Host; import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.network.RouterHealthCheckResult; @@ -1073,6 +1074,11 @@ private Pair, Integer> searchForUserVMsInternal(ListVMsCmd cm sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ); sb.and("podId", sb.entity().getPodId(), SearchCriteria.Op.EQ); sb.and("clusterId", sb.entity().getClusterId(), SearchCriteria.Op.EQ); + if (clusterId != null) { + sb.or().op("clusterHostId", sb.entity().getHostId(), Op.IN); + sb.or("clusterLastHostId", sb.entity().getLastHostId(), Op.IN); + sb.cp(); + } sb.and("hypervisorType", sb.entity().getHypervisorType(), SearchCriteria.Op.EQ); sb.and("hostIdEQ", sb.entity().getHostId(), SearchCriteria.Op.EQ); sb.and("templateId", sb.entity().getTemplateId(), SearchCriteria.Op.EQ); @@ -1268,6 +1274,10 @@ private Pair, Integer> searchForUserVMsInternal(ListVMsCmd cm if (clusterId != null) { sc.setParameters("clusterId", clusterId); + List hosts = _hostJoinDao.findByClusterId((Long)clusterId, Host.Type.Routing); + List hostIds = hosts.stream().map(HostJoinVO::getId).collect(Collectors.toList()); + sc.setParameters("clusterHostId", hostIds.toArray()); + sc.setParameters("clusterLastHostId", hostIds.toArray()); } if (hostId != null) { From 672a8cbc554678bef8880658cf54c0bb6008b9f5 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 8 Jun 2023 13:43:55 +0530 Subject: [PATCH 3/4] fix Signed-off-by: Abhishek Kumar --- .../src/main/java/com/cloud/api/query/QueryManagerImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index e37521dab847..6f3b1c757400 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -1073,9 +1073,9 @@ private Pair, Integer> searchForUserVMsInternal(ListVMsCmd cm sb.and("stateNIN", sb.entity().getState(), SearchCriteria.Op.NIN); sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ); sb.and("podId", sb.entity().getPodId(), SearchCriteria.Op.EQ); - sb.and("clusterId", sb.entity().getClusterId(), SearchCriteria.Op.EQ); if (clusterId != null) { - sb.or().op("clusterHostId", sb.entity().getHostId(), Op.IN); + sb.and().op("clusterId", sb.entity().getClusterId(), SearchCriteria.Op.EQ); + sb.or("clusterHostId", sb.entity().getHostId(), Op.IN); sb.or("clusterLastHostId", sb.entity().getLastHostId(), Op.IN); sb.cp(); } From 347023032babb2f13c8dfb9361d8ced14a336c2c Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 10 Oct 2023 15:46:06 +0530 Subject: [PATCH 4/4] fix issue Signed-off-by: Abhishek Kumar --- .../src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index 72a443f2da6b..916687baeb4d 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -991,9 +991,9 @@ public void updateSystemVmTemplateId(long templateId, Hypervisor.HypervisorType @Override public List listByHostOrLastHostOrHostPod(List hostIds, long podId) { SearchBuilder sb = createSearchBuilder(); - sb.or().op("hostId", sb.entity().getHostId(), Op.IN); + sb.and().op("hostId", sb.entity().getHostId(), Op.IN); sb.or("lastHostId", sb.entity().getLastHostId(), Op.IN); - sb.and().op("hostIdNull", sb.entity().getHostId(), SearchCriteria.Op.NULL); + sb.or().op("hostIdNull", sb.entity().getHostId(), SearchCriteria.Op.NULL); sb.and("lastHostIdNull", sb.entity().getHostId(), SearchCriteria.Op.NULL); sb.and("podId", sb.entity().getPodIdToDeployIn(), Op.EQ); sb.cp();