From fcaa40ad5f03dd97f9280e8ddf0e8f04d7f87d86 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 22 Sep 2023 13:19:43 +0530 Subject: [PATCH 01/34] api,server,ui: tagged resource limits Introduces the concept of tagged resource limits. Limits can be enforced on accounts and domains for the deployment of entities for a tagged resource. Current tagged resource limits can be used for the following resource types, Host limits user_vm cpu memory Storage limits volume primary_storage Following global settings can used to specify tags for which limit needs to be enforced, Host: resource.limit.host.tags Storage: resource.limit.storage.tags Option for specifying tagged resource limits and viewing tagged resource usage are made available in the UI. Enhances use of templatetag for VM deployment and template creation Adds option to list disk offering with suitability flag for a virtualmachine. A new parameter named virtualmachineid has been added to the listDiskOfferings API which when passed returns suitableforvirtualmachine param in the reponse. Signed-off-by: Abhishek Kumar --- .github/workflows/ci.yml | 3 +- .../java/com/cloud/capacity/Capacity.java | 7 + .../com/cloud/configuration/Resource.java | 1 + .../main/java/com/cloud/storage/Volume.java | 2 + .../com/cloud/user/ResourceLimitService.java | 76 +- .../apache/cloudstack/api/ApiConstants.java | 7 +- .../command/admin/domain/ListDomainsCmd.java | 21 +- .../admin/resource/ListCapacityCmd.java | 7 + .../command/user/account/ListAccountsCmd.java | 21 +- .../user/offering/ListDiskOfferingsCmd.java | 25 +- .../offering/ListServiceOfferingsCmd.java | 21 +- .../user/resource/ListResourceLimitsCmd.java | 14 +- .../user/resource/UpdateResourceCountCmd.java | 10 +- .../user/resource/UpdateResourceLimitCmd.java | 9 +- .../api/response/AccountResponse.java | 9 + .../api/response/CapacityResponse.java | 11 +- .../api/response/DiskOfferingResponse.java | 8 + .../api/response/DomainResponse.java | 10 + .../api/response/ProjectResponse.java | 9 + .../api/response/ResourceCountResponse.java | 16 +- .../ResourceLimitAndCountResponse.java | 4 + .../api/response/ResourceLimitResponse.java | 13 +- .../TaggedResourceLimitAndCountResponse.java | 74 ++ .../cloudstack/user/ResourceReservation.java | 5 +- .../com/cloud/vm/VirtualMachineManager.java | 2 + .../com/cloud/storage/StorageManager.java | 7 + .../cloud/vm/VirtualMachineManagerImpl.java | 65 +- .../orchestration/VolumeOrchestrator.java | 17 +- .../com/cloud/capacity/dao/CapacityDao.java | 5 +- .../cloud/capacity/dao/CapacityDaoImpl.java | 35 +- .../cloud/configuration/ResourceCountVO.java | 23 +- .../cloud/configuration/ResourceLimitVO.java | 18 +- .../configuration/dao/ResourceCountDao.java | 20 +- .../dao/ResourceCountDaoImpl.java | 98 +- .../configuration/dao/ResourceLimitDao.java | 4 +- .../dao/ResourceLimitDaoImpl.java | 56 +- .../src/main/java/com/cloud/host/HostVO.java | 27 +- .../main/java/com/cloud/host/dao/HostDao.java | 2 +- .../java/com/cloud/host/dao/HostDaoImpl.java | 6 +- .../cloud/service/dao/ServiceOfferingDao.java | 2 + .../service/dao/ServiceOfferingDaoImpl.java | 18 + .../cloud/storage/dao/DiskOfferingDao.java | 1 + .../storage/dao/DiskOfferingDaoImpl.java | 19 + .../cloud/storage/dao/StoragePoolTagsDao.java | 1 + .../storage/dao/StoragePoolTagsDaoImpl.java | 18 +- .../com/cloud/storage/dao/VMTemplateDao.java | 3 + .../cloud/storage/dao/VMTemplateDaoImpl.java | 10 + .../java/com/cloud/storage/dao/VolumeDao.java | 3 + .../com/cloud/storage/dao/VolumeDaoImpl.java | 24 + .../cloudstack/reservation/ReservationVO.java | 28 +- .../reservation/dao/ReservationDao.java | 7 +- .../reservation/dao/ReservationDaoImpl.java | 40 +- .../META-INF/db/schema-41810to41900.sql | 10 + .../META-INF/db/views/cloud.account_view.sql | 164 +++ .../META-INF/db/views/cloud.domain_view.sql | 134 +++ .../test/java/com/cloud/host/HostVOTest.java | 210 ++-- .../AbstractStoragePoolAllocator.java | 24 +- .../storage/volume/VolumeServiceImpl.java | 6 +- .../allocator/impl/RandomAllocator.java | 48 +- .../agent/manager/MockStorageManager.java | 3 + .../agent/manager/MockStorageManagerImpl.java | 30 + .../agent/manager/SimulatorManagerImpl.java | 5 +- .../metrics/PrometheusExporterImpl.java | 26 +- .../main/java/com/cloud/api/ApiDBUtils.java | 38 +- .../java/com/cloud/api/ApiResponseHelper.java | 6 +- .../com/cloud/api/query/QueryManagerImpl.java | 68 +- .../cloud/api/query/ViewResponseHelper.java | 8 +- .../cloud/api/query/dao/UserVmJoinDao.java | 5 +- .../api/query/dao/UserVmJoinDaoImpl.java | 27 + .../deploy/DeploymentPlanningManagerImpl.java | 8 +- .../resourcelimit/CheckedReservation.java | 92 +- .../ResourceLimitManagerImpl.java | 1002 ++++++++++++++--- .../cloud/server/ManagementServerImpl.java | 220 ++-- .../com/cloud/storage/StorageManagerImpl.java | 140 ++- .../cloud/storage/VolumeApiServiceImpl.java | 65 +- .../storage/snapshot/SnapshotManagerImpl.java | 4 +- .../template/HypervisorTemplateAdapter.java | 3 +- .../com/cloud/user/DomainManagerImpl.java | 48 +- .../main/java/com/cloud/vm/UserVmManager.java | 2 + .../java/com/cloud/vm/UserVmManagerImpl.java | 219 ++-- .../vm/UnmanagedVMsManagerImpl.java | 70 +- .../resourcelimit/CheckedReservationTest.java | 6 +- .../ResourceLimitManagerImplTest.java | 10 +- .../storage/VolumeApiServiceImplTest.java | 13 +- ...countManagerImplVolumeDeleteEventTest.java | 2 - .../vpc/MockResourceLimitManagerImpl.java | 165 ++- .../vm/UnmanagedVMsManagerImplTest.java | 1 - .../component/test_resource_limit_tags.py | 648 +++++++++++ ui/public/locales/en.json | 12 +- ui/src/components/view/ResourceCountUsage.vue | 140 ++- ui/src/components/view/ResourceLimitTab.vue | 94 +- ui/src/components/view/TreeView.vue | 6 +- ui/src/config/section/image.js | 2 +- ui/src/utils/plugins.js | 30 +- ui/src/views/compute/DeployVM.vue | 32 +- .../views/image/RegisterOrUploadTemplate.vue | 9 + ui/src/views/storage/CreateVolume.vue | 11 +- 97 files changed, 3928 insertions(+), 880 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/api/response/TaggedResourceLimitAndCountResponse.java create mode 100644 engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql create mode 100644 engine/schema/src/main/resources/META-INF/db/views/cloud.domain_view.sql create mode 100644 test/integration/component/test_resource_limit_tags.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6edc7bdb20a..043b5b1f56f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -179,7 +179,8 @@ jobs: "component/test_project_usage component/test_protocol_number_security_group component/test_public_ip - component/test_resource_limits", + component/test_resource_limits + component/test_resource_limit_tags", "component/test_regions_accounts component/test_routers component/test_snapshots diff --git a/api/src/main/java/com/cloud/capacity/Capacity.java b/api/src/main/java/com/cloud/capacity/Capacity.java index 684490a605c3..6e5db3fa7481 100644 --- a/api/src/main/java/com/cloud/capacity/Capacity.java +++ b/api/src/main/java/com/cloud/capacity/Capacity.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.capacity; +import java.util.List; + import org.apache.cloudstack.api.Identity; import org.apache.cloudstack.api.InternalIdentity; @@ -35,6 +37,11 @@ public interface Capacity extends InternalIdentity, Identity { public static final short CAPACITY_TYPE_CPU_CORE = 90; + public static final List STORAGE_CAPACITY_TYPES = List.of(CAPACITY_TYPE_STORAGE, + CAPACITY_TYPE_STORAGE_ALLOCATED, + CAPACITY_TYPE_SECONDARY_STORAGE, + CAPACITY_TYPE_LOCAL_STORAGE); + public Long getHostOrPoolId(); public Long getDataCenterId(); diff --git a/api/src/main/java/com/cloud/configuration/Resource.java b/api/src/main/java/com/cloud/configuration/Resource.java index 32db2fcafeaf..bf8fca9d9051 100644 --- a/api/src/main/java/com/cloud/configuration/Resource.java +++ b/api/src/main/java/com/cloud/configuration/Resource.java @@ -85,5 +85,6 @@ public String getName() { long getOwnerId(); ResourceOwnerType getResourceOwnerType(); + String getTag(); } diff --git a/api/src/main/java/com/cloud/storage/Volume.java b/api/src/main/java/com/cloud/storage/Volume.java index 4a14197bd303..72b7ca5866f1 100644 --- a/api/src/main/java/com/cloud/storage/Volume.java +++ b/api/src/main/java/com/cloud/storage/Volume.java @@ -30,6 +30,8 @@ public interface Volume extends ControlledEntity, Identity, InternalIdentity, BasedOn, StateObject, Displayable { + static final long DISK_OFFERING_SUITABILITY_CHECK_VOLUME_ID = -1; + // Managed storage volume parameters (specified in the compute/disk offering for PowerFlex) String BANDWIDTH_LIMIT_IN_MBPS = "bandwidthLimitInMbps"; String IOPS_LIMIT = "iopsLimit"; diff --git a/api/src/main/java/com/cloud/user/ResourceLimitService.java b/api/src/main/java/com/cloud/user/ResourceLimitService.java index f2d87a4390df..0a64cbb7440a 100644 --- a/api/src/main/java/com/cloud/user/ResourceLimitService.java +++ b/api/src/main/java/com/cloud/user/ResourceLimitService.java @@ -18,13 +18,18 @@ import java.util.List; +import org.apache.cloudstack.api.response.AccountResponse; +import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.framework.config.ConfigKey; + import com.cloud.configuration.Resource.ResourceType; import com.cloud.configuration.ResourceCount; import com.cloud.configuration.ResourceLimit; import com.cloud.domain.Domain; import com.cloud.exception.ResourceAllocationException; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.user.ResourceReservation; +import com.cloud.offering.DiskOffering; +import com.cloud.offering.ServiceOffering; +import com.cloud.template.VirtualMachineTemplate; public interface ResourceLimitService { @@ -34,6 +39,13 @@ public interface ResourceLimitService { "The default maximum secondary storage space (in GiB) that can be used for a project", false); static final ConfigKey ResourceCountCheckInterval = new ConfigKey<>("Advanced", Long.class, "resourcecount.check.interval", "300", "Time (in seconds) to wait before running resource recalculation and fixing task. Default is 300 seconds, Setting this to 0 disables execution of the task", false); + static final ConfigKey ResourceLimitHostTags = new ConfigKey<>("Advanced", String.class, "resource.limit.host.tags", "", + "A comma-separated list of tags for host resource limits", true); + static final ConfigKey ResourceLimitStorageTags = new ConfigKey<>("Advanced", String.class, "resource.limit.storage.tags", "", + "A comma-separated list of tags for storage resource limits", true); + + static final List HostTagsSupportingTypes = List.of(ResourceType.user_vm, ResourceType.cpu, ResourceType.memory); + static final List StorageTagsSupportingTypes = List.of(ResourceType.volume, ResourceType.primary_storage); /** * Updates an existing resource limit with the specified details. If a limit doesn't exist, will create one. @@ -46,22 +58,27 @@ public interface ResourceLimitService { * TODO * @param max * TODO + * @param tag + * tag for the resource type * * @return the updated/created resource limit */ - ResourceLimit updateResourceLimit(Long accountId, Long domainId, Integer resourceType, Long max); + ResourceLimit updateResourceLimit(Long accountId, Long domainId, Integer resourceType, Long max, String tag); /** * Updates an existing resource count details for the account/domain * * @param accountId - * TODO + * Id of the account for which resource recalculation to be done * @param domainId - * TODO + * Id of the domain for which resource recalculation to be doneDO * @param typeId - * TODO + * type of the resource for which recalculation to be done + * @param tag + * tag for the resource type for which recalculation to be done * @return the updated/created resource counts */ + List recalculateResourceCount(Long accountId, Long domainId, Integer typeId, String tag); List recalculateResourceCount(Long accountId, Long domainId, Integer typeId); /** @@ -77,7 +94,7 @@ public interface ResourceLimitService { * TODO * @return a list of limits that match the criteria */ - public List searchForLimits(Long id, Long accountId, Long domainId, ResourceType resourceType, Long startIndex, Long pageSizeVal); + public List searchForLimits(Long id, Long accountId, Long domainId, ResourceType resourceType, String tag, Long startIndex, Long pageSizeVal); /** * Finds the resource limit for a specified account and type. If the account has an infinite limit, will check @@ -85,9 +102,10 @@ public interface ResourceLimitService { * * @param account * @param type + * @param tag * @return resource limit */ - public long findCorrectResourceLimitForAccount(Account account, ResourceType type); + public long findCorrectResourceLimitForAccount(Account account, ResourceType type, String tag); /** * This call should be used when we have already queried resource limit for an account. This is to handle @@ -105,9 +123,10 @@ public interface ResourceLimitService { * * @param domain * @param type + * @param tag * @return resource limit */ - public long findCorrectResourceLimitForDomain(Domain domain, ResourceType type); + public long findCorrectResourceLimitForDomain(Domain domain, ResourceType type, String tag); /** * Finds the default resource limit for a specified type. @@ -122,9 +141,10 @@ public interface ResourceLimitService { * * @param domain * @param type + * @param tag * @return resource limit */ - public long findCorrectResourceLimitForAccountAndDomain(Account account, Domain domain, ResourceType type); + public long findCorrectResourceLimitForAccountAndDomain(Account account, Domain domain, ResourceType type, String tag); /** * Increments the resource count @@ -134,6 +154,7 @@ public interface ResourceLimitService { * @param delta */ public void incrementResourceCount(long accountId, ResourceType type, Long... delta); + public void incrementResourceCountWithTag(long accountId, ResourceType type, String tag, Long... delta); /** * Decrements the resource count @@ -143,6 +164,7 @@ public interface ResourceLimitService { * @param delta */ public void decrementResourceCount(long accountId, ResourceType type, Long... delta); + public void decrementResourceCountWithTag(long accountId, ResourceType type, String tag, Long... delta); /** * Checks if a limit has been exceeded for an account @@ -155,15 +177,17 @@ public interface ResourceLimitService { * @throws ResourceAllocationException */ public void checkResourceLimit(Account account, ResourceCount.ResourceType type, long... count) throws ResourceAllocationException; + public void checkResourceLimitWithTag(Account account, ResourceCount.ResourceType type, String tag, long... count) throws ResourceAllocationException; /** * Gets the count of resources for a resource type and account * * @param account * @param type + * @param tag * @return count of resources */ - public long getResourceCount(Account account, ResourceType type); + public long getResourceCount(Account account, ResourceType type, String tag); /** * Checks if a limit has been exceeded for an account if displayResource flag is on @@ -208,15 +232,25 @@ public interface ResourceLimitService { */ void decrementResourceCount(long accountId, ResourceType type, Boolean displayResource, Long... delta); - /** - * Adds a reservation that will be counted in subsequent calls to {count}getResourceCount{code} until {code}this[code} - * is closed. It will create a reservation record that will be counted when resource limits are checked. - * @param account The account for which the reservation is. - * @param displayResource whether this resource is shown to users at all (if not it is not counted to limits) - * @param type resource type - * @param delta amount to reserve (will not be <+ 0) - * @return a {code}AutoClosable{Code} object representing the resource the user needs - */ - ResourceReservation getReservation(Account account, Boolean displayResource, ResourceType type, Long delta) throws ResourceAllocationException; + List getResourceLimitHostTags(); + List getResourceLimitHostTags(ServiceOffering serviceOffering, VirtualMachineTemplate template); + List getResourceLimitStorageTags(); + List getResourceLimitStorageTags(DiskOffering diskOffering); + void updateTaggedResourceLimitsAndCountsForAccounts(List responses, String tag); + void updateTaggedResourceLimitsAndCountsForDomains(List responses, String tag); + void checkVolumeResourceLimit(Account owner, Boolean display, Long size, DiskOffering diskOffering) throws ResourceAllocationException; + void incrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering); + void decrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering); + void incrementVolumePrimaryStorageResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering); + void decrementVolumePrimaryStorageResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering); + void checkVmResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template) throws ResourceAllocationException; + void incrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template); + void decrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template); + void checkVmCpuResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long cpu) throws ResourceAllocationException; + void incrementVmCpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long cpu); + void decrementVmCpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long cpu); + void checkVmMemoryResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory) throws ResourceAllocationException; + void incrementVmMemoryResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory); + void decrementVmMemoryResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory); } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index d77677216670..5b49c3eb5dd4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -285,6 +285,7 @@ public class ApiConstants { public static final String LAST_SERVER_STOP = "lastserverstop"; public static final String LEVEL = "level"; public static final String LENGTH = "length"; + public static final String LIMIT = "limit"; public static final String LIMIT_CPU_USE = "limitcpuuse"; public static final String LIST_HOSTS = "listhosts"; public static final String LOCK = "lock"; @@ -379,6 +380,7 @@ public class ApiConstants { public static final String RECONNECT = "reconnect"; public static final String RECOVER = "recover"; public static final String REQUIRES_HVM = "requireshvm"; + public static final String RESOURCE_COUNT = "resourcecount"; public static final String RESOURCE_NAME = "resourcename"; public static final String RESOURCE_TYPE = "resourcetype"; public static final String RESOURCE_TYPE_NAME = "resourcetypename"; @@ -419,8 +421,9 @@ public class ApiConstants { public static final String SNAPSHOT_POLICY_ID = "snapshotpolicyid"; public static final String SNAPSHOT_TYPE = "snapshottype"; public static final String SNAPSHOT_QUIESCEVM = "quiescevm"; - public static final String SUPPORTS_STORAGE_SNAPSHOT = "supportsstoragesnapshot"; public static final String SOURCE_ZONE_ID = "sourcezoneid"; + public static final String SUITABLE_FOR_VM = "suitableforvirtualmachine"; + public static final String SUPPORTS_STORAGE_SNAPSHOT = "supportsstoragesnapshot"; public static final String START_DATE = "startdate"; public static final String START_ID = "startid"; public static final String START_IP = "startip"; @@ -448,6 +451,7 @@ public class ApiConstants { public static final String TIMEOUT = "timeout"; public static final String TIMEZONE = "timezone"; public static final String TIMEZONEOFFSET = "timezoneoffset"; + public static final String TOTAL = "total"; public static final String TOTAL_SUBNETS = "totalsubnets"; public static final String TYPE = "type"; public static final String TRUST_STORE = "truststore"; @@ -718,6 +722,7 @@ public class ApiConstants { public static final String POLICY_UUID = "policyuuid"; public static final String RULE_UUID = "ruleuuid"; public static final String DIRECTION = "direction"; + public static final String TAGGED_RESOURCES = "taggedresources"; public static final String TAG_UUID = "taguuid"; public static final String TAG_TYPE = "tagtype"; public static final String TAG_VALUE = "tagvalue"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/domain/ListDomainsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/domain/ListDomainsCmd.java index 8b6661f27ff2..40984b105828 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/domain/ListDomainsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/domain/ListDomainsCmd.java @@ -23,6 +23,7 @@ import com.cloud.server.ResourceIcon; import com.cloud.server.ResourceTag; import org.apache.cloudstack.api.response.ResourceIconResponse; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.apache.cloudstack.api.APICommand; @@ -73,6 +74,9 @@ public class ListDomainsCmd extends BaseListCmd implements UserCmd { description = "flag to display the resource icon for domains") private Boolean showIcon; + @Parameter(name = ApiConstants.TAG, type = CommandType.STRING, description = "Tag for resource type to return usage", since = "4.20.0") + private String tag; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -112,10 +116,14 @@ public EnumSet getDetails() throws InvalidParameterValueException return dv; } - public Boolean getShowIcon() { + public boolean getShowIcon() { return showIcon != null ? showIcon : false; } + public String getTag() { + return tag; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -130,12 +138,17 @@ public void execute() { ListResponse response = _queryService.searchForDomains(this); response.setResponseName(getCommandName()); this.setResponseObject(response); - if (response != null && response.getCount() > 0 && getShowIcon()) { - updateDomainResponse(response.getResponses()); - } + updateDomainResponse(response.getResponses()); } private void updateDomainResponse(List response) { + if (CollectionUtils.isEmpty(response)) { + return; + } + _resourceLimitService.updateTaggedResourceLimitsAndCountsForDomains(response, getTag()); + if (!getShowIcon()) { + return; + } for (DomainResponse domainResponse : response) { ResourceIcon resourceIcon = resourceIconManager.getByResourceTypeAndUuid(ResourceTag.ResourceObjectType.Domain, domainResponse.getId()); if (resourceIcon == null) { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListCapacityCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListCapacityCmd.java index 253677616f03..77f7a5bb5f10 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListCapacityCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListCapacityCmd.java @@ -73,6 +73,9 @@ public class ListCapacityCmd extends BaseListCmd { @Parameter(name = ApiConstants.SORT_BY, type = CommandType.STRING, since = "3.0.0", description = "Sort the results. Available values: Usage") private String sortBy; + @Parameter(name = ApiConstants.TAG, type = CommandType.STRING, description = "Tag for the resource type", since = "4.20.0") + private String tag; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -109,6 +112,10 @@ public String getSortBy() { return null; } + public String getTag() { + return tag; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/account/ListAccountsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/account/ListAccountsCmd.java index 66a4d91172af..6cd0c9cc389c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/account/ListAccountsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/account/ListAccountsCmd.java @@ -31,6 +31,7 @@ import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.ResourceIconResponse; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import com.cloud.exception.InvalidParameterValueException; @@ -75,6 +76,9 @@ public class ListAccountsCmd extends BaseListDomainResourcesCmd implements UserC description = "flag to display the resource icon for accounts") private Boolean showIcon; + @Parameter(name = ApiConstants.TAG, type = CommandType.STRING, description = "Tag for resource type to return usage", since = "4.20.0") + private String tag; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -118,10 +122,14 @@ public EnumSet getDetails() throws InvalidParameterValueException return dv; } - public Boolean getShowIcon() { + public boolean getShowIcon() { return showIcon != null ? showIcon : false; } + public String getTag() { + return tag; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -136,12 +144,17 @@ public void execute() { ListResponse response = _queryService.searchForAccounts(this); response.setResponseName(getCommandName()); setResponseObject(response); - if (response != null && response.getCount() > 0 && getShowIcon()) { - updateAccountResponse(response.getResponses()); - } + updateAccountResponse(response.getResponses()); } private void updateAccountResponse(List response) { + if (CollectionUtils.isEmpty(response)) { + return; + } + _resourceLimitService.updateTaggedResourceLimitsAndCountsForAccounts(response, getTag()); + if (!getShowIcon()) { + return; + } for (AccountResponse accountResponse : response) { ResourceIcon resourceIcon = resourceIconManager.getByResourceTypeAndUuid(ResourceTag.ResourceObjectType.Account, accountResponse.getObjectId()); if (resourceIcon == null) { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListDiskOfferingsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListDiskOfferingsCmd.java index e7284d515a26..28d682a8bee3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListDiskOfferingsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListDiskOfferingsCmd.java @@ -16,18 +16,17 @@ // under the License. package org.apache.cloudstack.api.command.user.offering; -import org.apache.cloudstack.api.BaseListProjectAndAccountResourcesCmd; -import org.apache.cloudstack.api.response.StoragePoolResponse; -import org.apache.cloudstack.api.response.VolumeResponse; -import org.apache.cloudstack.api.response.ZoneResponse; -import org.apache.log4j.Logger; - import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListProjectAndAccountResourcesCmd; import org.apache.cloudstack.api.Parameter; -import org.apache.cloudstack.api.BaseCmd.CommandType; import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.log4j.Logger; @APICommand(name = "listDiskOfferings", description = "Lists all available disk offerings.", responseObject = DiskOfferingResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) @@ -67,6 +66,13 @@ public class ListDiskOfferingsCmd extends BaseListProjectAndAccountResourcesCmd since = "4.19") private String storageType; + @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, + type = CommandType.UUID, + entityType = UserVmResponse.class, + description = "The ID of a virtual machine. Pass this in if you want to see the suitable disk offering that can be used to create and add a disk to the virtual machine. Suitability is returned with suitableforvirtualmachine flag in the response", + since = "4.20.0") + private Long virtualMachineId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -95,13 +101,16 @@ public String getStorageType() { return storageType; } + public Long getVirtualMachineId() { + return virtualMachineId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @Override public void execute() { - ListResponse response = _queryService.searchForDiskOfferings(this); response.setResponseName(getCommandName()); this.setResponseObject(response); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java index a9a699ed3ef0..9b6d9c2a0b25 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java @@ -16,17 +16,16 @@ // under the License. package org.apache.cloudstack.api.command.user.offering; -import org.apache.cloudstack.api.BaseListProjectAndAccountResourcesCmd; -import org.apache.cloudstack.api.response.ZoneResponse; -import org.apache.log4j.Logger; - import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListProjectAndAccountResourcesCmd; import org.apache.cloudstack.api.Parameter; -import org.apache.cloudstack.api.BaseCmd.CommandType; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.api.response.TemplateResponse; import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.log4j.Logger; @APICommand(name = "listServiceOfferings", description = "Lists all available service offerings.", responseObject = ServiceOfferingResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) @@ -95,6 +94,14 @@ public class ListServiceOfferingsCmd extends BaseListProjectAndAccountResourcesC since = "4.19") private String storageType; + @Parameter(name = ApiConstants.TEMPLATE_ID, + type = CommandType.UUID, + entityType = TemplateResponse.class, + description = "The ID of the template that listed offerings must support", + since = "4.20.0") + private Long templateId; + + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -141,6 +148,10 @@ public String getStorageType() { return storageType; } + public Long getTemplateId() { + return templateId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/resource/ListResourceLimitsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/resource/ListResourceLimitsCmd.java index adf1c93e4d03..55355953b8a2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/resource/ListResourceLimitsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/resource/ListResourceLimitsCmd.java @@ -19,8 +19,6 @@ import java.util.ArrayList; import java.util.List; -import com.cloud.configuration.Resource; -import com.cloud.exception.InvalidParameterValueException; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListProjectAndAccountResourcesCmd; @@ -29,7 +27,9 @@ import org.apache.cloudstack.api.response.ResourceLimitResponse; import org.apache.log4j.Logger; +import com.cloud.configuration.Resource; import com.cloud.configuration.ResourceLimit; +import com.cloud.exception.InvalidParameterValueException; @APICommand(name = "listResourceLimits", description = "Lists resource limits.", responseObject = ResourceLimitResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) @@ -74,6 +74,10 @@ public class ListResourceLimitsCmd extends BaseListProjectAndAccountResourcesCmd + "secondary_storage - SecondaryStorage. Total secondary storage space (in GiB) a user can use. ") private String resourceTypeName; + @Parameter(name = ApiConstants.TAG, type = CommandType.STRING, description = "Tag for the resource type", since = "4.20.0") + private String tag; + + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -90,6 +94,10 @@ public String getResourceTypeName() { return resourceTypeName; } + public String getTag() { + return tag; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -98,7 +106,7 @@ public String getResourceTypeName() { public void execute() { List result = _resourceLimitService.searchForLimits(id, _accountService.finalyzeAccountId(this.getAccountName(), this.getDomainId(), this.getProjectId(), false), this.getDomainId(), - getResourceTypeEnum(), this.getStartIndex(), this.getPageSizeVal()); + getResourceTypeEnum(), getTag(), this.getStartIndex(), this.getPageSizeVal()); ListResponse response = new ListResponse(); List limitResponses = new ArrayList(); for (ResourceLimit limit : result) { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/resource/UpdateResourceCountCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/resource/UpdateResourceCountCmd.java index 424087bc71ee..76b998edb922 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/resource/UpdateResourceCountCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/resource/UpdateResourceCountCmd.java @@ -77,6 +77,9 @@ public class UpdateResourceCountCmd extends BaseCmd { @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "Update resource limits for project") private Long projectId; + @Parameter(name = ApiConstants.TAG, type = CommandType.STRING, description = "Tag for the resource type", since = "4.20.0") + private String tag; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -93,6 +96,10 @@ public Integer getResourceType() { return resourceType; } + public String getTag() { + return tag; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -119,7 +126,7 @@ public long getEntityOwnerId() { @Override public void execute() { List result = - _resourceLimitService.recalculateResourceCount(_accountService.finalyzeAccountId(accountName, domainId, projectId, true), getDomainId(), getResourceType()); + _resourceLimitService.recalculateResourceCount(_accountService.finalyzeAccountId(accountName, domainId, projectId, true), getDomainId(), getResourceType(), getTag()); if ((result != null) && (result.size() > 0)) { ListResponse response = new ListResponse(); @@ -127,7 +134,6 @@ public void execute() { for (ResourceCount count : result) { ResourceCountResponse resourceCountResponse = _responseGenerator.createResourceCountResponse(count); - resourceCountResponse.setObjectName("resourcecount"); countResponses.add(resourceCountResponse); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/resource/UpdateResourceLimitCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/resource/UpdateResourceLimitCmd.java index 41676ed23ca2..4e50a6a131ac 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/resource/UpdateResourceLimitCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/resource/UpdateResourceLimitCmd.java @@ -72,6 +72,9 @@ public class UpdateResourceLimitCmd extends BaseCmd { + "11 - SecondaryStorage. Total secondary storage space (in GiB) a user can use. ") private Integer resourceType; + @Parameter(name = ApiConstants.TAG, type = CommandType.STRING, description = "Tag for the resource type", since = "4.20.0") + private String tag; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -84,6 +87,10 @@ public Long getDomainId() { return domainId; } + public String getTag() { + return tag; + } + public Integer getResourceType() { return resourceType; } @@ -104,7 +111,7 @@ public long getEntityOwnerId() { @Override public void execute() { - ResourceLimit result = _resourceLimitService.updateResourceLimit(_accountService.finalyzeAccountId(accountName, domainId, projectId, true), getDomainId(), resourceType, max); + ResourceLimit result = _resourceLimitService.updateResourceLimit(_accountService.finalyzeAccountId(accountName, domainId, projectId, true), getDomainId(), resourceType, max, getTag()); if (result != null || (result == null && max != null && max.longValue() == -1L)) { ResourceLimitResponse response = _responseGenerator.createResourceLimitResponse(result); response.setResponseName(getCommandName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java index 7ffe7d095e44..7a84e85a4a6f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java @@ -267,6 +267,10 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou @Param(description = "Base64 string representation of the resource icon", since = "4.16.0.0") ResourceIconResponse icon; + @SerializedName(ApiConstants.TAGGED_RESOURCES) + @Param(description = "The tagged resource limit and count for the account", since = "4.20.0") + List taggedResources; + @Override public String getObjectId() { return id; @@ -545,4 +549,9 @@ public void setGroups(List groups) { public void setResourceIconResponse(ResourceIconResponse icon) { this.icon = icon; } + + @Override + public void setTaggedResourceLimitsAndCounts(List taggedResourceLimitsAndCounts) { + this.taggedResources = taggedResourceLimitsAndCounts; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/CapacityResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/CapacityResponse.java index e9724497c38b..2d0e21586515 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/CapacityResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/CapacityResponse.java @@ -16,12 +16,11 @@ // under the License. package org.apache.cloudstack.api.response; -import com.google.gson.annotations.SerializedName; - import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; public class CapacityResponse extends BaseResponse { @SerializedName(ApiConstants.TYPE) @@ -72,6 +71,10 @@ public class CapacityResponse extends BaseResponse { @Param(description = "the percentage of capacity currently in use") private String percentUsed; + @SerializedName(ApiConstants.TAG) + @Param(description = "The tag for the capacity type", since = "4.20.0") + private String tag; + public Short getCapacityType() { return capacityType; } @@ -167,4 +170,8 @@ public String getPercentUsed() { public void setPercentUsed(String percentUsed) { this.percentUsed = percentUsed; } + + public void setTag(String tag) { + this.tag = tag; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/DiskOfferingResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/DiskOfferingResponse.java index b8244aebc608..4a453a46b648 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/DiskOfferingResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/DiskOfferingResponse.java @@ -169,6 +169,10 @@ public class DiskOfferingResponse extends BaseResponseWithAnnotations { @Param(description = "additional key/value details tied with this disk offering", since = "4.17") private Map details; + @SerializedName(ApiConstants.SUITABLE_FOR_VM) + @Param(description = "Returns true if the disk offering is suitable for the given virtual machine for disk creation otherwise false", since = "4.20.0") + private Boolean suitableForVm; + public Boolean getDisplayOffering() { return displayOffering; } @@ -391,4 +395,8 @@ public void setDiskSizeStrictness(Boolean diskSizeStrictness) { public void setDetails(Map details) { this.details = details; } + + public void setSuitableForVm(Boolean suitableForVm) { + this.suitableForVm = suitableForVm; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/DomainResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/DomainResponse.java index e4e409a40ee1..7c6ad3a91c38 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/DomainResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/DomainResponse.java @@ -26,6 +26,7 @@ import com.cloud.serializer.Param; import java.util.Date; +import java.util.List; import java.util.Map; @EntityReference(value = Domain.class) @@ -184,6 +185,10 @@ public class DomainResponse extends BaseResponseWithAnnotations implements Resou @Param(description = "details for the domain") private Map details; + @SerializedName(ApiConstants.TAGGED_RESOURCES) + @Param(description = "The tagged resource limit and count for the domain", since = "4.20.0") + List taggedResources; + public String getId() { return this.id; } @@ -447,4 +452,9 @@ public void setResourceIconResponse(ResourceIconResponse icon) { public void setDetails(Map details) { this.details = details; } + + @Override + public void setTaggedResourceLimitsAndCounts(List taggedResourceLimitsAndCounts) { + this.taggedResources = taggedResourceLimitsAndCounts; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ProjectResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ProjectResponse.java index c43dd09b127a..1c63697559b5 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ProjectResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ProjectResponse.java @@ -216,6 +216,10 @@ public class ProjectResponse extends BaseResponse implements ResourceLimitAndCou @Param(description = "the date this project was created", since = "4.16.0") private Date created; + @SerializedName(ApiConstants.TAGGED_RESOURCES) + @Param(description = "The tagged resource limit and count for the project", since = "4.20.0") + List taggedResources; + public void setId(String id) { this.id = id; } @@ -447,4 +451,9 @@ public Date getCreated() { public void setCreated(Date created) { this.created = created; } + + @Override + public void setTaggedResourceLimitsAndCounts(List taggedResourceLimitsAndCounts) { + this.taggedResources = taggedResourceLimitsAndCounts; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ResourceCountResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ResourceCountResponse.java index d0a4982f8724..3a6986185855 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ResourceCountResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ResourceCountResponse.java @@ -16,13 +16,12 @@ // under the License. package org.apache.cloudstack.api.response; -import com.cloud.configuration.Resource; -import com.google.gson.annotations.SerializedName; - import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; +import com.cloud.configuration.Resource; import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; @SuppressWarnings("unused") public class ResourceCountResponse extends BaseResponse implements ControlledEntityResponse { @@ -54,10 +53,14 @@ public class ResourceCountResponse extends BaseResponse implements ControlledEnt @Param(description = "resource type name. Values include user_vm, public_ip, volume, snapshot, template, project, network, vpc, cpu, memory, primary_storage, secondary_storage.") private String resourceTypeName; - @SerializedName("resourcecount") - @Param(description = "resource count") + @SerializedName(ApiConstants.RESOURCE_COUNT) + @Param(description = "The resource count") private long resourceCount; + @SerializedName(ApiConstants.TAG) + @Param(description = "Tag for the resource", since = "4.20.0") + private String tag; + @Override public void setAccountName(String accountName) { this.accountName = accountName; @@ -92,4 +95,7 @@ public void setProjectName(String projectName) { this.projectName = projectName; } + public void setTag(String tag) { + this.tag = tag; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java index f247be834cb1..f9e6df3a0386 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java @@ -20,6 +20,8 @@ package org.apache.cloudstack.api.response; +import java.util.List; + public interface ResourceLimitAndCountResponse { public void setNetworkLimit(String networkLimit); @@ -92,4 +94,6 @@ public interface ResourceLimitAndCountResponse { public void setVmRunning(Integer vmRunning); + public void setTaggedResourceLimitsAndCounts(List taggedResourceLimitsAndCounts); + } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ResourceLimitResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ResourceLimitResponse.java index 13e1198177d7..72c1c66f5e81 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ResourceLimitResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ResourceLimitResponse.java @@ -16,15 +16,14 @@ // under the License. package org.apache.cloudstack.api.response; -import com.cloud.configuration.Resource; -import com.google.gson.annotations.SerializedName; - import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import org.apache.cloudstack.api.EntityReference; +import com.cloud.configuration.Resource; import com.cloud.configuration.ResourceLimit; import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; @EntityReference(value = ResourceLimit.class) @SuppressWarnings("unused") @@ -61,6 +60,10 @@ public class ResourceLimitResponse extends BaseResponse implements ControlledEnt @Param(description = "the project name of the resource limit") private String projectName; + @SerializedName(ApiConstants.TAG) + @Param(description = "The tag for the resource limit", since = "4.20.0") + private String tag; + @Override public void setAccountName(String accountName) { this.accountName = accountName; @@ -94,4 +97,8 @@ public void setMax(Long max) { public void setProjectId(String projectId) { this.projectId = projectId; } + + public void setTag(String tag) { + this.tag = tag; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/TaggedResourceLimitAndCountResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/TaggedResourceLimitAndCountResponse.java new file mode 100644 index 000000000000..f0dfd64f74a5 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/TaggedResourceLimitAndCountResponse.java @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.response; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +import com.cloud.configuration.Resource; +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +public class TaggedResourceLimitAndCountResponse extends BaseResponse { + + @SerializedName(ApiConstants.RESOURCE_TYPE) + @Param(description = "Numerical value for the type of the resource. See the ResourceType for more information on these values.") + private Integer resourceType; + + @SerializedName(ApiConstants.RESOURCE_TYPE_NAME) + @Param(description = "Name for the type of the resource") + private String resourceTypeName; + + @SerializedName(ApiConstants.TAG) + @Param(description = "The tag for the resource type") + private String tag; + + @SerializedName(ApiConstants.LIMIT) + @Param(description = "The limit for the resource count for the type and tag for the owner") + private Long limit; + + @SerializedName(ApiConstants.TOTAL) + @Param(description = "The total amount of the resource for the type and tag that is used by the owner") + private Long total; + + @SerializedName(ApiConstants.AVAILABLE) + @Param(description = "The available amount of the resource for the type and tag that is available for the owner") + private Long available; + + + public void setResourceType(Resource.ResourceType resourceType) { + this.resourceType = resourceType.getOrdinal(); + this.resourceTypeName = resourceType.getName(); + } + + public void setTag(String tag) { + this.tag = tag; + } + + public void setLimit(Long limit) { + this.limit = limit; + } + + public void setTotal(Long total) { + this.total = total; + } + + public void setAvailable(Long available) { + this.available = available; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/user/ResourceReservation.java b/api/src/main/java/org/apache/cloudstack/user/ResourceReservation.java index 170193570cf1..9f3a661f485b 100644 --- a/api/src/main/java/org/apache/cloudstack/user/ResourceReservation.java +++ b/api/src/main/java/org/apache/cloudstack/user/ResourceReservation.java @@ -18,9 +18,10 @@ // package org.apache.cloudstack.user; -import com.cloud.configuration.Resource; import org.apache.cloudstack.api.InternalIdentity; +import com.cloud.configuration.Resource; + /** * an interface defining an {code}AutoClosable{code} reservation object */ @@ -33,5 +34,7 @@ Resource.ResourceType getResourceType(); + String getTag(); + Long getReservedAmount(); } diff --git a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java index 8cd67f253310..3f3bdefb6a06 100644 --- a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java +++ b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java @@ -290,4 +290,6 @@ static String getHypervisorHostname(String name) { HashMap> getVmNetworkStatistics(long hostId, String hostName, Map vmMap); + Map getDiskOfferingSuitabilityForVm(long vmId, List diskOfferingIds); + } diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index f5cf443211cf..427a58669e6d 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -277,6 +277,8 @@ static Boolean getFullCloneConfiguration(Long storeId) { CapacityVO getStoragePoolUsedStats(Long poolId, Long clusterId, Long podId, Long zoneId); + CapacityVO getStoragePoolUsedStats(Long zoneId, Long podId, Long clusterId, List poolIds); + List ListByDataCenterHypervisor(long datacenterId, HypervisorType type); List listByStoragePool(long storagePoolId); @@ -303,6 +305,9 @@ static Boolean getFullCloneConfiguration(Long storeId) { boolean storagePoolHasEnoughIops(List> volumeDiskProfilePairs, StoragePool pool); + boolean storagePoolHasEnoughIops(Long requestedIops, StoragePool pool); + boolean storagePoolHasEnoughSpace(Long size, StoragePool pool); + boolean storagePoolHasEnoughSpace(List> volumeDiskProfilePairs, StoragePool pool); /** @@ -335,6 +340,8 @@ static Boolean getFullCloneConfiguration(Long storeId) { boolean isStoragePoolCompliantWithStoragePolicy(List> volumes, StoragePool pool) throws StorageUnavailableException; + boolean isStoragePoolCompliantWithStoragePolicy(long diskOfferingId, StoragePool pool) throws StorageUnavailableException; + boolean registerHostListener(String providerUuid, HypervisorHostListener listener); boolean connectHostToSharedPool(long hostId, long poolId) throws StorageUnavailableException, StorageConflictException; diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index ccc005c39325..8c7f1209eddc 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -19,6 +19,7 @@ import static com.cloud.configuration.ConfigurationManagerImpl.MIGRATE_VM_ACROSS_CLUSTERS; +import java.lang.reflect.Field; import java.net.URI; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -149,7 +150,6 @@ import com.cloud.api.query.vo.DomainRouterJoinVO; import com.cloud.api.query.vo.UserVmJoinVO; import com.cloud.capacity.CapacityManager; -import com.cloud.configuration.Resource.ResourceType; import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterDetailsVO; import com.cloud.dc.ClusterVO; @@ -1132,7 +1132,7 @@ public void orchestrateStart(final String vmUuid, final Map> getVmNetworkStatistics(long } return vmNetworkStatsById; } + + protected boolean isDiskOfferingSuitableForVm(VMInstanceVO vm, VirtualMachineProfile profile, long podId, long clusterId, long hostId, long diskOfferingId) { + + DiskOfferingVO diskOffering = _diskOfferingDao.findById(diskOfferingId); + VolumeVO dummyVolume = new VolumeVO("Data", vm.getDataCenterId(), podId, vm.getAccountId(), + vm.getDomainId(), vm.getId(), null, null, diskOffering.getProvisioningType(), diskOffering.getDiskSize(), Type.DATADISK); + try { + Field idField = dummyVolume.getClass().getDeclaredField("id"); + idField.setAccessible(true); + idField.set(dummyVolume, Volume.DISK_OFFERING_SUITABILITY_CHECK_VOLUME_ID); + } catch (NoSuchFieldException | IllegalAccessException ignored) { + return false; + } + dummyVolume.setDiskOfferingId(diskOfferingId); + DiskProfile diskProfile = new DiskProfile(dummyVolume, diskOffering, profile.getHypervisorType()); + diskProfile.setMinIops(diskOffering.getMinIops()); + diskProfile.setMaxIops(diskOffering.getMaxIops()); + ExcludeList avoid = new ExcludeList(); + DataCenterDeployment plan = new DataCenterDeployment(vm.getDataCenterId(), podId, clusterId, hostId, null, null); + for (StoragePoolAllocator allocator : _storagePoolAllocators) { + List poolListFromAllocator = allocator.allocateToPool(diskProfile, profile, plan, avoid, 1); + if (CollectionUtils.isNotEmpty(poolListFromAllocator)) { + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("Found a suitable pool: %s for disk offering: %s", poolListFromAllocator.get(0).getName(), diskOffering.getName())); + } + return true; + } + } + return false; + } + + @Override + public Map getDiskOfferingSuitabilityForVm(long vmId, List diskOfferingIds) { + VMInstanceVO vm = _vmDao.findById(vmId); + VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm); + Pair clusterAndHost = findClusterAndHostIdForVm(vm, false); + Long clusterId = clusterAndHost.first(); + Cluster cluster = _clusterDao.findById(clusterId); + Map result = new HashMap<>(); + for (Long diskOfferingId : diskOfferingIds) { + result.put(diskOfferingId, isDiskOfferingSuitableForVm(vm, profile, cluster.getPodId(), clusterId, clusterAndHost.second(), diskOfferingId)); + } + return result; + } } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index b8f3e5a10e54..6b525c90ded2 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -131,6 +131,7 @@ import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.VMTemplateDetailsDao; @@ -247,6 +248,8 @@ public enum UserVmCloneType { PassphraseDao passphraseDao; @Inject StoragePoolHostDao storagePoolHostDao; + @Inject + DiskOfferingDao diskOfferingDao; @Inject protected SnapshotHelper snapshotHelper; @@ -880,9 +883,7 @@ public DiskProfile allocateRawVolume(Type type, String name, DiskOffering offeri if (vm.getType() == VirtualMachine.Type.User) { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), offering.getId(), null, size, Volume.class.getName(), vol.getUuid(), vol.isDisplayVolume()); - - _resourceLimitMgr.incrementResourceCount(vm.getAccountId(), ResourceType.volume, vol.isDisplayVolume()); - _resourceLimitMgr.incrementResourceCount(vm.getAccountId(), ResourceType.primary_storage, vol.isDisplayVolume(), new Long(vol.getSize())); + _resourceLimitMgr.incrementVolumeResourceCount(vm.getAccountId(), vol.isDisplayVolume(), vol.getSize(), offering); } DiskProfile diskProfile = toDiskProfile(vol, offering); @@ -964,8 +965,7 @@ private DiskProfile allocateTemplatedVolume(Type type, String name, DiskOffering UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), offeringId, vol.getTemplateId(), size, Volume.class.getName(), vol.getUuid(), vol.isDisplayVolume()); - _resourceLimitMgr.incrementResourceCount(vm.getAccountId(), ResourceType.volume, vol.isDisplayVolume()); - _resourceLimitMgr.incrementResourceCount(vm.getAccountId(), ResourceType.primary_storage, vol.isDisplayVolume(), new Long(vol.getSize())); + _resourceLimitMgr.incrementVolumeResourceCount(vm.getAccountId(), vol.isDisplayVolume(), vol.getSize(), offering); } return toDiskProfile(vol, offering); } @@ -1144,6 +1144,10 @@ public VolumeInfo createVolumeOnPrimaryStorage(VirtualMachine vm, VolumeInfo vol // Moving of Volume is successful, decrement the volume resource count from secondary for an account and increment it into primary storage under same account. _resourceLimitMgr.decrementResourceCount(volumeInfo.getAccountId(), ResourceType.secondary_storage, volumeInfo.getSize()); _resourceLimitMgr.incrementResourceCount(volumeInfo.getAccountId(), ResourceType.primary_storage, volumeInfo.getSize()); + List tags = _resourceLimitMgr.getResourceLimitStorageTags(diskVO); + for (String tag : tags) { + _resourceLimitMgr.incrementResourceCountWithTag(volumeInfo.getAccountId(), ResourceType.primary_storage, tag, volumeInfo.getSize()); + } } } @@ -2104,8 +2108,7 @@ public void destroyVolume(Volume volume) { if (volume.getState() == Volume.State.Allocated) { _volsDao.remove(volume.getId()); stateTransitTo(volume, Volume.Event.DestroyRequested); - _resourceLimitMgr.decrementResourceCount(volume.getAccountId(), ResourceType.volume, volume.isDisplay()); - _resourceLimitMgr.decrementResourceCount(volume.getAccountId(), ResourceType.primary_storage, volume.isDisplay(), new Long(volume.getSize())); + _resourceLimitMgr.decrementVolumeResourceCount(volume.getAccountId(), volume.isDisplay(), volume.getSize(), diskOfferingDao.findByIdIncludingRemoved(volume.getDiskOfferingId())); } else { destroyVolumeInContext(volume); } diff --git a/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDao.java b/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDao.java index 459a63a7ba13..9616f31d0c5c 100644 --- a/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDao.java +++ b/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDao.java @@ -44,6 +44,8 @@ public interface CapacityDao extends GenericDao { List findCapacityBy(Integer capacityType, Long zoneId, Long podId, Long clusterId); + List findFilteredCapacityBy(Integer capacityType, Long zoneId, Long podId, Long clusterId, List hostIds, List poolIds); + List listPodsByHostCapacities(long zoneId, int requiredCpu, long requiredRam, short capacityType); Pair, Map> orderPodsByAggregateCapacity(long zoneId, short capacityType); @@ -51,7 +53,8 @@ public interface CapacityDao extends GenericDao { List findCapacityBy(Integer capacityType, Long zoneId, Long podId, Long clusterId, String resourceState); - List listCapacitiesGroupedByLevelAndType(Integer capacityType, Long zoneId, Long podId, Long clusterId, int level, Long limit); + List listCapacitiesGroupedByLevelAndType(Integer capacityType, Long zoneId, Long podId, + Long clusterId, int level, List hostIds, List poolIds, Long limit); void updateCapacityState(Long dcId, Long podId, Long clusterId, Long hostId, String capacityState, short[] capacityType); diff --git a/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java b/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java index 302ffd8e760a..6c065831b3f7 100644 --- a/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java @@ -26,6 +26,7 @@ import javax.inject.Inject; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; @@ -341,7 +342,8 @@ public List findCapacityBy(Integer capacityType, Long zoneId, Lo } @Override - public List listCapacitiesGroupedByLevelAndType(Integer capacityType, Long zoneId, Long podId, Long clusterId, int level, Long limit) { + public List listCapacitiesGroupedByLevelAndType(Integer capacityType, Long zoneId, Long podId, + Long clusterId, int level, List hostIds, List poolIds, Long limit) { StringBuilder finalQuery = new StringBuilder(); TransactionLegacy txn = TransactionLegacy.currentTxn(); @@ -380,6 +382,18 @@ public List listCapacitiesGroupedByLevelAndType(Integer capacity finalQuery.append(" AND capacity_type = ?"); resourceIdList.add(capacityType.longValue()); } + if (CollectionUtils.isNotEmpty(hostIds)) { + finalQuery.append(String.format(" AND capacity.host_id IN (%s)", StringUtils.join(hostIds, ","))); + if (capacityType == null) { + finalQuery.append(String.format(" AND capacity_type NOT IN (%s)", StringUtils.join(Capacity.STORAGE_CAPACITY_TYPES, ","))); + } + } + if (CollectionUtils.isNotEmpty(poolIds)) { + finalQuery.append(String.format(" AND capacity.host_id IN (%s)", StringUtils.join(poolIds, ","))); + if (capacityType == null) { + finalQuery.append(String.format(" AND capacity_type IN (%s)", StringUtils.join(Capacity.STORAGE_CAPACITY_TYPES, ","))); + } + } switch (level) { case 1: // List all the capacities grouped by zone, capacity Type @@ -464,7 +478,7 @@ public Ternary findCapacityByZoneAndHostTag(Long zoneId, Strin } @Override - public List findCapacityBy(Integer capacityType, Long zoneId, Long podId, Long clusterId) { + public List findFilteredCapacityBy(Integer capacityType, Long zoneId, Long podId, Long clusterId, List hostIds, List poolIds) { TransactionLegacy txn = TransactionLegacy.currentTxn(); PreparedStatement pstmt = null; @@ -517,6 +531,18 @@ public List findCapacityBy(Integer capacityType, Long zoneId, Lo sql.append(" AND capacity.capacity_type = ?"); resourceIdList.add(capacityType.longValue()); } + if (CollectionUtils.isNotEmpty(hostIds)) { + sql.append(String.format(" AND capacity.host_id IN (%s)", StringUtils.join(hostIds, ","))); + if (capacityType == null) { + sql.append(String.format(" AND capacity_type NOT IN (%s)", StringUtils.join(Capacity.STORAGE_CAPACITY_TYPES, ","))); + } + } + if (CollectionUtils.isNotEmpty(poolIds)) { + sql.append(String.format(" AND capacity.host_id IN (%s)", StringUtils.join(poolIds, ","))); + if (capacityType == null) { + sql.append(String.format(" AND capacity_type IN (%s)", StringUtils.join(Capacity.STORAGE_CAPACITY_TYPES, ","))); + } + } if (podId == null && clusterId == null) { sql.append(" GROUP BY capacity_type, data_center_id"); @@ -593,6 +619,11 @@ public List findCapacityBy(Integer capacityType, Long zoneId, Lo } } + @Override + public List findCapacityBy(Integer capacityType, Long zoneId, Long podId, Long clusterId) { + return findFilteredCapacityBy(capacityType, zoneId, podId, clusterId, null, null); + } + public void updateAllocated(Long hostId, long allocatedAmount, short capacityType, boolean add) { TransactionLegacy txn = TransactionLegacy.currentTxn(); PreparedStatement pstmt = null; diff --git a/engine/schema/src/main/java/com/cloud/configuration/ResourceCountVO.java b/engine/schema/src/main/java/com/cloud/configuration/ResourceCountVO.java index ae8f3822704c..9e39a608f9ec 100644 --- a/engine/schema/src/main/java/com/cloud/configuration/ResourceCountVO.java +++ b/engine/schema/src/main/java/com/cloud/configuration/ResourceCountVO.java @@ -47,10 +47,13 @@ public class ResourceCountVO implements ResourceCount { @Column(name = "count") private long count; + @Column(name = "tag") + private String tag; + public ResourceCountVO() { } - public ResourceCountVO(ResourceType type, long count, long ownerId, ResourceOwnerType ownerType) { + public ResourceCountVO(ResourceType type, long count, long ownerId, ResourceOwnerType ownerType, String tag) { this.type = type; this.count = count; @@ -59,6 +62,11 @@ public ResourceCountVO(ResourceType type, long count, long ownerId, ResourceOwne } else if (ownerType == ResourceOwnerType.Domain) { this.domainId = ownerId; } + this.tag = tag; + } + + public ResourceCountVO(ResourceType type, long count, long ownerId, ResourceOwnerType ownerType) { + this(type, count, ownerId, ownerType, null); } @Override @@ -99,7 +107,7 @@ public Long getAccountId() { @Override public String toString() { - return new StringBuilder("REsourceCount[").append("-") + return new StringBuilder("ResourceCount[").append("-") .append(id) .append("-") .append(type) @@ -107,6 +115,8 @@ public String toString() { .append(accountId) .append("-") .append(domainId) + .append("-") + .append(tag) .append("]") .toString(); } @@ -136,4 +146,13 @@ public void setDomainId(Long domainId) { public void setAccountId(Long accountId) { this.accountId = accountId; } + + @Override + public String getTag() { + return tag; + } + + public void setTag(String tag) { + this.tag = tag; + } } diff --git a/engine/schema/src/main/java/com/cloud/configuration/ResourceLimitVO.java b/engine/schema/src/main/java/com/cloud/configuration/ResourceLimitVO.java index 392170919757..1619537ae744 100644 --- a/engine/schema/src/main/java/com/cloud/configuration/ResourceLimitVO.java +++ b/engine/schema/src/main/java/com/cloud/configuration/ResourceLimitVO.java @@ -47,10 +47,13 @@ public class ResourceLimitVO implements ResourceLimit { @Column(name = "max") private Long max; + @Column(name = "tag") + private String tag; + public ResourceLimitVO() { } - public ResourceLimitVO(ResourceCount.ResourceType type, Long max, long ownerId, ResourceOwnerType ownerType) { + public ResourceLimitVO(ResourceCount.ResourceType type, Long max, long ownerId, ResourceOwnerType ownerType, String tag) { this.type = type; this.max = max; @@ -59,6 +62,11 @@ public ResourceLimitVO(ResourceCount.ResourceType type, Long max, long ownerId, } else if (ownerType == ResourceOwnerType.Domain) { this.domainId = ownerId; } + this.tag = tag; + } + + public ResourceLimitVO(ResourceCount.ResourceType type, Long max, long ownerId, ResourceOwnerType ownerType) { + this(type, max, ownerId, ownerType, null); } @Override @@ -123,4 +131,12 @@ public void setAccountId(Long accountId) { this.accountId = accountId; } + @Override + public String getTag() { + return tag; + } + + public void setTag(String tag) { + this.tag = tag; + } } diff --git a/engine/schema/src/main/java/com/cloud/configuration/dao/ResourceCountDao.java b/engine/schema/src/main/java/com/cloud/configuration/dao/ResourceCountDao.java index 28f2a5360716..5f0a86c4b65b 100644 --- a/engine/schema/src/main/java/com/cloud/configuration/dao/ResourceCountDao.java +++ b/engine/schema/src/main/java/com/cloud/configuration/dao/ResourceCountDao.java @@ -26,18 +26,20 @@ public interface ResourceCountDao extends GenericDao { /** - * @param domainId the id of the domain to get the resource count + * @param ownerId the id of the owner to get the resource count * @param type the type of resource (e.g. user_vm, public_ip, volume) + * @param tag for the type of resource * @return the count of resources in use for the given type and domain */ - long getResourceCount(long ownerId, ResourceOwnerType ownerType, ResourceType type); + long getResourceCount(long ownerId, ResourceOwnerType ownerType, ResourceType type, String tag); /** - * @param domainId the id of the domain to set the resource count + * @param ownerId the id of the owner to set the resource count * @param type the type of resource (e.g. user_vm, public_ip, volume) - * @param the count of resources in use for the given type and domain + * @param tag the tag for the type of resource + * @param count the count of resources in use for the given type and domain */ - void setResourceCount(long ownerId, ResourceOwnerType ownerType, ResourceType type, long count); + void setResourceCount(long ownerId, ResourceOwnerType ownerType, ResourceType type, String tag, long count); boolean updateById(long id, boolean increment, long delta); @@ -45,13 +47,13 @@ public interface ResourceCountDao extends GenericDao { List listByOwnerId(long ownerId, ResourceOwnerType ownerType); - ResourceCountVO findByOwnerAndType(long ownerId, ResourceOwnerType ownerType, ResourceType type); + ResourceCountVO findByOwnerAndTypeAndTag(long ownerId, ResourceOwnerType ownerType, ResourceType type, String tag); List listResourceCountByOwnerType(ResourceOwnerType ownerType); - Set listAllRowsToUpdate(long ownerId, ResourceOwnerType ownerType, ResourceType type); + Set listAllRowsToUpdate(long ownerId, ResourceOwnerType ownerType, ResourceType type, String tag); - Set listRowsToUpdateForDomain(long domainId, ResourceType type); + Set listRowsToUpdateForDomain(long domainId, ResourceType type, String tag); long removeEntriesByOwner(long ownerId, ResourceOwnerType ownerType); @@ -68,4 +70,6 @@ public interface ResourceCountDao extends GenericDao { * Side note: This method is not using the "resource_count" table. It is executing the actual count instead. */ long countMemoryAllocatedToAccount(long accountId); + + void removeResourceCountsForNonMatchingTags(long ownerId, ResourceOwnerType ownerType, ResourceType type, List tags); } diff --git a/engine/schema/src/main/java/com/cloud/configuration/dao/ResourceCountDaoImpl.java b/engine/schema/src/main/java/com/cloud/configuration/dao/ResourceCountDaoImpl.java index ca6f13d2d645..ce042edbf8bb 100644 --- a/engine/schema/src/main/java/com/cloud/configuration/dao/ResourceCountDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/configuration/dao/ResourceCountDaoImpl.java @@ -20,6 +20,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -27,6 +28,8 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import com.cloud.configuration.Resource; @@ -37,6 +40,7 @@ import com.cloud.domain.DomainVO; import com.cloud.domain.dao.DomainDao; import com.cloud.user.AccountVO; +import com.cloud.user.ResourceLimitService; import com.cloud.user.dao.AccountDao; import com.cloud.utils.db.DB; import com.cloud.utils.db.GenericDaoBase; @@ -49,6 +53,8 @@ @Component public class ResourceCountDaoImpl extends GenericDaoBase implements ResourceCountDao { private final SearchBuilder TypeSearch; + private final SearchBuilder TypeNullTagSearch; + private final SearchBuilder NonMatchingTagsSearch; private final SearchBuilder AccountSearch; private final SearchBuilder DomainSearch; @@ -63,8 +69,24 @@ public ResourceCountDaoImpl() { TypeSearch.and("type", TypeSearch.entity().getType(), SearchCriteria.Op.EQ); TypeSearch.and("accountId", TypeSearch.entity().getAccountId(), SearchCriteria.Op.EQ); TypeSearch.and("domainId", TypeSearch.entity().getDomainId(), SearchCriteria.Op.EQ); + TypeSearch.and("tag", TypeSearch.entity().getTag(), SearchCriteria.Op.EQ); TypeSearch.done(); + TypeNullTagSearch = createSearchBuilder(); + TypeNullTagSearch.and("type", TypeNullTagSearch.entity().getType(), SearchCriteria.Op.EQ); + TypeNullTagSearch.and("accountId", TypeNullTagSearch.entity().getAccountId(), SearchCriteria.Op.EQ); + TypeNullTagSearch.and("domainId", TypeNullTagSearch.entity().getDomainId(), SearchCriteria.Op.EQ); + TypeNullTagSearch.and("tag", TypeNullTagSearch.entity().getTag(), SearchCriteria.Op.NULL); + TypeNullTagSearch.done(); + + NonMatchingTagsSearch = createSearchBuilder(); + NonMatchingTagsSearch.and("accountId", NonMatchingTagsSearch.entity().getAccountId(), SearchCriteria.Op.EQ); + NonMatchingTagsSearch.and("domainId", NonMatchingTagsSearch.entity().getDomainId(), SearchCriteria.Op.EQ); + NonMatchingTagsSearch.and("type", NonMatchingTagsSearch.entity().getType(), SearchCriteria.Op.EQ); + NonMatchingTagsSearch.and("tagNotNull", NonMatchingTagsSearch.entity().getTag(), SearchCriteria.Op.NNULL); + NonMatchingTagsSearch.and("tags", NonMatchingTagsSearch.entity().getTag(), SearchCriteria.Op.NIN); + NonMatchingTagsSearch.done(); + AccountSearch = createSearchBuilder(); DomainSearch = createSearchBuilder(); } @@ -85,9 +107,12 @@ protected void configure() { } @Override - public ResourceCountVO findByOwnerAndType(long ownerId, ResourceOwnerType ownerType, ResourceType type) { - SearchCriteria sc = TypeSearch.create(); + public ResourceCountVO findByOwnerAndTypeAndTag(long ownerId, ResourceOwnerType ownerType, ResourceType type, String tag) { + SearchCriteria sc = tag != null ? TypeSearch.create() : TypeNullTagSearch.create(); sc.setParameters("type", type); + if (tag != null) { + sc.setParameters("tag", tag); + } if (ownerType == ResourceOwnerType.Account) { sc.setParameters("accountId", ownerId); @@ -101,8 +126,8 @@ public ResourceCountVO findByOwnerAndType(long ownerId, ResourceOwnerType ownerT } @Override - public long getResourceCount(long ownerId, ResourceOwnerType ownerType, ResourceType type) { - ResourceCountVO vo = findByOwnerAndType(ownerId, ownerType, type); + public long getResourceCount(long ownerId, ResourceOwnerType ownerType, ResourceType type, String tag) { + ResourceCountVO vo = findByOwnerAndTypeAndTag(ownerId, ownerType, type, tag); if (vo != null) { return vo.getCount(); } else { @@ -111,8 +136,8 @@ public long getResourceCount(long ownerId, ResourceOwnerType ownerType, Resource } @Override - public void setResourceCount(long ownerId, ResourceOwnerType ownerType, ResourceType type, long count) { - ResourceCountVO resourceCountVO = findByOwnerAndType(ownerId, ownerType, type); + public void setResourceCount(long ownerId, ResourceOwnerType ownerType, ResourceType type, String tag, long count) { + ResourceCountVO resourceCountVO = findByOwnerAndTypeAndTag(ownerId, ownerType, type, tag); if (resourceCountVO != null && count != resourceCountVO.getCount()) { resourceCountVO.setCount(count); update(resourceCountVO.getId(), resourceCountVO); @@ -129,38 +154,59 @@ public boolean updateById(long id, boolean increment, long delta) { } @Override - public Set listRowsToUpdateForDomain(long domainId, ResourceType type) { + public Set listRowsToUpdateForDomain(long domainId, ResourceType type, String tag) { Set rowIds = new HashSet(); Set domainIdsToUpdate = _domainDao.getDomainParentIds(domainId); for (Long domainIdToUpdate : domainIdsToUpdate) { - ResourceCountVO domainCountRecord = findByOwnerAndType(domainIdToUpdate, ResourceOwnerType.Domain, type); + ResourceCountVO domainCountRecord = findByOwnerAndTypeAndTag(domainIdToUpdate, ResourceOwnerType.Domain, type, tag); if (domainCountRecord != null) { rowIds.add(domainCountRecord.getId()); + } else { + if (StringUtils.isNotEmpty(tag)) { + ResourceCountVO resourceCountVO = createTaggedResourceCount(domainIdToUpdate, ResourceOwnerType.Domain, type, tag); + rowIds.add(resourceCountVO.getId()); + } } } return rowIds; } @Override - public Set listAllRowsToUpdate(long ownerId, ResourceOwnerType ownerType, ResourceType type) { + public Set listAllRowsToUpdate(long ownerId, ResourceOwnerType ownerType, ResourceType type, String tag) { Set rowIds = new HashSet(); if (ownerType == ResourceOwnerType.Account) { //get records for account - ResourceCountVO accountCountRecord = findByOwnerAndType(ownerId, ResourceOwnerType.Account, type); + ResourceCountVO accountCountRecord = findByOwnerAndTypeAndTag(ownerId, ResourceOwnerType.Account, type, tag); if (accountCountRecord != null) { rowIds.add(accountCountRecord.getId()); + } else { + if (StringUtils.isNotEmpty(tag)) { + ResourceCountVO resourceCountVO = createTaggedResourceCount(ownerId, ownerType, type, tag); + rowIds.add(resourceCountVO.getId()); + } } //get records for account's domain and all its parent domains - rowIds.addAll(listRowsToUpdateForDomain(_accountDao.findByIdIncludingRemoved(ownerId).getDomainId(), type)); + rowIds.addAll(listRowsToUpdateForDomain(_accountDao.findByIdIncludingRemoved(ownerId).getDomainId(), type, tag)); } else if (ownerType == ResourceOwnerType.Domain) { - return listRowsToUpdateForDomain(ownerId, type); + rowIds = listRowsToUpdateForDomain(ownerId, type, tag); } return rowIds; } + protected ResourceCountVO createTaggedResourceCount(long ownerId, ResourceLimit.ResourceOwnerType ownerType, ResourceType resourceType, String tag) { + ResourceCountVO taggedResourceCountVO = new ResourceCountVO(resourceType, 0, ownerId, ownerType, tag); + return persist(taggedResourceCountVO); + } + + protected void createTaggedResourceCounts(long ownerId, ResourceLimit.ResourceOwnerType ownerType, ResourceType resourceType, List tags) { + for (String tag : tags) { + createTaggedResourceCount(ownerId, ownerType, resourceType, tag); + } + } + @Override @DB public void createResourceCounts(long ownerId, ResourceLimit.ResourceOwnerType ownerType) { @@ -169,9 +215,23 @@ public void createResourceCounts(long ownerId, ResourceLimit.ResourceOwnerType o txn.start(); ResourceType[] resourceTypes = Resource.ResourceType.values(); + List hostTags = new ArrayList<>(); + if (StringUtils.isNotEmpty(ResourceLimitService.ResourceLimitHostTags.value())) { + hostTags = Arrays.asList(ResourceLimitService.ResourceLimitHostTags.value().split(",")); + } + List storageTags = new ArrayList<>(); + if (StringUtils.isNotEmpty(ResourceLimitService.ResourceLimitStorageTags.value())) { + storageTags = Arrays.asList(ResourceLimitService.ResourceLimitStorageTags.value().split(",")); + } for (ResourceType resourceType : resourceTypes) { ResourceCountVO resourceCountVO = new ResourceCountVO(resourceType, 0, ownerId, ownerType); persist(resourceCountVO); + if (ResourceLimitService.HostTagsSupportingTypes.contains(resourceType)) { + createTaggedResourceCounts(ownerId, ownerType, resourceType, hostTags); + } + if (ResourceLimitService.StorageTagsSupportingTypes.contains(resourceType)) { + createTaggedResourceCounts(ownerId, ownerType, resourceType, storageTags); + } } txn.commit(); @@ -266,4 +326,18 @@ private long executeSqlCountComputingResourcesForAccount(long accountId, String } } + @Override + public void removeResourceCountsForNonMatchingTags(long ownerId, ResourceOwnerType ownerType, ResourceType type, List tags) { + SearchCriteria sc = NonMatchingTagsSearch.create(); + if (ResourceOwnerType.Account.equals(ownerType)) { + sc.setParameters("accountId", ownerId); + } else { + sc.setParameters("domainId", ownerId); + } + sc.setParameters("type", type); + if (CollectionUtils.isNotEmpty(tags)) { + sc.setParameters("tags", tags.toArray()); + } + remove(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/configuration/dao/ResourceLimitDao.java b/engine/schema/src/main/java/com/cloud/configuration/dao/ResourceLimitDao.java index e47b38340c2b..424ddc81a910 100644 --- a/engine/schema/src/main/java/com/cloud/configuration/dao/ResourceLimitDao.java +++ b/engine/schema/src/main/java/com/cloud/configuration/dao/ResourceLimitDao.java @@ -18,6 +18,7 @@ import java.util.List; +import com.cloud.configuration.Resource; import com.cloud.configuration.Resource.ResourceOwnerType; import com.cloud.configuration.ResourceCount; import com.cloud.configuration.ResourceLimitVO; @@ -31,7 +32,8 @@ public interface ResourceLimitDao extends GenericDao { ResourceCount.ResourceType getLimitType(String type); - ResourceLimitVO findByOwnerIdAndType(long ownerId, ResourceOwnerType ownerType, ResourceCount.ResourceType type); + ResourceLimitVO findByOwnerIdAndTypeAndTag(long ownerId, ResourceOwnerType ownerType, ResourceCount.ResourceType type, String tag); long removeEntriesByOwner(Long ownerId, ResourceOwnerType ownerType); + void removeResourceLimitsForNonMatchingTags(long ownerId, ResourceOwnerType ownerType, Resource.ResourceType type, List tags); } diff --git a/engine/schema/src/main/java/com/cloud/configuration/dao/ResourceLimitDaoImpl.java b/engine/schema/src/main/java/com/cloud/configuration/dao/ResourceLimitDaoImpl.java index 03c2d2a46247..25721c341ec5 100644 --- a/engine/schema/src/main/java/com/cloud/configuration/dao/ResourceLimitDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/configuration/dao/ResourceLimitDaoImpl.java @@ -20,6 +20,7 @@ import java.util.List; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.configuration.Resource; @@ -33,19 +34,36 @@ @Component public class ResourceLimitDaoImpl extends GenericDaoBase implements ResourceLimitDao { - private SearchBuilder IdTypeSearch; + private SearchBuilder IdTypeTagSearch; + private SearchBuilder IdTypeNullTagSearch; + private SearchBuilder NonMatchingTagsSearch; public ResourceLimitDaoImpl() { - IdTypeSearch = createSearchBuilder(); - IdTypeSearch.and("type", IdTypeSearch.entity().getType(), SearchCriteria.Op.EQ); - IdTypeSearch.and("domainId", IdTypeSearch.entity().getDomainId(), SearchCriteria.Op.EQ); - IdTypeSearch.and("accountId", IdTypeSearch.entity().getAccountId(), SearchCriteria.Op.EQ); - IdTypeSearch.done(); + IdTypeTagSearch = createSearchBuilder(); + IdTypeTagSearch.and("type", IdTypeTagSearch.entity().getType(), SearchCriteria.Op.EQ); + IdTypeTagSearch.and("domainId", IdTypeTagSearch.entity().getDomainId(), SearchCriteria.Op.EQ); + IdTypeTagSearch.and("accountId", IdTypeTagSearch.entity().getAccountId(), SearchCriteria.Op.EQ); + IdTypeTagSearch.and("tag", IdTypeTagSearch.entity().getTag(), SearchCriteria.Op.EQ); + + IdTypeNullTagSearch = createSearchBuilder(); + IdTypeNullTagSearch.and("type", IdTypeNullTagSearch.entity().getType(), SearchCriteria.Op.EQ); + IdTypeNullTagSearch.and("domainId", IdTypeNullTagSearch.entity().getDomainId(), SearchCriteria.Op.EQ); + IdTypeNullTagSearch.and("accountId", IdTypeNullTagSearch.entity().getAccountId(), SearchCriteria.Op.EQ); + IdTypeNullTagSearch.and("tag", IdTypeNullTagSearch.entity().getTag(), SearchCriteria.Op.NULL); + IdTypeNullTagSearch.done(); + + NonMatchingTagsSearch = createSearchBuilder(); + NonMatchingTagsSearch.and("accountId", NonMatchingTagsSearch.entity().getAccountId(), SearchCriteria.Op.EQ); + NonMatchingTagsSearch.and("domainId", NonMatchingTagsSearch.entity().getDomainId(), SearchCriteria.Op.EQ); + NonMatchingTagsSearch.and("type", NonMatchingTagsSearch.entity().getType(), SearchCriteria.Op.EQ); + NonMatchingTagsSearch.and("tagNotNull", NonMatchingTagsSearch.entity().getTag(), SearchCriteria.Op.NNULL); + NonMatchingTagsSearch.and("tags", NonMatchingTagsSearch.entity().getTag(), SearchCriteria.Op.NIN); + NonMatchingTagsSearch.done(); } @Override public List listByOwner(Long ownerId, ResourceOwnerType ownerType) { - SearchCriteria sc = IdTypeSearch.create(); + SearchCriteria sc = IdTypeTagSearch.create(); if (ownerType == ResourceOwnerType.Account) { sc.setParameters("accountId", ownerId); @@ -81,9 +99,12 @@ public ResourceCount.ResourceType getLimitType(String type) { } @Override - public ResourceLimitVO findByOwnerIdAndType(long ownerId, ResourceOwnerType ownerType, ResourceCount.ResourceType type) { - SearchCriteria sc = IdTypeSearch.create(); + public ResourceLimitVO findByOwnerIdAndTypeAndTag(long ownerId, ResourceOwnerType ownerType, ResourceCount.ResourceType type, String tag) { + SearchCriteria sc = tag != null ? IdTypeTagSearch.create() : IdTypeNullTagSearch.create(); sc.setParameters("type", type); + if (tag != null) { + sc.setParameters("tag", tag); + } if (ownerType == ResourceOwnerType.Account) { sc.setParameters("accountId", ownerId); @@ -98,7 +119,7 @@ public ResourceLimitVO findByOwnerIdAndType(long ownerId, ResourceOwnerType owne @Override public long removeEntriesByOwner(Long ownerId, ResourceOwnerType ownerType) { - SearchCriteria sc = IdTypeSearch.create(); + SearchCriteria sc = IdTypeTagSearch.create(); if (ownerType == ResourceOwnerType.Account) { sc.setParameters("accountId", ownerId); @@ -109,4 +130,19 @@ public long removeEntriesByOwner(Long ownerId, ResourceOwnerType ownerType) { } return 0; } + + @Override + public void removeResourceLimitsForNonMatchingTags(long ownerId, ResourceOwnerType ownerType, ResourceType type, List tags) { + SearchCriteria sc = NonMatchingTagsSearch.create(); + if (ResourceOwnerType.Account.equals(ownerType)) { + sc.setParameters("accountId", ownerId); + } else { + sc.setParameters("domainId", ownerId); + } + sc.setParameters("type", type); + if (CollectionUtils.isNotEmpty(tags)) { + sc.setParameters("tags", tags.toArray()); + } + remove(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/host/HostVO.java b/engine/schema/src/main/java/com/cloud/host/HostVO.java index 697401ad0696..c9055f431e90 100644 --- a/engine/schema/src/main/java/com/cloud/host/HostVO.java +++ b/engine/schema/src/main/java/com/cloud/host/HostVO.java @@ -16,8 +16,10 @@ // under the License. package com.cloud.host; +import java.util.ArrayList; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.UUID; @@ -44,6 +46,7 @@ import com.cloud.offering.ServiceOffering; import com.cloud.resource.ResourceState; import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.template.VirtualMachineTemplate; import com.cloud.utils.NumbersUtil; import com.cloud.utils.db.GenericDao; import java.util.Arrays; @@ -761,7 +764,28 @@ public void setUuid(String uuid) { this.uuid = uuid; } - public boolean checkHostServiceOfferingTags(ServiceOffering serviceOffering){ + public boolean checkHostServiceOfferingAndTemplateTags(ServiceOffering serviceOffering, VirtualMachineTemplate template) { + if (serviceOffering == null || template == null) { + return false; + } + if (StringUtils.isEmpty(serviceOffering.getHostTag()) && StringUtils.isEmpty(template.getTemplateTag())) { + return true; + } + if (getHostTags() == null) { + return false; + } + HashSet hostTagsSet = new HashSet<>(getHostTags()); + List tags = new ArrayList<>(); + if (StringUtils.isNotEmpty(serviceOffering.getHostTag())) { + tags.addAll(Arrays.asList(serviceOffering.getHostTag().split(","))); + } + if (StringUtils.isNotEmpty(template.getTemplateTag()) && !tags.contains(template.getTemplateTag())) { + tags.add(template.getTemplateTag()); + } + return hostTagsSet.containsAll(tags); + } + + public boolean checkHostServiceOfferingTags(ServiceOffering serviceOffering) { if (serviceOffering == null) { return false; } @@ -773,7 +797,6 @@ public boolean checkHostServiceOfferingTags(ServiceOffering serviceOffering){ if (StringUtils.isEmpty(serviceOffering.getHostTag())) { return true; } - List serviceOfferingTags = Arrays.asList(serviceOffering.getHostTag().split(",")); return this.getHostTags() != null && this.getHostTags().containsAll(serviceOfferingTags); } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java index fe30722feb18..ca180e2323fd 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java @@ -63,7 +63,7 @@ public interface HostDao extends GenericDao, StateDao listByHostTag(Host.Type type, Long clusterId, Long podId, long dcId, String hostTag); + List listByHostTag(Host.Type type, Long clusterId, Long podId, Long dcId, String hostTag); List findAndUpdateApplianceToLoad(long lastPingSecondsAfter, long managementServerId); diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index 136351527f6b..a0efe21973cd 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -783,7 +783,7 @@ public void markHostsAsDisconnected(long msId, long lastPing) { } @Override - public List listByHostTag(Host.Type type, Long clusterId, Long podId, long dcId, String hostTag) { + public List listByHostTag(Host.Type type, Long clusterId, Long podId, Long dcId, String hostTag) { SearchBuilder hostSearch = createSearchBuilder(); HostVO entity = hostSearch.entity(); hostSearch.and("type", entity.getType(), SearchCriteria.Op.EQ); @@ -801,7 +801,9 @@ public List listByHostTag(Host.Type type, Long clusterId, Long podId, lo if (clusterId != null) { sc.setParameters("cluster", clusterId); } - sc.setParameters("dc", dcId); + if (dcId != null) { + sc.setParameters("dc", dcId); + } sc.setParameters("status", Status.Up.toString()); sc.setParameters("resourceState", ResourceState.Enabled.toString()); diff --git a/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDao.java b/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDao.java index e2fc5b49ae84..d086ad1dac1f 100644 --- a/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDao.java +++ b/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDao.java @@ -55,4 +55,6 @@ List createSystemServiceOfferings(String name, String uniqueN List listPublicByCpuAndMemory(Integer cpus, Integer memory); ServiceOfferingVO findServiceOfferingByComputeOnlyDiskOffering(long diskOfferingId); + + List listByHostTag(String tag); } diff --git a/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDaoImpl.java b/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDaoImpl.java index 5c8e49938295..e03aaa1c3902 100644 --- a/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDaoImpl.java @@ -293,4 +293,22 @@ public ServiceOfferingVO findServiceOfferingByComputeOnlyDiskOffering(long diskO } return vos.get(0); } + + @Override + public List listByHostTag(String tag) { + SearchBuilder sb = createSearchBuilder(); + sb.and("tagNotNull", sb.entity().getHostTag(), SearchCriteria.Op.NNULL); + sb.and().op("tagEq", sb.entity().getHostTag(), SearchCriteria.Op.EQ); + sb.or("tagStartLike", sb.entity().getHostTag(), SearchCriteria.Op.LIKE); + sb.or("tagMidLike", sb.entity().getHostTag(), SearchCriteria.Op.LIKE); + sb.or("tagEndLike", sb.entity().getHostTag(), SearchCriteria.Op.LIKE); + sb.cp(); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("tagEq", tag); + sc.setParameters("tagStartLike", tag + ",%"); + sc.setParameters("tagMidLike", "%," + tag + ",%"); + sc.setParameters("tagEndLike", "%," + tag); + return listBy(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDao.java index 5a49d0b5192f..f726bca3c5d6 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDao.java @@ -31,5 +31,6 @@ public interface DiskOfferingDao extends GenericDao { List listAllBySizeAndProvisioningType(long size, Storage.ProvisioningType provisioningType); List findCustomDiskOfferings(); + List listByStorageTag(String tag); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDaoImpl.java index 78b2a542837f..d0547ec06d93 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDaoImpl.java @@ -157,4 +157,23 @@ public boolean remove(Long id) { return update(id, diskOffering); } + + @Override + public List listByStorageTag(String tag) { + SearchBuilder sb = createSearchBuilder(); + sb.and("s", sb.entity().getTags(), SearchCriteria.Op.NNULL); + sb.and("tagNotNull", sb.entity().getTags(), SearchCriteria.Op.NNULL); + sb.and().op("tagEq", sb.entity().getTags(), SearchCriteria.Op.EQ); + sb.or("tagStartLike", sb.entity().getTags(), SearchCriteria.Op.LIKE); + sb.or("tagMidLike", sb.entity().getTags(), SearchCriteria.Op.LIKE); + sb.or("tagEndLike", sb.entity().getTags(), SearchCriteria.Op.LIKE); + sb.cp(); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("tagEq", tag); + sc.setParameters("tagStartLike", tag + ",%"); + sc.setParameters("tagMidLike", "%," + tag + ",%"); + sc.setParameters("tagEndLike", "%," + tag); + return listBy(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDao.java index 9352ee21858c..a4b87ef025f3 100755 --- a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDao.java @@ -34,5 +34,6 @@ public interface StoragePoolTagsDao extends GenericDao { StorageTagResponse newStorageTagResponse(StoragePoolTagVO tag); List findStoragePoolTags(long poolId); + List listPoolIdsByTag(String tag); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDaoImpl.java index c01c66763af1..c4d7ed886072 100755 --- a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDaoImpl.java @@ -18,12 +18,10 @@ import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import javax.inject.Inject; -import com.cloud.utils.db.Transaction; -import com.cloud.utils.db.TransactionCallbackNoReturn; -import com.cloud.utils.db.TransactionStatus; import org.apache.cloudstack.api.response.StorageTagResponse; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; @@ -31,7 +29,10 @@ import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionLegacy; +import com.cloud.utils.db.TransactionStatus; public class StoragePoolTagsDaoImpl extends GenericDaoBase implements StoragePoolTagsDao { @@ -178,4 +179,15 @@ public List findStoragePoolTags(long poolId) { return search(sc, null); } + @Override + public List listPoolIdsByTag(String tag) { + SearchBuilder sb = createSearchBuilder(); + sb.and("tag", sb.entity().getTag(), SearchCriteria.Op.EQ); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("tag", tag); + List poolRefs = search(sc, null); + return poolRefs.stream().map(StoragePoolTagVO::getPoolId).collect(Collectors.toList()); + } + } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java index 708a77a8f9ec..1c5a2cb4256a 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java @@ -90,4 +90,7 @@ public interface VMTemplateDao extends GenericDao, StateDao< List findTemplatesLinkedToUserdata(long userdataId); List listByIds(List ids); + + List listByTemplateTag(String tag); + } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index 031bcb3af7b8..31c75ca99ac6 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -688,6 +688,16 @@ public boolean remove(Long id) { return result; } + @Override + public List listByTemplateTag(String tag) { + SearchBuilder sb = createSearchBuilder(); + sb.and("tag", sb.entity().getTemplateTag(), SearchCriteria.Op.EQ); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("tag", tag); + return listIncludingRemovedBy(sc); + } + @Override public boolean updateState( com.cloud.template.VirtualMachineTemplate.State currentState, diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java index be6588e3189f..4e9c63699ca9 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java @@ -155,4 +155,7 @@ public interface VolumeDao extends GenericDao, StateDao listByIds(List ids); + + List listAllocatedVolumesForAccountDiskOfferingIdsAndNotForVms(long accountId, List diskOfferingIds, List vmIds); + } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java index bf5566224635..49ab762bf4e7 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java @@ -825,4 +825,28 @@ public List listByIds(List ids) { sc.setParameters("idIN", ids.toArray()); return listBy(sc, null); } + + @Override + public List listAllocatedVolumesForAccountDiskOfferingIdsAndNotForVms(long accountId, List diskOfferingIds, List vmIds) { + SearchBuilder sb = createSearchBuilder(); + sb.and("account", sb.entity().getAccountId(), SearchCriteria.Op.EQ); + sb.and("state", sb.entity().getState(), SearchCriteria.Op.NIN); + sb.and("diskOfferingIds", sb.entity().getDiskOfferingId(), SearchCriteria.Op.IN); + sb.and("displayVolume", sb.entity().isDisplayVolume(), Op.EQ); + if (CollectionUtils.isNotEmpty(vmIds)) { + sb.and().op("instanceId", sb.entity().getInstanceId(), Op.NULL); + sb.or("notVmIds", sb.entity().getInstanceId(), Op.NIN); + sb.cp(); + } + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("account", accountId); + sc.setParameters("state", Volume.State.Destroy, Volume.State.Expunged); + sc.setParameters("diskOfferingIds", diskOfferingIds.toArray()); + sc.setParameters("displayVolume", 1); + if (CollectionUtils.isNotEmpty(vmIds)) { + sc.setParameters("notVmIds", vmIds.toArray()); + } + return listBy(sc); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/reservation/ReservationVO.java b/engine/schema/src/main/java/org/apache/cloudstack/reservation/ReservationVO.java index e5636f0bfc99..9ebdfb8042be 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/reservation/ReservationVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/reservation/ReservationVO.java @@ -18,10 +18,6 @@ // package org.apache.cloudstack.reservation; -import com.cloud.configuration.Resource; -import org.apache.cloudstack.user.ResourceReservation; -import com.cloud.utils.exception.CloudRuntimeException; - import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.GeneratedValue; @@ -29,6 +25,11 @@ import javax.persistence.Id; import javax.persistence.Table; +import org.apache.cloudstack.user.ResourceReservation; + +import com.cloud.configuration.Resource; +import com.cloud.utils.exception.CloudRuntimeException; + @Entity @Table(name = "resource_reservation") public class ReservationVO implements ResourceReservation { @@ -47,22 +48,30 @@ public class ReservationVO implements ResourceReservation { @Column(name = "resource_type", nullable = false) Resource.ResourceType resourceType; + @Column(name = "tag") + String tag; + @Column(name = "amount") long amount; - protected ReservationVO() - {} + protected ReservationVO() { + } - public ReservationVO(Long accountId, Long domainId, Resource.ResourceType resourceType, Long delta) { + public ReservationVO(Long accountId, Long domainId, Resource.ResourceType resourceType, String tag, Long delta) { if (delta == null || delta <= 0) { throw new CloudRuntimeException("resource reservations can not be made for no resources"); } this.accountId = accountId; this.domainId = domainId; this.resourceType = resourceType; + this.tag = tag; this.amount = delta; } + public ReservationVO(Long accountId, Long domainId, Resource.ResourceType resourceType, Long delta) { + this(accountId, domainId, resourceType, null, delta); + } + @Override public long getId() { return this.id; @@ -83,6 +92,11 @@ public Resource.ResourceType getResourceType() { return resourceType; } + @Override + public String getTag() { + return tag; + } + @Override public Long getReservedAmount() { return amount; diff --git a/engine/schema/src/main/java/org/apache/cloudstack/reservation/dao/ReservationDao.java b/engine/schema/src/main/java/org/apache/cloudstack/reservation/dao/ReservationDao.java index eead91c7b8e9..478a9087c44c 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/reservation/dao/ReservationDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/reservation/dao/ReservationDao.java @@ -18,11 +18,12 @@ // package org.apache.cloudstack.reservation.dao; -import com.cloud.configuration.Resource; import org.apache.cloudstack.reservation.ReservationVO; + +import com.cloud.configuration.Resource; import com.cloud.utils.db.GenericDao; public interface ReservationDao extends GenericDao { - long getAccountReservation(Long account, Resource.ResourceType resourceType); - long getDomainReservation(Long domain, Resource.ResourceType resourceType); + long getAccountReservation(Long account, Resource.ResourceType resourceType, String tag); + long getDomainReservation(Long domain, Resource.ResourceType resourceType, String tag); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/reservation/dao/ReservationDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/reservation/dao/ReservationDaoImpl.java index 6703de0b13c4..5239aa50ad0e 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/reservation/dao/ReservationDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/reservation/dao/ReservationDaoImpl.java @@ -18,41 +18,63 @@ // package org.apache.cloudstack.reservation.dao; +import java.util.List; + +import org.apache.cloudstack.reservation.ReservationVO; + import com.cloud.configuration.Resource; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; -import org.apache.cloudstack.reservation.ReservationVO; - -import java.util.List; public class ReservationDaoImpl extends GenericDaoBase implements ReservationDao { private static final String RESOURCE_TYPE = "resourceType"; + private static final String RESOURCE_TAG = "resourceTag"; private static final String ACCOUNT_ID = "accountId"; private static final String DOMAIN_ID = "domainId"; private final SearchBuilder listAccountAndTypeSearch; + private final SearchBuilder listAccountAndTypeAndNoTagSearch; private final SearchBuilder listDomainAndTypeSearch; + private final SearchBuilder listDomainAndTypeAndNoTagSearch; public ReservationDaoImpl() { listAccountAndTypeSearch = createSearchBuilder(); listAccountAndTypeSearch.and(ACCOUNT_ID, listAccountAndTypeSearch.entity().getAccountId(), SearchCriteria.Op.EQ); listAccountAndTypeSearch.and(RESOURCE_TYPE, listAccountAndTypeSearch.entity().getResourceType(), SearchCriteria.Op.EQ); + listAccountAndTypeSearch.and(RESOURCE_TAG, listAccountAndTypeSearch.entity().getTag(), SearchCriteria.Op.EQ); listAccountAndTypeSearch.done(); + listAccountAndTypeAndNoTagSearch = createSearchBuilder(); + listAccountAndTypeAndNoTagSearch.and(ACCOUNT_ID, listAccountAndTypeAndNoTagSearch.entity().getAccountId(), SearchCriteria.Op.EQ); + listAccountAndTypeAndNoTagSearch.and(RESOURCE_TYPE, listAccountAndTypeAndNoTagSearch.entity().getResourceType(), SearchCriteria.Op.EQ); + listAccountAndTypeAndNoTagSearch.and(RESOURCE_TAG, listAccountAndTypeAndNoTagSearch.entity().getTag(), SearchCriteria.Op.NULL); + listAccountAndTypeAndNoTagSearch.done(); + listDomainAndTypeSearch = createSearchBuilder(); listDomainAndTypeSearch.and(DOMAIN_ID, listDomainAndTypeSearch.entity().getDomainId(), SearchCriteria.Op.EQ); listDomainAndTypeSearch.and(RESOURCE_TYPE, listDomainAndTypeSearch.entity().getResourceType(), SearchCriteria.Op.EQ); + listDomainAndTypeSearch.and(RESOURCE_TAG, listDomainAndTypeSearch.entity().getTag(), SearchCriteria.Op.EQ); listDomainAndTypeSearch.done(); + + listDomainAndTypeAndNoTagSearch = createSearchBuilder(); + listDomainAndTypeAndNoTagSearch.and(ACCOUNT_ID, listDomainAndTypeAndNoTagSearch.entity().getAccountId(), SearchCriteria.Op.EQ); + listDomainAndTypeAndNoTagSearch.and(RESOURCE_TYPE, listDomainAndTypeAndNoTagSearch.entity().getResourceType(), SearchCriteria.Op.EQ); + listDomainAndTypeAndNoTagSearch.and(RESOURCE_TAG, listDomainAndTypeAndNoTagSearch.entity().getTag(), SearchCriteria.Op.NULL); + listDomainAndTypeAndNoTagSearch.done(); } @Override - public long getAccountReservation(Long accountId, Resource.ResourceType resourceType) { + public long getAccountReservation(Long accountId, Resource.ResourceType resourceType, String tag) { long total = 0; - SearchCriteria sc = listAccountAndTypeSearch.create(); + SearchCriteria sc = tag == null ? + listAccountAndTypeAndNoTagSearch.create() : listAccountAndTypeSearch.create(); sc.setParameters(ACCOUNT_ID, accountId); sc.setParameters(RESOURCE_TYPE, resourceType); + if (tag != null) { + sc.setParameters(RESOURCE_TAG, tag); + } List reservations = listBy(sc); for (ReservationVO reservation : reservations) { total += reservation.getReservedAmount(); @@ -61,11 +83,15 @@ public long getAccountReservation(Long accountId, Resource.ResourceType resource } @Override - public long getDomainReservation(Long domainId, Resource.ResourceType resourceType) { + public long getDomainReservation(Long domainId, Resource.ResourceType resourceType, String tag) { long total = 0; - SearchCriteria sc = listDomainAndTypeSearch.create(); + SearchCriteria sc = tag == null ? + listDomainAndTypeAndNoTagSearch.create() : listDomainAndTypeSearch.create(); sc.setParameters(DOMAIN_ID, domainId); sc.setParameters(RESOURCE_TYPE, resourceType); + if (tag != null) { + sc.setParameters(RESOURCE_TAG, tag); + } List reservations = listBy(sc); for (ReservationVO reservation : reservations) { total += reservation.getReservedAmount(); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql index 27170fcac149..cbf41ae39079 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql @@ -314,3 +314,13 @@ CREATE TABLE `cloud_usage`.`bucket_statistics` ( `size` bigint unsigned COMMENT 'total size of bucket objects', PRIMARY KEY(`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- Add tag column to tables +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.resource_limit', 'tag', 'varchar(64) DEFAULT NULL COMMENT "tag for the limit" '); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.resource_count', 'tag', 'varchar(64) DEFAULT NULL COMMENT "tag for the resource count" '); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.resource_reservation', 'tag', 'varchar(64) DEFAULT NULL COMMENT "tag for the resource reservation" '); +ALTER TABLE `resource_count` +DROP INDEX `i_resource_count__type_accountId`, +DROP INDEX `i_resource_count__type_domaintId`, +ADD UNIQUE INDEX `i_resource_count__type_tag_accountId` (`type`,`tag`,`account_id`), +ADD UNIQUE INDEX `i_resource_count__type_tag_domaintId` (`type`,`tag`,`domain_id`); diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql new file mode 100644 index 000000000000..87546a9d1188 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql @@ -0,0 +1,164 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud`.`account_view`; + +DROP VIEW IF EXISTS `cloud`.`account_view`; +CREATE VIEW `cloud`.`account_view` AS +select + `account`.`id` AS `id`, + `account`.`uuid` AS `uuid`, + `account`.`account_name` AS `account_name`, + `account`.`type` AS `type`, + `account`.`role_id` AS `role_id`, + `account`.`state` AS `state`, + `account`.`created` AS `created`, + `account`.`removed` AS `removed`, + `account`.`cleanup_needed` AS `cleanup_needed`, + `account`.`network_domain` AS `network_domain` , + `account`.`default` AS `default`, + `domain`.`id` AS `domain_id`, + `domain`.`uuid` AS `domain_uuid`, + `domain`.`name` AS `domain_name`, + `domain`.`path` AS `domain_path`, + `data_center`.`id` AS `data_center_id`, + `data_center`.`uuid` AS `data_center_uuid`, + `data_center`.`name` AS `data_center_name`, + `account_netstats_view`.`bytesReceived` AS `bytesReceived`, + `account_netstats_view`.`bytesSent` AS `bytesSent`, + `vmlimit`.`max` AS `vmLimit`, + `vmcount`.`count` AS `vmTotal`, + `runningvm`.`vmcount` AS `runningVms`, + `stoppedvm`.`vmcount` AS `stoppedVms`, + `iplimit`.`max` AS `ipLimit`, + `ipcount`.`count` AS `ipTotal`, + `free_ip_view`.`free_ip` AS `ipFree`, + `volumelimit`.`max` AS `volumeLimit`, + `volumecount`.`count` AS `volumeTotal`, + `snapshotlimit`.`max` AS `snapshotLimit`, + `snapshotcount`.`count` AS `snapshotTotal`, + `templatelimit`.`max` AS `templateLimit`, + `templatecount`.`count` AS `templateTotal`, + `vpclimit`.`max` AS `vpcLimit`, + `vpccount`.`count` AS `vpcTotal`, + `projectlimit`.`max` AS `projectLimit`, + `projectcount`.`count` AS `projectTotal`, + `networklimit`.`max` AS `networkLimit`, + `networkcount`.`count` AS `networkTotal`, + `cpulimit`.`max` AS `cpuLimit`, + `cpucount`.`count` AS `cpuTotal`, + `memorylimit`.`max` AS `memoryLimit`, + `memorycount`.`count` AS `memoryTotal`, + `primary_storage_limit`.`max` AS `primaryStorageLimit`, + `primary_storage_count`.`count` AS `primaryStorageTotal`, + `secondary_storage_limit`.`max` AS `secondaryStorageLimit`, + `secondary_storage_count`.`count` AS `secondaryStorageTotal`, + `async_job`.`id` AS `job_id`, + `async_job`.`uuid` AS `job_uuid`, + `async_job`.`job_status` AS `job_status`, + `async_job`.`account_id` AS `job_account_id` +from + `cloud`.`free_ip_view`, + `cloud`.`account` + inner join + `cloud`.`domain` ON account.domain_id = domain.id + left join + `cloud`.`data_center` ON account.default_zone_id = data_center.id + left join + `cloud`.`account_netstats_view` ON account.id = account_netstats_view.account_id + left join + `cloud`.`resource_limit` vmlimit ON account.id = vmlimit.account_id + and vmlimit.type = 'user_vm' and vmlimit.tag IS NULL + left join + `cloud`.`resource_count` vmcount ON account.id = vmcount.account_id + and vmcount.type = 'user_vm' and vmcount.tag IS NULL + left join + `cloud`.`account_vmstats_view` runningvm ON account.id = runningvm.account_id + and runningvm.state = 'Running' + left join + `cloud`.`account_vmstats_view` stoppedvm ON account.id = stoppedvm.account_id + and stoppedvm.state = 'Stopped' + left join + `cloud`.`resource_limit` iplimit ON account.id = iplimit.account_id + and iplimit.type = 'public_ip' + left join + `cloud`.`resource_count` ipcount ON account.id = ipcount.account_id + and ipcount.type = 'public_ip' + left join + `cloud`.`resource_limit` volumelimit ON account.id = volumelimit.account_id + and volumelimit.type = 'volume' and volumelimit.tag IS NULL + left join + `cloud`.`resource_count` volumecount ON account.id = volumecount.account_id + and volumecount.type = 'volume' and volumecount.tag IS NULL + left join + `cloud`.`resource_limit` snapshotlimit ON account.id = snapshotlimit.account_id + and snapshotlimit.type = 'snapshot' + left join + `cloud`.`resource_count` snapshotcount ON account.id = snapshotcount.account_id + and snapshotcount.type = 'snapshot' + left join + `cloud`.`resource_limit` templatelimit ON account.id = templatelimit.account_id + and templatelimit.type = 'template' + left join + `cloud`.`resource_count` templatecount ON account.id = templatecount.account_id + and templatecount.type = 'template' + left join + `cloud`.`resource_limit` vpclimit ON account.id = vpclimit.account_id + and vpclimit.type = 'vpc' + left join + `cloud`.`resource_count` vpccount ON account.id = vpccount.account_id + and vpccount.type = 'vpc' + left join + `cloud`.`resource_limit` projectlimit ON account.id = projectlimit.account_id + and projectlimit.type = 'project' + left join + `cloud`.`resource_count` projectcount ON account.id = projectcount.account_id + and projectcount.type = 'project' + left join + `cloud`.`resource_limit` networklimit ON account.id = networklimit.account_id + and networklimit.type = 'network' + left join + `cloud`.`resource_count` networkcount ON account.id = networkcount.account_id + and networkcount.type = 'network' + left join + `cloud`.`resource_limit` cpulimit ON account.id = cpulimit.account_id + and cpulimit.type = 'cpu' and cpulimit.tag IS NULL + left join + `cloud`.`resource_count` cpucount ON account.id = cpucount.account_id + and cpucount.type = 'cpu' and cpucount.tag IS NULL + left join + `cloud`.`resource_limit` memorylimit ON account.id = memorylimit.account_id + and memorylimit.type = 'memory' and memorylimit.tag IS NULL + left join + `cloud`.`resource_count` memorycount ON account.id = memorycount.account_id + and memorycount.type = 'memory' and memorycount.tag IS NULL + left join + `cloud`.`resource_limit` primary_storage_limit ON account.id = primary_storage_limit.account_id + and primary_storage_limit.type = 'primary_storage' and primary_storage_limit.tag IS NULL + left join + `cloud`.`resource_count` primary_storage_count ON account.id = primary_storage_count.account_id + and primary_storage_count.type = 'primary_storage' and primary_storage_count.tag IS NULL + left join + `cloud`.`resource_limit` secondary_storage_limit ON account.id = secondary_storage_limit.account_id + and secondary_storage_limit.type = 'secondary_storage' + left join + `cloud`.`resource_count` secondary_storage_count ON account.id = secondary_storage_count.account_id + and secondary_storage_count.type = 'secondary_storage' + left join + `cloud`.`async_job` ON async_job.instance_id = account.id + and async_job.instance_type = 'Account' + and async_job.job_status = 0; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.domain_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.domain_view.sql new file mode 100644 index 000000000000..af885b9413ff --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.domain_view.sql @@ -0,0 +1,134 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud`.`domain_view`; + +DROP VIEW IF EXISTS `cloud`.`domain_view`; +CREATE VIEW `cloud`.`domain_view` AS +select + `domain`.`id` AS `id`, + `domain`.`parent` AS `parent`, + `domain`.`name` AS `name`, + `domain`.`uuid` AS `uuid`, + `domain`.`owner` AS `owner`, + `domain`.`path` AS `path`, + `domain`.`level` AS `level`, + `domain`.`child_count` AS `child_count`, + `domain`.`next_child_seq` AS `next_child_seq`, + `domain`.`created` AS `created`, + `domain`.`removed` AS `removed`, + `domain`.`state` AS `state`, + `domain`.`network_domain` AS `network_domain`, + `domain`.`type` AS `type`, + `vmlimit`.`max` AS `vmLimit`, + `vmcount`.`count` AS `vmTotal`, + `iplimit`.`max` AS `ipLimit`, + `ipcount`.`count` AS `ipTotal`, + `volumelimit`.`max` AS `volumeLimit`, + `volumecount`.`count` AS `volumeTotal`, + `snapshotlimit`.`max` AS `snapshotLimit`, + `snapshotcount`.`count` AS `snapshotTotal`, + `templatelimit`.`max` AS `templateLimit`, + `templatecount`.`count` AS `templateTotal`, + `vpclimit`.`max` AS `vpcLimit`, + `vpccount`.`count` AS `vpcTotal`, + `projectlimit`.`max` AS `projectLimit`, + `projectcount`.`count` AS `projectTotal`, + `networklimit`.`max` AS `networkLimit`, + `networkcount`.`count` AS `networkTotal`, + `cpulimit`.`max` AS `cpuLimit`, + `cpucount`.`count` AS `cpuTotal`, + `memorylimit`.`max` AS `memoryLimit`, + `memorycount`.`count` AS `memoryTotal`, + `primary_storage_limit`.`max` AS `primaryStorageLimit`, + `primary_storage_count`.`count` AS `primaryStorageTotal`, + `secondary_storage_limit`.`max` AS `secondaryStorageLimit`, + `secondary_storage_count`.`count` AS `secondaryStorageTotal` +from + `cloud`.`domain` + left join + `cloud`.`resource_limit` vmlimit ON domain.id = vmlimit.domain_id + and vmlimit.type = 'user_vm' and vmlimit.tag IS NULL + left join + `cloud`.`resource_count` vmcount ON domain.id = vmcount.domain_id + and vmcount.type = 'user_vm' and vmcount.tag IS NULL + left join + `cloud`.`resource_limit` iplimit ON domain.id = iplimit.domain_id + and iplimit.type = 'public_ip' + left join + `cloud`.`resource_count` ipcount ON domain.id = ipcount.domain_id + and ipcount.type = 'public_ip' + left join + `cloud`.`resource_limit` volumelimit ON domain.id = volumelimit.domain_id + and volumelimit.type = 'volume' and volumelimit.tag IS NULL + left join + `cloud`.`resource_count` volumecount ON domain.id = volumecount.domain_id + and volumecount.type = 'volume' and volumecount.tag IS NULL + left join + `cloud`.`resource_limit` snapshotlimit ON domain.id = snapshotlimit.domain_id + and snapshotlimit.type = 'snapshot' + left join + `cloud`.`resource_count` snapshotcount ON domain.id = snapshotcount.domain_id + and snapshotcount.type = 'snapshot' + left join + `cloud`.`resource_limit` templatelimit ON domain.id = templatelimit.domain_id + and templatelimit.type = 'template' + left join + `cloud`.`resource_count` templatecount ON domain.id = templatecount.domain_id + and templatecount.type = 'template' + left join + `cloud`.`resource_limit` vpclimit ON domain.id = vpclimit.domain_id + and vpclimit.type = 'vpc' + left join + `cloud`.`resource_count` vpccount ON domain.id = vpccount.domain_id + and vpccount.type = 'vpc' + left join + `cloud`.`resource_limit` projectlimit ON domain.id = projectlimit.domain_id + and projectlimit.type = 'project' + left join + `cloud`.`resource_count` projectcount ON domain.id = projectcount.domain_id + and projectcount.type = 'project' + left join + `cloud`.`resource_limit` networklimit ON domain.id = networklimit.domain_id + and networklimit.type = 'network' + left join + `cloud`.`resource_count` networkcount ON domain.id = networkcount.domain_id + and networkcount.type = 'network' + left join + `cloud`.`resource_limit` cpulimit ON domain.id = cpulimit.domain_id + and cpulimit.type = 'cpu' and cpulimit.tag IS NULL + left join + `cloud`.`resource_count` cpucount ON domain.id = cpucount.domain_id + and cpucount.type = 'cpu' and cpucount.tag IS NULL + left join + `cloud`.`resource_limit` memorylimit ON domain.id = memorylimit.domain_id + and memorylimit.type = 'memory' and memorylimit.tag IS NULL + left join + `cloud`.`resource_count` memorycount ON domain.id = memorycount.domain_id + and memorycount.type = 'memory' and memorycount.tag IS NULL + left join + `cloud`.`resource_limit` primary_storage_limit ON domain.id = primary_storage_limit.domain_id + and primary_storage_limit.type = 'primary_storage' and primary_storage_limit.tag IS NULL + left join + `cloud`.`resource_count` primary_storage_count ON domain.id = primary_storage_count.domain_id + and primary_storage_count.type = 'primary_storage' and primary_storage_count.tag IS NULL + left join + `cloud`.`resource_limit` secondary_storage_limit ON domain.id = secondary_storage_limit.domain_id + and secondary_storage_limit.type = 'secondary_storage' + left join + `cloud`.`resource_count` secondary_storage_count ON domain.id = secondary_storage_count.domain_id + and secondary_storage_count.type = 'secondary_storage'; diff --git a/engine/schema/src/test/java/com/cloud/host/HostVOTest.java b/engine/schema/src/test/java/com/cloud/host/HostVOTest.java index 76bc5270b4f7..cd9ac3cc1723 100755 --- a/engine/schema/src/test/java/com/cloud/host/HostVOTest.java +++ b/engine/schema/src/test/java/com/cloud/host/HostVOTest.java @@ -1,84 +1,126 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.host; - -import com.cloud.service.ServiceOfferingVO; -import com.cloud.vm.VirtualMachine; -import java.util.Arrays; -import java.util.List; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import org.junit.Test; -import org.junit.Before; - -public class HostVOTest { - HostVO host; - ServiceOfferingVO offering; - - @Before - public void setUp() throws Exception { - host = new HostVO(); - offering = new ServiceOfferingVO("TestSO", 0, 0, 0, 0, 0, - false, "TestSO", false,VirtualMachine.Type.User,false); - } - - @Test - public void testNoSO() { - assertFalse(host.checkHostServiceOfferingTags(null)); - } - - @Test - public void testNoTag() { - assertTrue(host.checkHostServiceOfferingTags(offering)); - } - - @Test - public void testRightTag() { - host.setHostTags(Arrays.asList("tag1","tag2"), false); - offering.setHostTag("tag2,tag1"); - assertTrue(host.checkHostServiceOfferingTags(offering)); - } - - @Test - public void testWrongTag() { - host.setHostTags(Arrays.asList("tag1","tag2"), false); - offering.setHostTag("tag2,tag4"); - assertFalse(host.checkHostServiceOfferingTags(offering)); - } - - @Test - public void checkHostServiceOfferingTagsTestRuleTagWithServiceTagThatMatches() { - host.setHostTags(List.of("tags[0] == 'A'"), true); - offering.setHostTag("A"); - assertTrue(host.checkHostServiceOfferingTags(offering)); - } - - @Test - public void checkHostServiceOfferingTagsTestRuleTagWithServiceTagThatDoesNotMatch() { - host.setHostTags(List.of("tags[0] == 'A'"), true); - offering.setHostTag("B"); - assertFalse(host.checkHostServiceOfferingTags(offering)); - } - - @Test - public void checkHostServiceOfferingTagsTestRuleTagWithNullServiceTag() { - host.setHostTags(List.of("tags[0] == 'A'"), true); - offering.setHostTag(null); - assertFalse(host.checkHostServiceOfferingTags(offering)); - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.host; + +import com.cloud.offering.ServiceOffering; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.vm.VirtualMachine; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import org.junit.Test; +import org.junit.Before; +import org.mockito.Mockito; + +public class HostVOTest { + HostVO host; + ServiceOfferingVO offering; + + @Before + public void setUp() throws Exception { + host = new HostVO(); + offering = new ServiceOfferingVO("TestSO", 0, 0, 0, 0, 0, + false, "TestSO", false,VirtualMachine.Type.User,false); + } + + @Test + public void testNoSO() { + assertFalse(host.checkHostServiceOfferingTags(null)); + } + + @Test + public void testNoTag() { + assertTrue(host.checkHostServiceOfferingTags(offering)); + } + + @Test + public void testRightTag() { + host.setHostTags(Arrays.asList("tag1","tag2"), false); + offering.setHostTag("tag2,tag1"); + assertTrue(host.checkHostServiceOfferingTags(offering)); + } + + @Test + public void testWrongTag() { + host.setHostTags(Arrays.asList("tag1","tag2"), false); + offering.setHostTag("tag2,tag4"); + assertFalse(host.checkHostServiceOfferingTags(offering)); + } + + @Test + public void checkHostServiceOfferingTagsTestRuleTagWithServiceTagThatMatches() { + host.setHostTags(List.of("tags[0] == 'A'"), true); + offering.setHostTag("A"); + assertTrue(host.checkHostServiceOfferingTags(offering)); + } + + @Test + public void checkHostServiceOfferingTagsTestRuleTagWithServiceTagThatDoesNotMatch() { + host.setHostTags(List.of("tags[0] == 'A'"), true); + offering.setHostTag("B"); + assertFalse(host.checkHostServiceOfferingTags(offering)); + } + + @Test + public void checkHostServiceOfferingTagsTestRuleTagWithNullServiceTag() { + host.setHostTags(List.of("tags[0] == 'A'"), true); + offering.setHostTag(null); + assertFalse(host.checkHostServiceOfferingTags(offering)); + } + + @Test + public void testEitherNoSOOrTemplate() { + assertFalse(host.checkHostServiceOfferingAndTemplateTags(null, Mockito.mock(VirtualMachineTemplate.class))); + assertFalse(host.checkHostServiceOfferingAndTemplateTags(Mockito.mock(ServiceOffering.class), null)); + } + + @Test + public void testNoTagOfferingTemplate() { + assertTrue(host.checkHostServiceOfferingAndTemplateTags(offering, Mockito.mock(VirtualMachineTemplate.class))); + } + + @Test + public void testRightTagOfferingTemplate() { + host.setHostTags(Arrays.asList("tag1", "tag2"), false); + offering.setHostTag("tag2,tag1"); + assertTrue(host.checkHostServiceOfferingAndTemplateTags(offering, Mockito.mock(VirtualMachineTemplate.class))); + host.setHostTags(Arrays.asList("tag1", "tag2", "tag3"), false); + offering.setHostTag("tag2,tag1"); + VirtualMachineTemplate template = Mockito.mock(VirtualMachineTemplate.class); + Mockito.when(template.getTemplateTag()).thenReturn("tag3"); + assertTrue(host.checkHostServiceOfferingAndTemplateTags(offering, template)); + host.setHostTags(List.of("tag3"), false); + offering.setHostTag(null); + assertTrue(host.checkHostServiceOfferingAndTemplateTags(offering, template)); + } + + @Test + public void testWrongOfferingTag() { + host.setHostTags(Arrays.asList("tag1","tag2"), false); + offering.setHostTag("tag2,tag4"); + VirtualMachineTemplate template = Mockito.mock(VirtualMachineTemplate.class); + Mockito.when(template.getTemplateTag()).thenReturn("tag1"); + assertFalse(host.checkHostServiceOfferingAndTemplateTags(offering, template)); + offering.setHostTag("tag1,tag2"); + template = Mockito.mock(VirtualMachineTemplate.class); + Mockito.when(template.getTemplateTag()).thenReturn("tag3"); + assertFalse(host.checkHostServiceOfferingAndTemplateTags(offering, template)); + } +} diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java index 89a7b577ae7f..74b7df85d395 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java @@ -306,10 +306,14 @@ protected boolean filter(ExcludeList avoid, StoragePool pool, DiskProfile dskCh, return false; } - Volume volume = volumeDao.findById(dskCh.getVolumeId()); - if(!storageMgr.storagePoolCompatibleWithVolumePool(pool, volume)) { - s_logger.debug(String.format("Pool [%s] is not compatible with volume [%s], skipping it.", pool, volume)); - return false; + Volume volume = null; + boolean isTempVolume = dskCh.getVolumeId() == Volume.DISK_OFFERING_SUITABILITY_CHECK_VOLUME_ID; + if (!isTempVolume) { + volume = volumeDao.findById(dskCh.getVolumeId()); + if (!storageMgr.storagePoolCompatibleWithVolumePool(pool, volume)) { + s_logger.debug(String.format("Pool [%s] is not compatible with volume [%s], skipping it.", pool, volume)); + return false; + } } if (pool.isManaged() && !storageUtil.managedStoragePoolCanScale(pool, plan.getClusterId(), plan.getHostId())) { @@ -334,8 +338,10 @@ protected boolean filter(ExcludeList avoid, StoragePool pool, DiskProfile dskCh, } try { - boolean isStoragePoolStoragepolicyComplaince = storageMgr.isStoragePoolCompliantWithStoragePolicy(requestVolumeDiskProfilePairs, pool); - if (!isStoragePoolStoragepolicyComplaince) { + boolean isStoragePoolStoragePolicyCompliance = isTempVolume ? + storageMgr.isStoragePoolCompliantWithStoragePolicy(dskCh.getDiskOfferingId(), pool) : + storageMgr.isStoragePoolCompliantWithStoragePolicy(requestVolumeDiskProfilePairs, pool); + if (!isStoragePoolStoragePolicyCompliance) { s_logger.debug(String.format("Skipping allocation of pool [%s] to volume [%s] because this pool is not compliant with the storage policy required by the volume.", pool, volume)); return false; } @@ -344,7 +350,11 @@ protected boolean filter(ExcludeList avoid, StoragePool pool, DiskProfile dskCh, return false; } } - return storageMgr.storagePoolHasEnoughIops(requestVolumeDiskProfilePairs, pool) && storageMgr.storagePoolHasEnoughSpace(requestVolumeDiskProfilePairs, pool, plan.getClusterId()); + return isTempVolume ? + (storageMgr.storagePoolHasEnoughIops(dskCh.getMinIops(), pool) && + storageMgr.storagePoolHasEnoughSpace(dskCh.getSize(), pool)): + (storageMgr.storagePoolHasEnoughIops(requestVolumeDiskProfilePairs, pool) && + storageMgr.storagePoolHasEnoughSpace(requestVolumeDiskProfilePairs, pool, plan.getClusterId())); } private boolean checkDiskProvisioningSupport(DiskProfile dskCh, StoragePool pool) { diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index c0ef227251cc..ccb7f0cfd738 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -34,6 +34,7 @@ import org.apache.cloudstack.secret.dao.PassphraseDao; import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.resource.StorageProcessor; import org.apache.cloudstack.annotation.AnnotationService; @@ -205,6 +206,8 @@ public class VolumeServiceImpl implements VolumeService { private SnapshotApiService snapshotApiService; @Inject private PassphraseDao passphraseDao; + @Inject + private DiskOfferingDao diskOfferingDao; private final static String SNAPSHOT_ID = "SNAPSHOT_ID"; @@ -1611,8 +1614,7 @@ public void destroyVolume(long volumeId) { if (vol.getAttachedVM() == null || vol.getAttachedVM().getType() == VirtualMachine.Type.User) { // Decrement the resource count for volumes and primary storage belonging user VM's only - _resourceLimitMgr.decrementResourceCount(vol.getAccountId(), ResourceType.volume, vol.isDisplay()); - _resourceLimitMgr.decrementResourceCount(vol.getAccountId(), ResourceType.primary_storage, vol.isDisplay(), new Long(vol.getSize())); + _resourceLimitMgr.decrementVolumeResourceCount(vol.getAccountId(), vol.isDisplay(), vol.getSize(), diskOfferingDao.findById(vol.getDiskOfferingId())); } } diff --git a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java b/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java index 70920df5eb5c..ae73a9b17d99 100644 --- a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java +++ b/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java @@ -22,9 +22,10 @@ import javax.inject.Inject; -import com.cloud.utils.exception.CloudRuntimeException; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.ListUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -40,8 +41,10 @@ import com.cloud.host.dao.HostDao; import com.cloud.offering.ServiceOffering; import com.cloud.resource.ResourceManager; +import com.cloud.storage.VMTemplateVO; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; @@ -59,6 +62,27 @@ public class RandomAllocator extends AdapterBase implements HostAllocator { @Inject private CapacityManager capacityManager; + protected List listHostsByTags(Host.Type type, long dcId, Long podId, Long clusterId, String offeringHostTag, String templateTag) { + List taggedHosts = new ArrayList<>(); + if (offeringHostTag != null) { + taggedHosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, offeringHostTag); + } + if (templateTag != null) { + List templateTaggedHosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, templateTag); + if (taggedHosts.isEmpty()) { + taggedHosts = templateTaggedHosts; + } else { + taggedHosts.retainAll(templateTaggedHosts); + } + } + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("Found %d hosts %s with type: %s, zone ID: %d, pod ID: %d, cluster ID: %s, offering host tag(s): %s, template tag: %s", + taggedHosts.size(), + (taggedHosts.isEmpty() ? "" : String.format("(%s)", StringUtils.join(taggedHosts.stream().map(HostVO::getId).toArray(), ","))), + type.name(), dcId, podId, clusterId, offeringHostTag, templateTag)); + } + return taggedHosts; + } private List findSuitableHosts(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, List hosts, int returnUpTo, boolean considerReservedCapacity) { @@ -72,30 +96,34 @@ private List findSuitableHosts(VirtualMachineProfile vmProfile, Deployment if (type == Host.Type.Storage) { return suitableHosts; } - String hostTag = offering.getHostTag(); - if (hostTag != null) { - s_logger.debug(String.format("Looking for hosts in dc [%s], pod [%s], cluster [%s] and complying with host tag [%s].", dcId, podId, clusterId, hostTag)); + String offeringHostTag = offering.getHostTag(); + VMTemplateVO template = (VMTemplateVO)vmProfile.getTemplate(); + String templateTag = template.getTemplateTag(); + String hostTag = null; + if (ObjectUtils.anyNull(offeringHostTag, templateTag)) { + hostTag = offeringHostTag; + hostTag = hostTag == null ? templateTag : String.format("%s, %s", hostTag, templateTag); + s_logger.debug(String.format("Looking for hosts in dc [%s], pod [%s], cluster [%s] and complying with host tag(s): [%s]", dcId, podId, clusterId, hostTag)); } else { s_logger.debug("Looking for hosts in dc: " + dcId + " pod:" + podId + " cluster:" + clusterId); } if (hosts != null) { // retain all computing hosts, regardless of whether they support routing...it's random after all hostsCopy = new ArrayList(hosts); - if (hostTag != null) { - hostsCopy.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTag)); + if (ObjectUtils.anyNotNull(offeringHostTag, templateTag)) { + hostsCopy.retainAll(listHostsByTags(type, dcId, podId, clusterId, offeringHostTag, templateTag)); } else { hostsCopy.retainAll(_hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId)); } } else { // list all computing hosts, regardless of whether they support routing...it's random after all - hostsCopy = new ArrayList(); - if (hostTag != null) { - hostsCopy = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTag); + if (offeringHostTag != null) { + hostsCopy = listHostsByTags(type, dcId, podId, clusterId, offeringHostTag, templateTag); } else { hostsCopy = _hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId); } } - hostsCopy = ListUtils.union(hostsCopy, _hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(hostTag)); + hostsCopy = ListUtils.union(hostsCopy, _hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(offeringHostTag)); if (hostsCopy.isEmpty()) { s_logger.error(String.format("No suitable host found for vm [%s] with tags [%s].", vmProfile, hostTag)); diff --git a/plugins/hypervisors/simulator/src/main/java/com/cloud/agent/manager/MockStorageManager.java b/plugins/hypervisors/simulator/src/main/java/com/cloud/agent/manager/MockStorageManager.java index 70066dcf7ddc..a05d7d7ff8d6 100644 --- a/plugins/hypervisors/simulator/src/main/java/com/cloud/agent/manager/MockStorageManager.java +++ b/plugins/hypervisors/simulator/src/main/java/com/cloud/agent/manager/MockStorageManager.java @@ -50,6 +50,7 @@ import com.cloud.agent.api.storage.ListVolumeCommand; import com.cloud.agent.api.storage.PrimaryStorageDownloadAnswer; import com.cloud.agent.api.storage.PrimaryStorageDownloadCommand; +import com.cloud.agent.api.storage.ResizeVolumeCommand; import com.cloud.utils.component.Manager; public interface MockStorageManager extends Manager { @@ -113,4 +114,6 @@ public interface MockStorageManager extends Manager { public UploadStatusAnswer getUploadStatus(UploadStatusCommand cmd); Answer handleConfigDriveIso(HandleConfigDriveIsoCommand cmd); + + Answer handleResizeVolume(ResizeVolumeCommand cmd); } diff --git a/plugins/hypervisors/simulator/src/main/java/com/cloud/agent/manager/MockStorageManagerImpl.java b/plugins/hypervisors/simulator/src/main/java/com/cloud/agent/manager/MockStorageManagerImpl.java index 27b4a716af1b..5f9dc273e615 100644 --- a/plugins/hypervisors/simulator/src/main/java/com/cloud/agent/manager/MockStorageManagerImpl.java +++ b/plugins/hypervisors/simulator/src/main/java/com/cloud/agent/manager/MockStorageManagerImpl.java @@ -79,6 +79,8 @@ import com.cloud.agent.api.storage.ListVolumeCommand; import com.cloud.agent.api.storage.PrimaryStorageDownloadAnswer; import com.cloud.agent.api.storage.PrimaryStorageDownloadCommand; +import com.cloud.agent.api.storage.ResizeVolumeAnswer; +import com.cloud.agent.api.storage.ResizeVolumeCommand; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.StorageFilerTO; @@ -1309,4 +1311,32 @@ public UploadStatusAnswer getUploadStatus(UploadStatusCommand cmd) { return new Answer(cmd); } + + @Override + public Answer handleResizeVolume(ResizeVolumeCommand cmd) { + Long currentSize = cmd.getCurrentSize(); + Long newSize = cmd.getNewSize(); + MockStoragePoolVO storagePool = null; + TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.SIMULATOR_DB); + try { + txn.start(); + storagePool = _mockStoragePoolDao.findByUuid(cmd.getPoolUuid()); + txn.commit(); + if (storagePool == null) { + return new ResizeVolumeAnswer(cmd, false, "Failed to find storage pool: " + cmd.getPoolUuid()); + } + } catch (Exception ex) { + txn.rollback(); + throw new CloudRuntimeException("Error when finding storage " + cmd.getPoolUuid(), ex); + } finally { + txn.close(); + txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB); + txn.close(); + } + + if (newSize >= currentSize) { + return new ResizeVolumeAnswer(cmd, true, "", newSize); + } + return new ResizeVolumeAnswer(cmd, false, "Failed to resize"); + } } diff --git a/plugins/hypervisors/simulator/src/main/java/com/cloud/agent/manager/SimulatorManagerImpl.java b/plugins/hypervisors/simulator/src/main/java/com/cloud/agent/manager/SimulatorManagerImpl.java index 159f22236c27..ff24298eb921 100644 --- a/plugins/hypervisors/simulator/src/main/java/com/cloud/agent/manager/SimulatorManagerImpl.java +++ b/plugins/hypervisors/simulator/src/main/java/com/cloud/agent/manager/SimulatorManagerImpl.java @@ -115,6 +115,7 @@ import com.cloud.agent.api.storage.ListTemplateCommand; import com.cloud.agent.api.storage.ListVolumeCommand; import com.cloud.agent.api.storage.PrimaryStorageDownloadCommand; +import com.cloud.agent.api.storage.ResizeVolumeCommand; import com.cloud.api.commands.CleanupSimulatorMockCmd; import com.cloud.api.commands.ConfigureSimulatorCmd; import com.cloud.api.commands.ConfigureSimulatorHAProviderState; @@ -441,7 +442,9 @@ public Answer simulate(final Command cmd, final String hostGuid) { } else if (cmd instanceof FenceCommand) { answer = _mockVmMgr.fence((FenceCommand)cmd); } else if (cmd instanceof HandleConfigDriveIsoCommand) { - answer = _mockStorageMgr.handleConfigDriveIso((HandleConfigDriveIsoCommand)cmd); + answer = _mockStorageMgr.handleConfigDriveIso((HandleConfigDriveIsoCommand)cmd);; + } else if (cmd instanceof ResizeVolumeCommand) { + answer = _mockStorageMgr.handleResizeVolume((ResizeVolumeCommand)cmd); } else if (cmd instanceof GetRouterAlertsCommand || cmd instanceof VpnUsersCfgCommand || cmd instanceof RemoteAccessVpnCfgCommand diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java index 17fbd48181ad..df893f27ce48 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java @@ -25,15 +25,9 @@ import javax.inject.Inject; -import com.cloud.configuration.dao.ResourceCountDao; -import com.cloud.dc.DedicatedResourceVO; -import com.cloud.dc.dao.DedicatedResourceDao; -import com.cloud.host.HostStats; -import com.cloud.host.HostTagVO; -import com.cloud.user.Account; -import com.cloud.user.dao.AccountDao; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import com.cloud.alert.AlertManager; @@ -44,16 +38,21 @@ import com.cloud.api.query.vo.StoragePoolJoinVO; import com.cloud.capacity.Capacity; import com.cloud.capacity.CapacityManager; -import com.cloud.capacity.CapacityVO; import com.cloud.capacity.CapacityState; +import com.cloud.capacity.CapacityVO; import com.cloud.capacity.dao.CapacityDao; import com.cloud.capacity.dao.CapacityDaoImpl; import com.cloud.configuration.Resource; +import com.cloud.configuration.dao.ResourceCountDao; import com.cloud.dc.DataCenterVO; +import com.cloud.dc.DedicatedResourceVO; import com.cloud.dc.Vlan; import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.DataCenterIpAddressDao; +import com.cloud.dc.dao.DedicatedResourceDao; import com.cloud.host.Host; +import com.cloud.host.HostStats; +import com.cloud.host.HostTagVO; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; @@ -64,7 +63,8 @@ import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; -import org.apache.commons.lang3.StringUtils; +import com.cloud.user.Account; +import com.cloud.user.dao.AccountDao; import com.cloud.utils.Ternary; import com.cloud.utils.component.Manager; import com.cloud.utils.component.ManagerBase; @@ -440,13 +440,13 @@ private void addDomainResourceCount(final List metricsList) { } long memoryUsed = _resourceCountDao.getResourceCount(domain.getId(), Resource.ResourceOwnerType.Domain, - Resource.ResourceType.memory); + Resource.ResourceType.memory, null); long cpuUsed = _resourceCountDao.getResourceCount(domain.getId(), Resource.ResourceOwnerType.Domain, - Resource.ResourceType.cpu); + Resource.ResourceType.cpu, null); long primaryStorageUsed = _resourceCountDao.getResourceCount(domain.getId(), Resource.ResourceOwnerType.Domain, - Resource.ResourceType.primary_storage); + Resource.ResourceType.primary_storage, null); long secondaryStorageUsed = _resourceCountDao.getResourceCount(domain.getId(), Resource.ResourceOwnerType.Domain, - Resource.ResourceType.secondary_storage); + Resource.ResourceType.secondary_storage, null); metricsList.add(new ItemPerDomainResourceCount(memoryUsed, domain.getPath(), Resource.ResourceType.memory.getName())); metricsList.add(new ItemPerDomainResourceCount(cpuUsed, domain.getPath(), Resource.ResourceType.cpu.getName())); diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index 3cade046c747..b73046aa73e7 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -23,6 +23,7 @@ import java.util.ListIterator; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import javax.annotation.PostConstruct; import javax.inject.Inject; @@ -53,6 +54,7 @@ import org.apache.cloudstack.api.response.ImageStoreResponse; import org.apache.cloudstack.api.response.InstanceGroupResponse; import org.apache.cloudstack.api.response.NetworkOfferingResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; import org.apache.cloudstack.api.response.ProjectAccountResponse; import org.apache.cloudstack.api.response.ProjectInvitationResponse; import org.apache.cloudstack.api.response.ProjectResponse; @@ -85,6 +87,8 @@ import org.apache.cloudstack.resourcedetail.SnapshotPolicyDetailVO; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; import org.apache.cloudstack.resourcedetail.dao.SnapshotPolicyDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; @@ -350,10 +354,6 @@ import com.cloud.vm.snapshot.VMSnapshot; import com.cloud.vm.snapshot.dao.VMSnapshotDao; -import org.apache.cloudstack.api.response.ObjectStoreResponse; -import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; -import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; - public class ApiDBUtils { private static ManagementServer s_ms; static AsyncJobManager s_asyncMgr; @@ -937,7 +937,7 @@ public static long findCorrectResourceLimitForDomain(ResourceType type, long dom return -1; } - return s_resourceLimitMgr.findCorrectResourceLimitForDomain(domain, type); + return s_resourceLimitMgr.findCorrectResourceLimitForDomain(domain, type, null); } public static long findCorrectResourceLimitForDomain(Long limit, boolean isRootDomain, ResourceType type, long domainId) { @@ -954,16 +954,6 @@ public static long findCorrectResourceLimitForDomain(Long limit, boolean isRootD } } - public static long findCorrectResourceLimit(ResourceType type, long accountId) { - AccountVO account = s_accountDao.findById(accountId); - - if (account == null) { - return -1; - } - - return s_resourceLimitMgr.findCorrectResourceLimitForAccount(account, type); - } - public static long findCorrectResourceLimit(Long limit, long accountId, ResourceType type) { return s_resourceLimitMgr.findCorrectResourceLimitForAccount(accountId, limit, type); } @@ -984,7 +974,7 @@ public static long getResourceCount(ResourceType type, long accountId) { return -1; } - return s_resourceLimitMgr.getResourceCount(account, type); + return s_resourceLimitMgr.getResourceCount(account, type, null); } public static String getSecurityGroupsNamesForVm(long vmId) { @@ -2098,6 +2088,22 @@ public static AsyncJobJoinVO newAsyncJobView(AsyncJob e) { return s_jobJoinDao.newAsyncJobView(e); } + public static List newDiskOfferingResponses(Long vmId, List offerings) { + List list = new ArrayList<>(); + Map suitability = null; + if (vmId != null) { + suitability = s_userVmMgr.getDiskOfferingSuitabilityForVm(vmId, offerings.stream().map(DiskOfferingJoinVO::getId).collect(Collectors.toList())); + } + for (DiskOfferingJoinVO offering : offerings) { + DiskOfferingResponse response = s_diskOfferingJoinDao.newDiskOfferingResponse(offering); + if (vmId != null) { + response.setSuitableForVm(suitability.get(offering.getId())); + } + list.add(response); + } + return list; + } + public static DiskOfferingResponse newDiskOfferingResponse(DiskOfferingJoinVO offering) { return s_diskOfferingJoinDao.newDiskOfferingResponse(offering); } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 5424a36cdd85..2cf93b75c7ba 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -558,6 +558,7 @@ public ResourceLimitResponse createResourceLimitResponse(ResourceLimit limit) { } else { resourceLimitResponse.setMax(limit.getMax()); } + resourceLimitResponse.setTag(limit.getTag()); resourceLimitResponse.setObjectName("resourcelimit"); return resourceLimitResponse; @@ -579,7 +580,10 @@ public ResourceCountResponse createResourceCountResponse(ResourceCount resourceC resourceCountResponse.setResourceType(resourceCount.getType()); resourceCountResponse.setResourceCount(resourceCount.getCount()); - resourceCountResponse.setObjectName("resourcecount"); + resourceCountResponse.setObjectName(ApiConstants.RESOURCE_COUNT); + if (StringUtils.isNotEmpty(resourceCount.getTag())) { + resourceCountResponse.setTag(resourceCount.getTag()); + } return resourceCountResponse; } diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index c3807fdb8d7e..ea39babcd9b9 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -282,6 +282,7 @@ import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.TemplateType; +import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.StoragePoolTagVO; import com.cloud.storage.VMTemplateVO; @@ -523,6 +524,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Inject private ResourceIconDao resourceIconDao; + @Inject + StorageManager storageManager; @Inject private ManagementServerHostDao msHostDao; @@ -3155,8 +3158,8 @@ private Pair, Integer> searchForCacheStoresInternal(ListS @Override public ListResponse searchForDiskOfferings(ListDiskOfferingsCmd cmd) { Pair, Integer> result = searchForDiskOfferingsInternal(cmd); - ListResponse response = new ListResponse(); - List offeringResponses = ViewResponseHelper.createDiskOfferingResponse(result.first().toArray(new DiskOfferingJoinVO[result.first().size()])); + ListResponse response = new ListResponse<>(); + List offeringResponses = ViewResponseHelper.createDiskOfferingResponses(cmd.getVirtualMachineId(), result.first()); response.setResponses(offeringResponses, result.second()); return response; } @@ -3191,6 +3194,7 @@ private Pair, Integer> searchForDiskOfferingsInternal(L Long storagePoolId = cmd.getStoragePoolId(); Boolean encrypt = cmd.getEncrypt(); String storageType = cmd.getStorageType(); + final Long vmId = cmd.getVirtualMachineId(); // Keeping this logic consistent with domain specific zones // if a domainId is provided, we just return the disk offering @@ -3298,6 +3302,16 @@ private Pair, Integer> searchForDiskOfferingsInternal(L sc.addAnd("domainId", SearchCriteria.Op.SC, scc); } + if (vmId != null) { + UserVmVO vm = userVmDao.findById(vmId); + if (vm == null) { + throw new InvalidParameterValueException("Unable to find the VM instance with the specified ID"); + } + if (!isRootAdmin) { + accountMgr.checkAccess(account, null, false, vm); + } + } + Pair, Integer> result = _diskOfferingJoinDao.searchAndCount(sc, searchFilter); String[] requiredTagsArray = new String[0]; if (CollectionUtils.isNotEmpty(result.first()) && VolumeApiServiceImpl.MatchStoragePoolTagsWithDiskOffering.valueIn(zoneId)) { @@ -3387,6 +3401,7 @@ private Pair, Integer> searchForServiceOfferingsInte Integer cpuSpeed = cmd.getCpuSpeed(); Boolean encryptRoot = cmd.getEncryptRoot(); String storageType = cmd.getStorageType(); + final Long templateId = cmd.getTemplateId(); final Account owner = accountMgr.finalizeOwner(caller, accountName, domainId, projectId); SearchCriteria sc = _srvOfferingJoinDao.createSearchCriteria(); @@ -3581,6 +3596,20 @@ private Pair, Integer> searchForServiceOfferingsInte sc.addAnd("domainId", SearchCriteria.Op.SC, scc); } + List hostTags = new ArrayList<>(); + if (templateId != null) { + VMTemplateVO template = _templateDao.findByIdIncludingRemoved(templateId); + if (template == null) { + throw new InvalidParameterValueException("Unable to find template with the specified ID"); + } + if (caller.getType() != Account.Type.ADMIN) { + accountMgr.checkAccess(caller, null, false, template); + } + if (StringUtils.isNotEmpty(template.getTemplateTag())) { + hostTags.add(template.getTemplateTag()); + } + } + if (currentVmOffering != null) { DiskOfferingVO diskOffering = _diskOfferingDao.findByIdIncludingRemoved(currentVmOffering.getDiskOfferingId()); List storageTags = com.cloud.utils.StringUtils.csvTagsToList(diskOffering.getTags()); @@ -3598,25 +3627,28 @@ private Pair, Integer> searchForServiceOfferingsInte sc.addAnd("storageTags", SearchCriteria.Op.SC, scc); } - List hostTags = com.cloud.utils.StringUtils.csvTagsToList(currentVmOffering.getHostTag()); - if (!hostTags.isEmpty()) { - SearchBuilder hostTagsSearchBuilder = _srvOfferingJoinDao.createSearchBuilder(); - for(String tag : hostTags) { - hostTagsSearchBuilder.and(tag, hostTagsSearchBuilder.entity().getHostTag(), Op.FIND_IN_SET); - } - hostTagsSearchBuilder.done(); + List offeringHostTags = com.cloud.utils.StringUtils.csvTagsToList(currentVmOffering.getHostTag()); + if (!offeringHostTags.isEmpty()) { + hostTags.addAll(offeringHostTags); + } + } + if (CollectionUtils.isNotEmpty(hostTags)) { + SearchBuilder hostTagsSearchBuilder = _srvOfferingJoinDao.createSearchBuilder(); + for(String tag : hostTags) { + hostTagsSearchBuilder.and(tag, hostTagsSearchBuilder.entity().getHostTag(), Op.FIND_IN_SET); + } + hostTagsSearchBuilder.done(); - SearchCriteria hostTagsSearchCriteria = hostTagsSearchBuilder.create(); - for(String tag : hostTags) { - hostTagsSearchCriteria.setParameters(tag, tag); - } + SearchCriteria hostTagsSearchCriteria = hostTagsSearchBuilder.create(); + for(String tag : hostTags) { + hostTagsSearchCriteria.setParameters(tag, tag); + } - SearchCriteria finalHostTagsSearchCriteria = _srvOfferingJoinDao.createSearchCriteria(); - finalHostTagsSearchCriteria.addOr("hostTag", Op.NULL); - finalHostTagsSearchCriteria.addOr("hostTag", Op.SC, hostTagsSearchCriteria); + SearchCriteria finalHostTagsSearchCriteria = _srvOfferingJoinDao.createSearchCriteria(); + finalHostTagsSearchCriteria.addOr("hostTag", Op.NULL); + finalHostTagsSearchCriteria.addOr("hostTag", Op.SC, hostTagsSearchCriteria); - sc.addAnd("hostTagsConstraint", SearchCriteria.Op.SC, finalHostTagsSearchCriteria); - } + sc.addAnd("hostTagsConstraint", SearchCriteria.Op.SC, finalHostTagsSearchCriteria); } return _srvOfferingJoinDao.searchAndCount(sc, searchFilter); diff --git a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java index 44096a799b7a..24c556184d5a 100644 --- a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java +++ b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java @@ -556,12 +556,8 @@ public static List createAsyncJobResponse(AsyncJobJoinVO... jo return respList; } - public static List createDiskOfferingResponse(DiskOfferingJoinVO... offerings) { - List respList = new ArrayList(); - for (DiskOfferingJoinVO vt : offerings) { - respList.add(ApiDBUtils.newDiskOfferingResponse(vt)); - } - return respList; + public static List createDiskOfferingResponses(Long vmId, List offerings) { + return ApiDBUtils.newDiskOfferingResponses(vmId, offerings); } public static List createServiceOfferingResponse(ServiceOfferingJoinVO... offerings) { diff --git a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDao.java index 6356addcb4f7..b4427a6315a9 100644 --- a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDao.java @@ -19,7 +19,6 @@ import java.util.List; import java.util.Set; -import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.api.ApiConstants.VMDetails; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.response.UserVmResponse; @@ -28,6 +27,7 @@ import com.cloud.user.Account; import com.cloud.uservm.UserVm; import com.cloud.utils.db.GenericDao; +import com.cloud.vm.VirtualMachine; public interface UserVmJoinDao extends GenericDao { @@ -43,4 +43,7 @@ UserVmResponse newUserVmResponse(ResponseView view, String objectName, UserVmJoi List searchByIds(Long... ids); List listActiveByIsoId(Long isoId); + + List listByAccountServiceOfferingTemplateAndNotInState(long accountId, List states, + List offeringIds, List templateIds); } diff --git a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java index e5cc9ee72342..82c3ad70b0d3 100644 --- a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java @@ -43,6 +43,7 @@ import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.query.QueryService; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; @@ -681,4 +682,30 @@ public List newUserVmView(VirtualMachine... vms) { return searchByIds(vmIdSet.toArray(new Long[vmIdSet.size()])); } + @Override + public List listByAccountServiceOfferingTemplateAndNotInState(long accountId, List states, + List offeringIds, List templateIds) { + SearchBuilder userVmSearch = createSearchBuilder(); + userVmSearch.and("accountId", userVmSearch.entity().getAccountId(), Op.EQ); + userVmSearch.and("serviceOfferingId", userVmSearch.entity().getServiceOfferingId(), Op.IN); + userVmSearch.and("templateId", userVmSearch.entity().getTemplateId(), Op.IN); + userVmSearch.and("state", userVmSearch.entity().getState(), SearchCriteria.Op.NIN); + userVmSearch.and("displayVm", userVmSearch.entity().isDisplayVm(), Op.EQ); + userVmSearch.groupBy(userVmSearch.entity().getId()); // select distinct + userVmSearch.done(); + + SearchCriteria sc = userVmSearch.create(); + sc.setParameters("accountId", accountId); + if (CollectionUtils.isNotEmpty(offeringIds)) { + sc.setParameters("serviceOfferingId", offeringIds.toArray()); + } + if (CollectionUtils.isNotEmpty(templateIds)) { + sc.setParameters("templateId", templateIds.toArray()); + } + if (CollectionUtils.isNotEmpty(states)) { + sc.setParameters("state", states.toArray()); + } + sc.setParameters("displayVm", 1); + return listBy(sc); + } } diff --git a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java index cb22e81f3669..2cd76eca746b 100644 --- a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -121,6 +121,7 @@ import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; +import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.AccountManager; import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; @@ -731,10 +732,11 @@ public DeploymentPlanner getDeploymentPlannerByName(String plannerName) { protected boolean checkVmProfileAndHost(final VirtualMachineProfile vmProfile, final HostVO host) { ServiceOffering offering = vmProfile.getServiceOffering(); - if (offering.getHostTag() != null) { + VirtualMachineTemplate template = vmProfile.getTemplate(); + if (offering.getHostTag() != null || template.getTemplateTag() != null) { _hostDao.loadHostTags(host); - if (!host.checkHostServiceOfferingTags(offering)) { - s_logger.debug("Service Offering host tag does not match the last host of this VM"); + if (!host.checkHostServiceOfferingAndTemplateTags(offering, template)) { + s_logger.debug("Service Offering host tag or template tag does not match the last host of this VM"); return false; } } diff --git a/server/src/main/java/com/cloud/resourcelimit/CheckedReservation.java b/server/src/main/java/com/cloud/resourcelimit/CheckedReservation.java index 5650af153395..cdc74c4f9036 100644 --- a/server/src/main/java/com/cloud/resourcelimit/CheckedReservation.java +++ b/server/src/main/java/com/cloud/resourcelimit/CheckedReservation.java @@ -18,10 +18,16 @@ // package com.cloud.resourcelimit; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.reservation.ReservationVO; import org.apache.cloudstack.reservation.dao.ReservationDao; import org.apache.cloudstack.user.ResourceReservation; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.jetbrains.annotations.NotNull; @@ -33,19 +39,33 @@ import com.cloud.utils.exception.CloudRuntimeException; -public class CheckedReservation implements AutoCloseable, ResourceReservation { +public class CheckedReservation implements AutoCloseable { private static final Logger LOG = Logger.getLogger(CheckedReservation.class); private static final int TRY_TO_GET_LOCK_TIME = 120; private GlobalLock quotaLimitLock; ReservationDao reservationDao; + + ResourceLimitService resourceLimitService; private final Account account; private final ResourceType resourceType; private Long amount; - private ResourceReservation reservation; + private List reservations; + private List resourceLimitTags; private String getContextParameterKey() { - return String.format("%s-%s", ResourceReservation.class.getSimpleName(), resourceType.getName()); + return getResourceReservationContextParameterKey(resourceType); + } + + public static String getResourceReservationContextParameterKey(final ResourceType type) { + return String.format("%s-%s", ResourceReservation.class.getSimpleName(), type.getName()); + } + + protected void checkLimitAndPersistReservation(Account account, ResourceType resourceType, String tag, Long amount) throws ResourceAllocationException { + resourceLimitService.checkResourceLimitWithTag(account, resourceType, tag, amount); + ReservationVO reservationVO = new ReservationVO(account.getAccountId(), account.getDomainId(), resourceType, tag, amount); + ResourceReservation reservation = reservationDao.persist(reservationVO); + this.reservations.add(reservation); } /** @@ -56,13 +76,16 @@ private String getContextParameterKey() { * @param amount positive number of the resource type to reserve * @throws ResourceAllocationException */ - public CheckedReservation(Account account, ResourceType resourceType, Long amount, ReservationDao reservationDao, ResourceLimitService resourceLimitService) throws ResourceAllocationException { + public CheckedReservation(Account account, ResourceType resourceType, List resourceLimitTags, Long amount, + ReservationDao reservationDao, ResourceLimitService resourceLimitService) throws ResourceAllocationException { this.reservationDao = reservationDao; + this.resourceLimitService = resourceLimitService; this.account = account; this.resourceType = resourceType; this.amount = amount; - this.reservation = null; - setGlobalLock(account, resourceType); + this.reservations = new ArrayList<>(); + this.resourceLimitTags = resourceLimitTags; + setGlobalLock(); if (this.amount != null && this.amount <= 0) { if(LOG.isDebugEnabled()){ LOG.debug(String.format("not reserving no amount of resources for %s in domain %d, type: %s, %s ", account.getAccountName(), account.getDomainId(), resourceType, amount)); @@ -73,10 +96,13 @@ public CheckedReservation(Account account, ResourceType resourceType, Long amoun if (this.amount != null) { if(quotaLimitLock.lock(TRY_TO_GET_LOCK_TIME)) { try { - resourceLimitService.checkResourceLimit(account,resourceType,amount); - ReservationVO reservationVO = new ReservationVO(account.getAccountId(), account.getDomainId(), resourceType, amount); - this.reservation = reservationDao.persist(reservationVO); - CallContext.current().putContextParameter(getContextParameterKey(), reservation.getId()); + checkLimitAndPersistReservation(account, resourceType, null, amount); + if (CollectionUtils.isNotEmpty(resourceLimitTags)) { + for (String tag: resourceLimitTags) { + checkLimitAndPersistReservation(account, resourceType, tag, amount); + } + } + CallContext.current().putContextParameter(getContextParameterKey(), getIds()); } catch (NullPointerException npe) { throw new CloudRuntimeException("not enough means to check limits", npe); } finally { @@ -87,13 +113,18 @@ public CheckedReservation(Account account, ResourceType resourceType, Long amoun } } else { if(LOG.isDebugEnabled()){ - LOG.debug(String.format("not reserving no amount of resources for %s in domain %d, type: %s ", account.getAccountName(), account.getDomainId(), resourceType)); + LOG.debug(String.format("not reserving no amount of resources for %s in domain %d, type: %s, tag: %s", account.getAccountName(), account.getDomainId(), resourceType, getResourceLimitTagsAsString())); } } } + public CheckedReservation(Account account, ResourceType resourceType, Long amount, ReservationDao reservationDao, + ResourceLimitService resourceLimitService) throws ResourceAllocationException { + this(account, resourceType, null, amount, reservationDao, resourceLimitService); + } + @NotNull - private void setGlobalLock(Account account, ResourceType resourceType) { + private void setGlobalLock() { String lockName = String.format("CheckedReservation-%s/%d", account.getDomainId(), resourceType.getOrdinal()); setQuotaLimitLock(GlobalLock.getInternLock(lockName)); } @@ -104,39 +135,36 @@ protected void setQuotaLimitLock(GlobalLock quotaLimitLock) { @Override public void close() throws Exception { - if (this.reservation != null) { - CallContext.current().removeContextParameter(getContextParameterKey()); + if (CollectionUtils.isEmpty(reservations)) { + return; + } + CallContext.current().removeContextParameter(getContextParameterKey()); + for (ResourceReservation reservation : reservations) { reservationDao.remove(reservation.getId()); - reservation = null; } + reservations = null; } public Account getAccount() { return account; } - @Override - public Long getAccountId() { - return account.getId(); + public String getResourceLimitTagsAsString() { + return CollectionUtils.isNotEmpty(resourceLimitTags) ? StringUtils.join(resourceLimitTags) : null; } - @Override - public Long getDomainId() { - return account.getDomainId(); - } - - @Override - public ResourceType getResourceType() { - return resourceType; - } - - @Override public Long getReservedAmount() { return amount; } - @Override - public long getId() { - return this.reservation.getId(); + public List getReservations() { + return reservations; + } + + public List getIds() { + if (CollectionUtils.isEmpty(reservations)) { + return new ArrayList<>(); + } + return reservations.stream().map(ResourceReservation::getId).collect(Collectors.toList()); } } diff --git a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java index 959a0dc3bb2c..a4e8a75a09a7 100644 --- a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java +++ b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java @@ -20,21 +20,24 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.EnumMap; +import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.cluster.ManagementServerHostVO; -import com.cloud.cluster.dao.ManagementServerHostDao; -import com.cloud.utils.db.GlobalLock; import org.apache.cloudstack.acl.SecurityChecker.AccessType; +import org.apache.cloudstack.api.response.AccountResponse; +import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ResourceLimitAndCountResponse; +import org.apache.cloudstack.api.response.TaggedResourceLimitAndCountResponse; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.framework.config.ConfigKey; @@ -46,15 +49,17 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; -import org.apache.cloudstack.user.ResourceReservation; import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; import com.cloud.alert.AlertManager; import com.cloud.api.query.dao.UserVmJoinDao; import com.cloud.api.query.vo.UserVmJoinVO; +import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.dao.ManagementServerHostDao; import com.cloud.configuration.Config; import com.cloud.configuration.Resource; import com.cloud.configuration.Resource.ResourceOwnerType; @@ -76,29 +81,39 @@ import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.vpc.dao.VpcDao; +import com.cloud.offering.DiskOffering; +import com.cloud.offering.ServiceOffering; import com.cloud.projects.Project; import com.cloud.projects.ProjectAccount.Role; import com.cloud.projects.dao.ProjectAccountDao; import com.cloud.projects.dao.ProjectDao; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.SnapshotVO; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeDaoImpl.SumCount; +import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountVO; import com.cloud.user.ResourceLimitService; import com.cloud.user.dao.AccountDao; +import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.DB; import com.cloud.utils.db.EntityManager; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericSearchBuilder; +import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.JoinBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -110,6 +125,7 @@ import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.UserVmDao; @@ -169,6 +185,10 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim private VlanDao _vlanDao; @Inject private ManagementServerHostDao managementServerHostDao; + @Inject + ServiceOfferingDao serviceOfferingDao; + @Inject + DiskOfferingDao diskOfferingDao; protected GenericSearchBuilder templateSizeSearch; protected GenericSearchBuilder snapshotSizeSearch; @@ -176,20 +196,24 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim protected SearchBuilder ResourceCountSearch; ScheduledExecutorService _rcExecutor; long _resourceCountCheckInterval = 0; - Map accountResourceLimitMap = new EnumMap(ResourceType.class); - Map domainResourceLimitMap = new EnumMap(ResourceType.class); - Map projectResourceLimitMap = new EnumMap(ResourceType.class); + Map accountResourceLimitMap = new HashMap<>(); + Map domainResourceLimitMap = new HashMap<>(); + Map projectResourceLimitMap = new HashMap<>(); - protected void removeResourceReservationIfNeededAndIncrementResourceCount(final long accountId, final ResourceType type, final long numToIncrement) { + @SuppressWarnings("unchecked") + protected void removeResourceReservationIfNeededAndIncrementResourceCount(final long accountId, final ResourceType type, String tag, final long numToIncrement) { Transaction.execute(new TransactionCallbackWithExceptionNoReturn() { @Override public void doInTransactionWithoutResult(TransactionStatus status) throws CloudRuntimeException { - Object obj = CallContext.current().getContextParameter(String.format("%s-%s", ResourceReservation.class.getSimpleName(), type.getName())); - if (obj instanceof Long) { - reservationDao.remove((long)obj); + Object obj = CallContext.current().getContextParameter(CheckedReservation.getResourceReservationContextParameterKey(type)); + if (obj instanceof List) { + List reservationIds = (List)obj; // This complains an unchecked casting warning + for (Long reservationId : reservationIds) { + reservationDao.remove(reservationId); + } } - if (!updateResourceCountForAccount(accountId, type, true, numToIncrement)) { + if (!updateResourceCountForAccount(accountId, type, tag, true, numToIncrement)) { // we should fail the operation (resource creation) when failed to update the resource count throw new CloudRuntimeException("Failed to increment resource count of type " + type + " for account id=" + accountId); } @@ -243,41 +267,41 @@ public boolean configure(final String name, final Map params) th } try { - projectResourceLimitMap.put(Resource.ResourceType.public_ip, Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectPublicIPs.key()))); - projectResourceLimitMap.put(Resource.ResourceType.snapshot, Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectSnapshots.key()))); - projectResourceLimitMap.put(Resource.ResourceType.template, Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectTemplates.key()))); - projectResourceLimitMap.put(Resource.ResourceType.user_vm, Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectUserVms.key()))); - projectResourceLimitMap.put(Resource.ResourceType.volume, Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectVolumes.key()))); - projectResourceLimitMap.put(Resource.ResourceType.network, Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectNetworks.key()))); - projectResourceLimitMap.put(Resource.ResourceType.vpc, Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectVpcs.key()))); - projectResourceLimitMap.put(Resource.ResourceType.cpu, Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectCpus.key()))); - projectResourceLimitMap.put(Resource.ResourceType.memory, Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectMemory.key()))); - projectResourceLimitMap.put(Resource.ResourceType.primary_storage, Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectPrimaryStorage.key()))); - projectResourceLimitMap.put(Resource.ResourceType.secondary_storage, MaxProjectSecondaryStorage.value()); - - accountResourceLimitMap.put(Resource.ResourceType.public_ip, Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountPublicIPs.key()))); - accountResourceLimitMap.put(Resource.ResourceType.snapshot, Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountSnapshots.key()))); - accountResourceLimitMap.put(Resource.ResourceType.template, Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountTemplates.key()))); - accountResourceLimitMap.put(Resource.ResourceType.user_vm, Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountUserVms.key()))); - accountResourceLimitMap.put(Resource.ResourceType.volume, Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountVolumes.key()))); - accountResourceLimitMap.put(Resource.ResourceType.network, Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountNetworks.key()))); - accountResourceLimitMap.put(Resource.ResourceType.vpc, Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountVpcs.key()))); - accountResourceLimitMap.put(Resource.ResourceType.cpu, Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountCpus.key()))); - accountResourceLimitMap.put(Resource.ResourceType.memory, Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountMemory.key()))); - accountResourceLimitMap.put(Resource.ResourceType.primary_storage, Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountPrimaryStorage.key()))); - accountResourceLimitMap.put(Resource.ResourceType.secondary_storage, MaxAccountSecondaryStorage.value()); - - domainResourceLimitMap.put(Resource.ResourceType.public_ip, Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainPublicIPs.key()))); - domainResourceLimitMap.put(Resource.ResourceType.snapshot, Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainSnapshots.key()))); - domainResourceLimitMap.put(Resource.ResourceType.template, Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainTemplates.key()))); - domainResourceLimitMap.put(Resource.ResourceType.user_vm, Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainUserVms.key()))); - domainResourceLimitMap.put(Resource.ResourceType.volume, Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainVolumes.key()))); - domainResourceLimitMap.put(Resource.ResourceType.network, Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainNetworks.key()))); - domainResourceLimitMap.put(Resource.ResourceType.vpc, Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainVpcs.key()))); - domainResourceLimitMap.put(Resource.ResourceType.cpu, Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainCpus.key()))); - domainResourceLimitMap.put(Resource.ResourceType.memory, Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainMemory.key()))); - domainResourceLimitMap.put(Resource.ResourceType.primary_storage, Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainPrimaryStorage.key()))); - domainResourceLimitMap.put(Resource.ResourceType.secondary_storage, Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainSecondaryStorage.key()))); + projectResourceLimitMap.put(Resource.ResourceType.public_ip.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectPublicIPs.key()))); + projectResourceLimitMap.put(Resource.ResourceType.snapshot.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectSnapshots.key()))); + projectResourceLimitMap.put(Resource.ResourceType.template.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectTemplates.key()))); + projectResourceLimitMap.put(Resource.ResourceType.user_vm.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectUserVms.key()))); + projectResourceLimitMap.put(Resource.ResourceType.volume.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectVolumes.key()))); + projectResourceLimitMap.put(Resource.ResourceType.network.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectNetworks.key()))); + projectResourceLimitMap.put(Resource.ResourceType.vpc.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectVpcs.key()))); + projectResourceLimitMap.put(Resource.ResourceType.cpu.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectCpus.key()))); + projectResourceLimitMap.put(Resource.ResourceType.memory.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectMemory.key()))); + projectResourceLimitMap.put(Resource.ResourceType.primary_storage.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectPrimaryStorage.key()))); + projectResourceLimitMap.put(Resource.ResourceType.secondary_storage.name(), MaxProjectSecondaryStorage.value()); + + accountResourceLimitMap.put(Resource.ResourceType.public_ip.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountPublicIPs.key()))); + accountResourceLimitMap.put(Resource.ResourceType.snapshot.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountSnapshots.key()))); + accountResourceLimitMap.put(Resource.ResourceType.template.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountTemplates.key()))); + accountResourceLimitMap.put(Resource.ResourceType.user_vm.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountUserVms.key()))); + accountResourceLimitMap.put(Resource.ResourceType.volume.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountVolumes.key()))); + accountResourceLimitMap.put(Resource.ResourceType.network.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountNetworks.key()))); + accountResourceLimitMap.put(Resource.ResourceType.vpc.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountVpcs.key()))); + accountResourceLimitMap.put(Resource.ResourceType.cpu.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountCpus.key()))); + accountResourceLimitMap.put(Resource.ResourceType.memory.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountMemory.key()))); + accountResourceLimitMap.put(Resource.ResourceType.primary_storage.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountPrimaryStorage.key()))); + accountResourceLimitMap.put(Resource.ResourceType.secondary_storage.name(), MaxAccountSecondaryStorage.value()); + + domainResourceLimitMap.put(Resource.ResourceType.public_ip.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainPublicIPs.key()))); + domainResourceLimitMap.put(Resource.ResourceType.snapshot.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainSnapshots.key()))); + domainResourceLimitMap.put(Resource.ResourceType.template.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainTemplates.key()))); + domainResourceLimitMap.put(Resource.ResourceType.user_vm.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainUserVms.key()))); + domainResourceLimitMap.put(Resource.ResourceType.volume.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainVolumes.key()))); + domainResourceLimitMap.put(Resource.ResourceType.network.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainNetworks.key()))); + domainResourceLimitMap.put(Resource.ResourceType.vpc.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainVpcs.key()))); + domainResourceLimitMap.put(Resource.ResourceType.cpu.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainCpus.key()))); + domainResourceLimitMap.put(Resource.ResourceType.memory.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainMemory.key()))); + domainResourceLimitMap.put(Resource.ResourceType.primary_storage.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainPrimaryStorage.key()))); + domainResourceLimitMap.put(Resource.ResourceType.secondary_storage.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainSecondaryStorage.key()))); } catch (NumberFormatException e) { s_logger.error("NumberFormatException during configuration", e); throw new ConfigurationException("Configuration failed due to NumberFormatException, see log for the stacktrace"); @@ -287,7 +311,7 @@ public boolean configure(final String name, final Map params) th } @Override - public void incrementResourceCount(long accountId, ResourceType type, Long... delta) { + public void incrementResourceCountWithTag(long accountId, ResourceType type, String tag, Long... delta) { // don't upgrade resource count for system account if (accountId == Account.ACCOUNT_ID_SYSTEM) { s_logger.trace("Not incrementing resource count for system accounts, returning"); @@ -295,11 +319,16 @@ public void incrementResourceCount(long accountId, ResourceType type, Long... de } final long numToIncrement = (delta.length == 0) ? 1 : delta[0].longValue(); - removeResourceReservationIfNeededAndIncrementResourceCount(accountId, type, numToIncrement); + removeResourceReservationIfNeededAndIncrementResourceCount(accountId, type, tag, numToIncrement); } @Override - public void decrementResourceCount(long accountId, ResourceType type, Long... delta) { + public void incrementResourceCount(long accountId, ResourceType type, Long... delta) { + incrementResourceCountWithTag(accountId, type, null, delta); + } + + @Override + public void decrementResourceCountWithTag(long accountId, ResourceType type, String tag, Long... delta) { // don't upgrade resource count for system account if (accountId == Account.ACCOUNT_ID_SYSTEM) { s_logger.trace("Not decrementing resource count for system accounts, returning"); @@ -307,14 +336,19 @@ public void decrementResourceCount(long accountId, ResourceType type, Long... de } long numToDecrement = (delta.length == 0) ? 1 : delta[0].longValue(); - if (!updateResourceCountForAccount(accountId, type, false, numToDecrement)) { + if (!updateResourceCountForAccount(accountId, type, tag, false, numToDecrement)) { _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPDATE_RESOURCE_COUNT, 0L, 0L, "Failed to decrement resource count of type " + type + " for account id=" + accountId, "Failed to decrement resource count of type " + type + " for account id=" + accountId + "; use updateResourceCount API to recalculate/fix the problem"); } } @Override - public long findCorrectResourceLimitForAccount(Account account, ResourceType type) { + public void decrementResourceCount(long accountId, ResourceType type, Long... delta) { + decrementResourceCountWithTag(accountId, type, null, delta); + } + + @Override + public long findCorrectResourceLimitForAccount(Account account, ResourceType type, String tag) { long max = Resource.RESOURCE_UNLIMITED; // if resource limit is not found, then we treat it as unlimited @@ -323,18 +357,22 @@ public long findCorrectResourceLimitForAccount(Account account, ResourceType typ return max; } - ResourceLimitVO limit = _resourceLimitDao.findByOwnerIdAndType(account.getId(), ResourceOwnerType.Account, type); + ResourceLimitVO limit = _resourceLimitDao.findByOwnerIdAndTypeAndTag(account.getId(), ResourceOwnerType.Account, type, tag); // Check if limit is configured for account if (limit != null) { max = limit.getMax().longValue(); } else { + String resourceTypeName = type.name(); // If the account has an no limit set, then return global default account limits Long value = null; if (account.getType() == Account.Type.PROJECT) { - value = projectResourceLimitMap.get(type); + value = projectResourceLimitMap.get(resourceTypeName); } else { - value = accountResourceLimitMap.get(type); + if (StringUtils.isNotEmpty(tag)) { + return findCorrectResourceLimitForAccount(account, type, null); + } + value = accountResourceLimitMap.get(resourceTypeName); } if (value != null) { if (value < 0) { // return unlimit if value is set to negative @@ -373,9 +411,9 @@ public long findCorrectResourceLimitForAccount(long accountId, Long limit, Resou // If the account has an no limit set, then return global default account limits Long value = null; if (account.getType() == Account.Type.PROJECT) { - value = projectResourceLimitMap.get(type); + value = projectResourceLimitMap.get(type.getName()); } else { - value = accountResourceLimitMap.get(type); + value = accountResourceLimitMap.get(type.getName()); } if (value != null) { if (value < 0) { // return unlimit if value is set to negative @@ -392,7 +430,7 @@ public long findCorrectResourceLimitForAccount(long accountId, Long limit, Resou } @Override - public long findCorrectResourceLimitForDomain(Domain domain, ResourceType type) { + public long findCorrectResourceLimitForDomain(Domain domain, ResourceType type, String tag) { long max = Resource.RESOURCE_UNLIMITED; // no limits on ROOT domain @@ -400,7 +438,7 @@ public long findCorrectResourceLimitForDomain(Domain domain, ResourceType type) return Resource.RESOURCE_UNLIMITED; } // Check account - ResourceLimitVO limit = _resourceLimitDao.findByOwnerIdAndType(domain.getId(), ResourceOwnerType.Domain, type); + ResourceLimitVO limit = _resourceLimitDao.findByOwnerIdAndTypeAndTag(domain.getId(), ResourceOwnerType.Domain, type, tag); if (limit != null) { max = limit.getMax().longValue(); @@ -411,7 +449,7 @@ public long findCorrectResourceLimitForDomain(Domain domain, ResourceType type) if (domainId == Domain.ROOT_DOMAIN) { break; } - limit = _resourceLimitDao.findByOwnerIdAndType(domainId, ResourceOwnerType.Domain, type); + limit = _resourceLimitDao.findByOwnerIdAndTypeAndTag(domainId, ResourceOwnerType.Domain, type, tag); DomainVO tmpDomain = _domainDao.findById(domainId); domainId = tmpDomain.getParent(); } @@ -419,8 +457,11 @@ public long findCorrectResourceLimitForDomain(Domain domain, ResourceType type) if (limit != null) { max = limit.getMax().longValue(); } else { + if (StringUtils.isNotEmpty(tag)) { + return findCorrectResourceLimitForDomain(domain, type, null); + } Long value = null; - value = domainResourceLimitMap.get(type); + value = domainResourceLimitMap.get(type.name()); if (value != null) { if (value < 0) { // return unlimit if value is set to negative return max; @@ -436,7 +477,7 @@ public long findCorrectResourceLimitForDomain(Domain domain, ResourceType type) return max; } - private void checkDomainResourceLimit(final Account account, final Project project, final ResourceType type, long numResources) throws ResourceAllocationException { + private void checkDomainResourceLimit(final Account account, final Project project, final ResourceType type, String tag, long numResources) throws ResourceAllocationException { // check all domains in the account's domain hierarchy Long domainId = null; if (project != null) { @@ -449,9 +490,9 @@ private void checkDomainResourceLimit(final Account account, final Project proje DomainVO domain = _domainDao.findById(domainId); // no limit check if it is ROOT domain if (domainId != Domain.ROOT_DOMAIN) { - long domainResourceLimit = findCorrectResourceLimitForDomain(domain, type); - long currentDomainResourceCount = _resourceCountDao.getResourceCount(domainId, ResourceOwnerType.Domain, type); - long currentResourceReservation = reservationDao.getDomainReservation(domainId, type); + long domainResourceLimit = findCorrectResourceLimitForDomain(domain, type, tag); + long currentDomainResourceCount = _resourceCountDao.getResourceCount(domainId, ResourceOwnerType.Domain, type, tag); + long currentResourceReservation = reservationDao.getDomainReservation(domainId, type, tag); long requestedDomainResourceCount = currentDomainResourceCount + currentResourceReservation + numResources; String convDomainResourceLimit = String.valueOf(domainResourceLimit); @@ -466,10 +507,14 @@ private void checkDomainResourceLimit(final Account account, final Project proje convNumResources = toHumanReadableSize(numResources); } + String typeString = type.getName(); + if (StringUtils.isNotEmpty(tag)) { + typeString = String.format("%s (tag: %s)", typeString, tag); + } String messageSuffix = String.format( " domain resource limits of Type '%s' for Domain Id = %s is exceeded: Domain Resource Limit = %s, " + "Current Domain Resource Amount = %s, Current Resource Reservation = %s, Requested Resource Amount = %s.", - type, domainId, convDomainResourceLimit, + typeString, domain.getUuid(), convDomainResourceLimit, convCurrentDomainResourceCount, convCurrentResourceReservation, convNumResources ); @@ -488,11 +533,11 @@ private void checkDomainResourceLimit(final Account account, final Project proje } } - private void checkAccountResourceLimit(final Account account, final Project project, final ResourceType type, long numResources) throws ResourceAllocationException { + private void checkAccountResourceLimit(final Account account, final Project project, final ResourceType type, String tag, long numResources) throws ResourceAllocationException { // Check account limits - long accountResourceLimit = findCorrectResourceLimitForAccount(account, type); - long currentResourceCount = _resourceCountDao.getResourceCount(account.getId(), ResourceOwnerType.Account, type); - long currentResourceReservation = reservationDao.getAccountReservation(account.getId(), type); + long accountResourceLimit = findCorrectResourceLimitForAccount(account, type, tag); + long currentResourceCount = _resourceCountDao.getResourceCount(account.getId(), ResourceOwnerType.Account, type, tag); + long currentResourceReservation = reservationDao.getAccountReservation(account.getId(), type, tag); long requestedResourceCount = currentResourceCount + currentResourceReservation + numResources; String convertedAccountResourceLimit = String.valueOf(accountResourceLimit); @@ -508,9 +553,9 @@ private void checkAccountResourceLimit(final Account account, final Project proj } String messageSuffix = String.format( - " amount of resources of Type = '%s' for %s in Domain Id = %s is exceeded: " + + " amount of resources of Type = '%s', tag = '%s' for %s in Domain Id = %s is exceeded: " + "Account Resource Limit = %s, Current Account Resource Amount = %s, Current Account Resource Reservation = %s, Requested Resource Amount = %s.", - type, (project == null ? "Account Name = " + account.getAccountName() : "Project Name = " + project.getName()), account.getDomainId(), + type, tag, (project == null ? "Account Name = " + account.getAccountName() : "Project Name = " + project.getName()), account.getDomainId(), convertedAccountResourceLimit, convertedCurrentResourceCount, convertedCurrentResourceReservation, convertedNumResources ); @@ -526,15 +571,15 @@ private void checkAccountResourceLimit(final Account account, final Project proj } } - private List lockAccountAndOwnerDomainRows(long accountId, final ResourceType type) { - Set rowIdsToLock = _resourceCountDao.listAllRowsToUpdate(accountId, ResourceOwnerType.Account, type); + private List lockAccountAndOwnerDomainRows(long accountId, final ResourceType type, String tag) { + Set rowIdsToLock = _resourceCountDao.listAllRowsToUpdate(accountId, ResourceOwnerType.Account, type, tag); SearchCriteria sc = ResourceCountSearch.create(); sc.setParameters("id", rowIdsToLock.toArray()); return _resourceCountDao.lockRows(sc, null, true); } - private List lockDomainRows(long domainId, final ResourceType type) { - Set rowIdsToLock = _resourceCountDao.listAllRowsToUpdate(domainId, ResourceOwnerType.Domain, type); + private List lockDomainRows(long domainId, final ResourceType type, String tag) { + Set rowIdsToLock = _resourceCountDao.listAllRowsToUpdate(domainId, ResourceOwnerType.Domain, type, tag); SearchCriteria sc = ResourceCountSearch.create(); sc.setParameters("id", rowIdsToLock.toArray()); return _resourceCountDao.lockRows(sc, null, true); @@ -543,7 +588,7 @@ private List lockDomainRows(long domainId, final ResourceType t @Override public long findDefaultResourceLimitForDomain(ResourceType resourceType) { Long resourceLimit = null; - resourceLimit = domainResourceLimitMap.get(resourceType); + resourceLimit = domainResourceLimitMap.get(resourceType.getName()); if (resourceLimit != null && (resourceType == ResourceType.primary_storage || resourceType == ResourceType.secondary_storage)) { if (! Long.valueOf(Resource.RESOURCE_UNLIMITED).equals(resourceLimit)) { resourceLimit = resourceLimit * ResourceType.bytesToGiB; @@ -555,9 +600,9 @@ public long findDefaultResourceLimitForDomain(ResourceType resourceType) { } @Override - public long findCorrectResourceLimitForAccountAndDomain(Account account, Domain domain, ResourceType type) { - long maxSecondaryStorageForAccount = findCorrectResourceLimitForAccount(account, type); - long maxSecondaryStorageForDomain = findCorrectResourceLimitForDomain(domain, type); + public long findCorrectResourceLimitForAccountAndDomain(Account account, Domain domain, ResourceType type, String tag) { + long maxSecondaryStorageForAccount = findCorrectResourceLimitForAccount(account, type, tag); + long maxSecondaryStorageForDomain = findCorrectResourceLimitForDomain(domain, type, tag); if (maxSecondaryStorageForDomain == Resource.RESOURCE_UNLIMITED || maxSecondaryStorageForAccount == Resource.RESOURCE_UNLIMITED) { return Math.max(maxSecondaryStorageForDomain, maxSecondaryStorageForAccount); @@ -569,6 +614,12 @@ public long findCorrectResourceLimitForAccountAndDomain(Account account, Domain @Override @DB public void checkResourceLimit(final Account account, final ResourceType type, long... count) throws ResourceAllocationException { + checkResourceLimitWithTag(account, type, null, count); + } + + @Override + @DB + public void checkResourceLimitWithTag(final Account account, final ResourceType type, String tag, long... count) throws ResourceAllocationException { final long numResources = ((count.length == 0) ? 1 : count[0]); Project project = null; @@ -586,17 +637,42 @@ public void checkResourceLimit(final Account account, final ResourceType type, l @Override public void doInTransactionWithoutResult(TransactionStatus status) throws ResourceAllocationException { // Lock all rows first so nobody else can read it - lockAccountAndOwnerDomainRows(account.getId(), type); + lockAccountAndOwnerDomainRows(account.getId(), type, tag); // Check account limits - checkAccountResourceLimit(account, projectFinal, type, numResources); + checkAccountResourceLimit(account, projectFinal, type, tag, numResources); // check all domains in the account's domain hierarchy - checkDomainResourceLimit(account, projectFinal, type, numResources); + checkDomainResourceLimit(account, projectFinal, type, tag, numResources); } }); } + /** + * To retrieve host and storage limit tags lists with or without a given tag string + * while searching for limits for an account or domain + * @param tag - tag string to filter list of host and storage limit tags + * @return a pair of host tags list and storage tags list + */ + protected Pair, List> getResourceLimitTagsForLimitSearch(String tag) { + List hostTags = getResourceLimitHostTags(); + List storageTags = getResourceLimitStorageTags(); + if (tag == null) { + return new Pair<>(hostTags, storageTags); + } + if (hostTags.contains(tag)) { + hostTags = List.of(tag); + } else { + hostTags = new ArrayList<>(); + } + if (storageTags.contains(tag)) { + storageTags = List.of(tag); + } else { + storageTags = new ArrayList<>(); + } + return new Pair<>(hostTags, storageTags); + } + @Override - public List searchForLimits(Long id, Long accountId, Long domainId, ResourceType resourceType, Long startIndex, Long pageSizeVal) { + public List searchForLimits(Long id, Long accountId, Long domainId, ResourceType resourceType, String tag, Long startIndex, Long pageSizeVal) { Account caller = CallContext.current().getCallingAccount(); List limits = new ArrayList(); boolean isAccount = true; @@ -659,6 +735,7 @@ public List searchForLimits(Long id, Long accountId, Long domai sb.and("accountId", sb.entity().getAccountId(), SearchCriteria.Op.EQ); sb.and("domainId", sb.entity().getDomainId(), SearchCriteria.Op.EQ); sb.and("type", sb.entity().getType(), SearchCriteria.Op.EQ); + sb.and("tag", sb.entity().getTag(), SearchCriteria.Op.EQ); SearchCriteria sc = sb.create(); Filter filter = new Filter(ResourceLimitVO.class, "id", true, startIndex, pageSizeVal); @@ -676,15 +753,29 @@ public List searchForLimits(Long id, Long accountId, Long domai sc.setParameters("type", resourceType); } + if (tag != null) { + sc.setParameters("tag", tag); + } + List foundLimits = _resourceLimitDao.search(sc, filter); + Pair, List> tagsPair = getResourceLimitTagsForLimitSearch(tag); + List hostTags = tagsPair.first(); + List storageTags = tagsPair.second(); + if (resourceType != null) { if (foundLimits.isEmpty()) { + ResourceOwnerType ownerType = ResourceOwnerType.Domain; + Long ownerId = domainId; + long max = 0; if (isAccount) { - limits.add(new ResourceLimitVO(resourceType, findCorrectResourceLimitForAccount(_accountMgr.getAccount(accountId), resourceType), accountId, ResourceOwnerType.Account)); + ownerType = ResourceOwnerType.Account; + ownerId = accountId; + max = findCorrectResourceLimitForAccount(_accountMgr.getAccount(accountId), resourceType, tag); } else { - limits.add(new ResourceLimitVO(resourceType, findCorrectResourceLimitForDomain(_domainDao.findById(domainId), resourceType), domainId, ResourceOwnerType.Domain)); + max = findCorrectResourceLimitForDomain(_domainDao.findById(domainId), resourceType, tag); } + limits.add(new ResourceLimitVO(resourceType, max, ownerId, ownerType)); } else { limits.addAll(foundLimits); } @@ -709,28 +800,83 @@ public List searchForLimits(Long id, Long accountId, Long domai if (accountLimitStr.size() < resourceTypes.length) { for (ResourceType rt : resourceTypes) { if (!accountLimitStr.contains(rt.toString())) { - limits.add(new ResourceLimitVO(rt, findCorrectResourceLimitForAccount(_accountMgr.getAccount(accountId), rt), accountId, ResourceOwnerType.Account)); + limits.add(new ResourceLimitVO(rt, findCorrectResourceLimitForAccount(_accountMgr.getAccount(accountId), rt, null), accountId, ResourceOwnerType.Account)); } } } - } else { if (domainLimitStr.size() < resourceTypes.length) { for (ResourceType rt : resourceTypes) { if (!domainLimitStr.contains(rt.toString())) { - limits.add(new ResourceLimitVO(rt, findCorrectResourceLimitForDomain(_domainDao.findById(domainId), rt), domainId, ResourceOwnerType.Domain)); + limits.add(new ResourceLimitVO(rt, findCorrectResourceLimitForDomain(_domainDao.findById(domainId), rt, null), domainId, ResourceOwnerType.Domain)); } } } } } } - + addTaggedResourceLimits(limits, resourceType, isAccount ? ResourceOwnerType.Account : ResourceOwnerType.Domain, isAccount ? accountId : domainId, hostTags, storageTags); return limits; } + protected void addTaggedResourceLimits(List limits, List types, List tags, ResourceOwnerType ownerType, long ownerId) { + if (CollectionUtils.isEmpty(tags)) { + return; + } + if (CollectionUtils.isEmpty(types)) { + return; + } + for (String tag : tags) { + for (ResourceType type : types) { + if (limits.stream().noneMatch(l -> type.equals(l.getType()) && tag.equals(l.getTag()))) { + limits.add(new ResourceLimitVO(type, ResourceOwnerType.Domain.equals(ownerType) ? + findCorrectResourceLimitForDomain(_domainDao.findById(ownerId), type, tag) : + findCorrectResourceLimitForAccount(_accountDao.findById(ownerId), type, tag), + ownerId, ownerType, tag)); + } + } + } + } + + protected void removeUndesiredTaggedLimits(List limits, List hostTags, List storageTags) { + Iterator itr = limits.iterator(); + while (itr.hasNext()) { + ResourceLimitVO limit = itr.next(); + if (StringUtils.isEmpty(limit.getTag())) { + continue; + } + if (HostTagsSupportingTypes.contains(limit.getType()) && + (CollectionUtils.isEmpty(hostTags) || !hostTags.contains(limit.getTag()))) { + itr.remove(); + } + if (StorageTagsSupportingTypes.contains(limit.getType()) && + (CollectionUtils.isEmpty(storageTags) || !storageTags.contains(limit.getTag()))) { + itr.remove(); + } + } + } + + protected void addTaggedResourceLimits(List limits, ResourceType resourceType, ResourceOwnerType ownerType, long ownerId, List hostTags, List storageTags) { + if (CollectionUtils.isEmpty(hostTags) && CollectionUtils.isEmpty(storageTags)) { + return; + } + List types = resourceType != null ? HostTagsSupportingTypes.contains(resourceType) ? List.of(resourceType) : null : HostTagsSupportingTypes; + addTaggedResourceLimits(limits, types, hostTags, ownerType, ownerId); + types = resourceType != null ? StorageTagsSupportingTypes.contains(resourceType) ? List.of(resourceType) : null : StorageTagsSupportingTypes; + addTaggedResourceLimits(limits, types, storageTags, ownerType, ownerId); + removeUndesiredTaggedLimits(limits, hostTags, storageTags); + limits.sort((o1, o2) -> { + Integer type1 = o1.getType().getOrdinal(); + Integer type2 = o2.getType().getOrdinal(); + if (type1.equals(type2)) { + return StringUtils.defaultString(o1.getTag(), "").compareTo(StringUtils.defaultString(o2.getTag(), "")); + } + return type1.compareTo(type2); + }); + } + @Override - public ResourceLimitVO updateResourceLimit(Long accountId, Long domainId, Integer typeId, Long max) { + public ResourceLimitVO updateResourceLimit(Long accountId, Long domainId, Integer typeId, Long max, String tag) { Account caller = CallContext.current().getCallingAccount(); if (max == null) { @@ -752,6 +898,12 @@ public ResourceLimitVO updateResourceLimit(Long accountId, Long domainId, Intege } } + if (StringUtils.isNotEmpty(tag) && + !(HostTagsSupportingTypes.contains(resourceType) || + StorageTagsSupportingTypes.contains(resourceType))) { + throw new InvalidParameterValueException(String.format("Resource limit with a tag is not supported for resource type %d", typeId)); + } + //Convert max storage size from GiB to bytes if ((resourceType == ResourceType.primary_storage || resourceType == ResourceType.secondary_storage) && max >= 0) { max *= ResourceType.bytesToGiB; @@ -787,6 +939,13 @@ public ResourceLimitVO updateResourceLimit(Long accountId, Long domainId, Intege ownerType = ResourceOwnerType.Account; ownerId = accountId; + if (StringUtils.isNotEmpty(tag)) { + long untaggedLimit = findCorrectResourceLimitForAccount(account, resourceType, null); + if (untaggedLimit > 0 && max > untaggedLimit) { + throw new InvalidParameterValueException(String.format("Maximum untagged resource limit for account %s for resource type %s is %d, please specify a value less than or equal to that", + account.getAccountName(), resourceType, untaggedLimit)); + } + } } else if (domainId != null) { Domain domain = _entityMgr.findById(Domain.class, domainId); @@ -801,13 +960,20 @@ public ResourceLimitVO updateResourceLimit(Long accountId, Long domainId, Intege // if the admin is trying to update their own domain, disallow... throw new PermissionDeniedException("Unable to update resource limit for domain " + domainId + ", permission denied"); } + if (StringUtils.isNotEmpty(tag)) { + long untaggedLimit = findCorrectResourceLimitForDomain(domain, resourceType, null); + if (untaggedLimit > 0 && max > untaggedLimit) { + throw new InvalidParameterValueException(String.format("Maximum untagged resource limit for domain %s for resource type %s is %d, please specify a value less than or equal to that", + domain.getName(), resourceType, untaggedLimit)); + } + } Long parentDomainId = domain.getParent(); if (parentDomainId != null) { DomainVO parentDomain = _domainDao.findById(parentDomainId); - long parentMaximum = findCorrectResourceLimitForDomain(parentDomain, resourceType); + long parentMaximum = findCorrectResourceLimitForDomain(parentDomain, resourceType, tag); if ((parentMaximum >= 0) && (max.longValue() > parentMaximum)) { throw new InvalidParameterValueException("Domain " + domain.getName() + "(id: " + parentDomain.getId() + ") has maximum allowed resource limit " + parentMaximum + " for " - + resourceType + ", please specify a value less that or equal to " + parentMaximum); + + resourceType + ", please specify a value less than or equal to " + parentMaximum); } } ownerType = ResourceOwnerType.Domain; @@ -818,18 +984,73 @@ public ResourceLimitVO updateResourceLimit(Long accountId, Long domainId, Intege throw new InvalidParameterValueException("AccountId or domainId have to be specified in order to update resource limit"); } - ResourceLimitVO limit = _resourceLimitDao.findByOwnerIdAndType(ownerId, ownerType, resourceType); + ResourceLimitVO limit = _resourceLimitDao.findByOwnerIdAndTypeAndTag(ownerId, ownerType, resourceType, tag); if (limit != null) { // Update the existing limit _resourceLimitDao.update(limit.getId(), max); return _resourceLimitDao.findById(limit.getId()); } else { - return _resourceLimitDao.persist(new ResourceLimitVO(resourceType, max, ownerId, ownerType)); + return _resourceLimitDao.persist(new ResourceLimitVO(resourceType, max, ownerId, ownerType, tag)); + } + } + + protected boolean isTaggedResourceCountRecalculationNotNeeded(ResourceType type, List hostTags, List storageTags) { + if (!HostTagsSupportingTypes.contains(type) && !StorageTagsSupportingTypes.contains(type)) { + return true; + } + return CollectionUtils.isEmpty(hostTags) && CollectionUtils.isEmpty(storageTags); + } + + protected List recalculateAccountTaggedResourceCount(long accountId, ResourceType type, final List hostTags, final List storageTags) { + List result = new ArrayList<>(); + final List allTags = new ArrayList<>(hostTags); + allTags.addAll(storageTags); + _resourceCountDao.removeResourceCountsForNonMatchingTags(accountId, ResourceOwnerType.Account, type, allTags); + _resourceLimitDao.removeResourceLimitsForNonMatchingTags(accountId, ResourceOwnerType.Account, type, allTags); + if (isTaggedResourceCountRecalculationNotNeeded(type, hostTags, storageTags)) { + return result; } + if (HostTagsSupportingTypes.contains(type) && CollectionUtils.isNotEmpty(hostTags)) { + for (String tag : hostTags) { + long count = recalculateAccountResourceCount(accountId, type, tag); + result.add(new ResourceCountVO(type, count, accountId, ResourceOwnerType.Account, tag)); + } + } + if (StorageTagsSupportingTypes.contains(type) && CollectionUtils.isNotEmpty(storageTags)) { + for (String tag : storageTags) { + long count = recalculateAccountResourceCount(accountId, type, tag); + result.add(new ResourceCountVO(type, count, accountId, ResourceOwnerType.Account, tag)); + } + } + return result; + } + + protected List recalculateDomainTaggedResourceCount(long domainId, ResourceType type, final List hostTags, final List storageTags) { + List result = new ArrayList<>(); + final List allTags = new ArrayList<>(hostTags); + allTags.addAll(storageTags); + _resourceCountDao.removeResourceCountsForNonMatchingTags(domainId, ResourceOwnerType.Domain, type, allTags); + _resourceLimitDao.removeResourceLimitsForNonMatchingTags(domainId, ResourceOwnerType.Domain, type, allTags); + if (isTaggedResourceCountRecalculationNotNeeded(type, hostTags, storageTags)) { + return result; + } + if (HostTagsSupportingTypes.contains(type) && CollectionUtils.isNotEmpty(hostTags)) { + for (String tag : hostTags) { + long count = recalculateDomainResourceCount(domainId, type, tag); + result.add(new ResourceCountVO(type, count, domainId, ResourceOwnerType.Domain, tag)); + } + } + if (StorageTagsSupportingTypes.contains(type) && CollectionUtils.isNotEmpty(storageTags)) { + for (String tag : storageTags) { + long count = recalculateDomainResourceCount(domainId, type, tag); + result.add(new ResourceCountVO(type, count, domainId, ResourceOwnerType.Domain, tag)); + } + } + return result; } @Override - public List recalculateResourceCount(Long accountId, Long domainId, Integer typeId) throws InvalidParameterValueException, CloudRuntimeException, PermissionDeniedException { + public List recalculateResourceCount(Long accountId, Long domainId, Integer typeId, String tag) throws CloudRuntimeException { Account callerAccount = CallContext.current().getCallingAccount(); long count = 0; List counts = new ArrayList(); @@ -846,6 +1067,11 @@ public List recalculateResourceCount(Long accountId, Long domai if (resourceType == null) { throw new InvalidParameterValueException("Please specify valid resource type"); } + if (StringUtils.isNotEmpty(tag) && + !(HostTagsSupportingTypes.contains(resourceType) || + StorageTagsSupportingTypes.contains(resourceType))) { + throw new InvalidParameterValueException(String.format("Resource count with a tag is not supported for resource type %d", typeId)); + } } DomainVO domain = _domainDao.findById(domainId); @@ -860,35 +1086,48 @@ public List recalculateResourceCount(Long accountId, Long domai resourceTypes = Arrays.asList(Resource.ResourceType.values()); } + List hostTags = getResourceLimitHostTags(); + List storageTags = getResourceLimitStorageTags(); for (ResourceType type : resourceTypes) { if (accountId != null) { - count = recalculateAccountResourceCount(accountId, type); + count = recalculateAccountResourceCount(accountId, type, tag); counts.add(new ResourceCountVO(type, count, accountId, ResourceOwnerType.Account)); - + if (StringUtils.isEmpty(tag)) { + counts.addAll(recalculateAccountTaggedResourceCount(accountId, type, hostTags, storageTags)); + } } else { - count = recalculateDomainResourceCount(domainId, type); + count = recalculateDomainResourceCount(domainId, type, tag); counts.add(new ResourceCountVO(type, count, domainId, ResourceOwnerType.Domain)); + if (StringUtils.isEmpty(tag)) { + counts.addAll(recalculateDomainTaggedResourceCount(domainId, type, hostTags, storageTags)); + } } } return counts; } + @Override + public List recalculateResourceCount(Long accountId, Long domainId, Integer typeId) throws CloudRuntimeException { + return recalculateResourceCount(accountId, domainId, typeId, null); + } + @DB - protected boolean updateResourceCountForAccount(final long accountId, final ResourceType type, final boolean increment, final long delta) { + protected boolean updateResourceCountForAccount(final long accountId, final ResourceType type, String tag, final boolean increment, final long delta) { if (s_logger.isDebugEnabled()) { String convertedDelta = String.valueOf(delta); if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage){ convertedDelta = toHumanReadableSize(delta); } - s_logger.debug("Updating resource Type = " + type + " count for Account = " + accountId + " Operation = " + (increment ? "increasing" : "decreasing") + " Amount = " + convertedDelta); + String typeStr = StringUtils.isNotEmpty(tag) ? String.format("%s (tag: %s)", type, tag) : type.getName(); + s_logger.debug("Updating resource Type = " + typeStr + " count for Account = " + accountId + " Operation = " + (increment ? "increasing" : "decreasing") + " Amount = " + convertedDelta); } try { return Transaction.execute(new TransactionCallback() { @Override public Boolean doInTransaction(TransactionStatus status) { boolean result = true; - List rowsToUpdate = lockAccountAndOwnerDomainRows(accountId, type); + List rowsToUpdate = lockAccountAndOwnerDomainRows(accountId, type, tag); for (ResourceCountVO rowToUpdate : rowsToUpdate) { if (!_resourceCountDao.updateById(rowToUpdate.getId(), increment, delta)) { s_logger.trace("Unable to update resource count for the row " + rowToUpdate); @@ -913,13 +1152,13 @@ public Boolean doInTransaction(TransactionStatus status) { * @return the resulting new resource count */ @DB - protected long recalculateDomainResourceCount(final long domainId, final ResourceType type) { + protected long recalculateDomainResourceCount(final long domainId, final ResourceType type, String tag) { return Transaction.execute(new TransactionCallback() { @Override public Long doInTransaction(TransactionStatus status) { long newResourceCount = 0; - lockDomainRows(domainId, type); - ResourceCountVO domainRC = _resourceCountDao.findByOwnerAndType(domainId, ResourceOwnerType.Domain, type); + lockDomainRows(domainId, type, tag); + ResourceCountVO domainRC = _resourceCountDao.findByOwnerAndTypeAndTag(domainId, ResourceOwnerType.Domain, type, tag); long oldResourceCount = domainRC.getCount(); List domainChildren = _domainDao.findImmediateChildrenForParent(domainId); @@ -931,15 +1170,16 @@ public Long doInTransaction(TransactionStatus status) { } for (DomainVO childDomain : domainChildren) { - long childDomainResourceCount = recalculateDomainResourceCount(childDomain.getId(), type); + long childDomainResourceCount = recalculateDomainResourceCount(childDomain.getId(), type, tag); newResourceCount += childDomainResourceCount; // add the child domain count to parent domain count } + List accounts = _accountDao.findActiveAccountsForDomain(domainId); for (AccountVO account : accounts) { - long accountResourceCount = recalculateAccountResourceCount(account.getId(), type); + long accountResourceCount = recalculateAccountResourceCount(account.getId(), type, tag); newResourceCount += accountResourceCount; // add account's resource count to parent domain count } - _resourceCountDao.setResourceCount(domainId, ResourceOwnerType.Domain, type, newResourceCount); + _resourceCountDao.setResourceCount(domainId, ResourceOwnerType.Domain, type, tag, newResourceCount); if (oldResourceCount != newResourceCount) { s_logger.warn("Discrepency in the resource count has been detected " + "(original count = " + oldResourceCount + " correct count = " + newResourceCount + ") for Type = " + type @@ -952,13 +1192,12 @@ public Long doInTransaction(TransactionStatus status) { } @DB - protected long recalculateAccountResourceCount(final long accountId, final ResourceType type) { + protected long recalculateAccountResourceCount(final long accountId, final ResourceType type, String tag) { final Long newCount; if (type == Resource.ResourceType.user_vm) { - newCount = _userVmDao.countAllocatedVMsForAccount(accountId, VirtualMachineManager.ResourceCountRunningVMsonly.value()); + newCount = calculateVmCountForAccount(accountId, tag); } else if (type == Resource.ResourceType.volume) { - long virtualRouterCount = _vmDao.findIdsOfAllocatedVirtualRoutersForAccount(accountId).size(); - newCount = _volumeDao.countAllocatedVolumesForAccount(accountId) - virtualRouterCount; // don't count the volumes of virtual router + newCount = calculateVolumeCountForAccount(accountId, tag); } else if (type == Resource.ResourceType.snapshot) { newCount = _snapshotDao.countSnapshotsForAccount(accountId); } else if (type == Resource.ResourceType.public_ip) { @@ -972,12 +1211,11 @@ protected long recalculateAccountResourceCount(final long accountId, final Resou } else if (type == Resource.ResourceType.vpc) { newCount = _vpcDao.countByAccountId(accountId); } else if (type == Resource.ResourceType.cpu) { - newCount = countCpusForAccount(accountId); + newCount = calculateVmCpuCountForAccount(accountId, tag); } else if (type == Resource.ResourceType.memory) { - newCount = calculateMemoryForAccount(accountId); + newCount = calculateVmMemoryCountForAccount(accountId, tag); } else if (type == Resource.ResourceType.primary_storage) { - List virtualRouters = _vmDao.findIdsOfAllocatedVirtualRoutersForAccount(accountId); - newCount = _volumeDao.primaryStorageUsedForAccount(accountId, virtualRouters); + newCount = calculatePrimaryStorageForAccount(accountId, tag); } else if (type == Resource.ResourceType.secondary_storage) { newCount = calculateSecondaryStorageForAccount(accountId); } else { @@ -985,7 +1223,7 @@ protected long recalculateAccountResourceCount(final long accountId, final Resou } long oldCount = 0; - final ResourceCountVO accountRC = _resourceCountDao.findByOwnerAndType(accountId, ResourceOwnerType.Account, type); + final ResourceCountVO accountRC = _resourceCountDao.findByOwnerAndTypeAndTag(accountId, ResourceOwnerType.Account, type, tag); if (accountRC != null) { oldCount = accountRC.getCount(); } @@ -994,8 +1232,8 @@ protected long recalculateAccountResourceCount(final long accountId, final Resou Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(TransactionStatus status) { - lockAccountAndOwnerDomainRows(accountId, type); - _resourceCountDao.setResourceCount(accountId, ResourceOwnerType.Account, type, (newCount == null) ? 0 : newCount); + lockAccountAndOwnerDomainRows(accountId, type, tag); + _resourceCountDao.setResourceCount(accountId, ResourceOwnerType.Account, type, tag, (newCount == null) ? 0 : newCount); } }); } @@ -1011,50 +1249,94 @@ public void doInTransactionWithoutResult(TransactionStatus status) { return (newCount == null) ? 0 : newCount; } + protected List getVmsWithAccountAndTag(long accountId, String tag) { + List states = Arrays.asList(State.Destroyed, State.Error, State.Expunging); + if (VirtualMachineManager.ResourceCountRunningVMsonly.value()) { + states.add(State.Stopped); + } + if (StringUtils.isEmpty(tag)) { + return _userVmJoinDao.listByAccountServiceOfferingTemplateAndNotInState(accountId, states, null, null); + } + List offerings = serviceOfferingDao.listByHostTag(tag); + List templates = _vmTemplateDao.listByTemplateTag(tag); + if (CollectionUtils.isEmpty(offerings) && CollectionUtils.isEmpty(templates)) { + return new ArrayList<>(); + } + return _userVmJoinDao.listByAccountServiceOfferingTemplateAndNotInState(accountId, states, + offerings.stream().map(ServiceOfferingVO::getId).collect(Collectors.toList()), + templates.stream().map(VMTemplateVO::getId).collect(Collectors.toList())); + } + + protected List getVmsWithAccount(long accountId) { + return getVmsWithAccountAndTag(accountId, null); + } + + protected List getVolumesWithAccountAndTag(long accountId, String tag) { + List offerings = diskOfferingDao.listByStorageTag(tag); + if (CollectionUtils.isEmpty(offerings)) { + return new ArrayList<>(); + } + List vrIds = _vmDao.findIdsOfAllocatedVirtualRoutersForAccount(accountId); + return _volumeDao.listAllocatedVolumesForAccountDiskOfferingIdsAndNotForVms(accountId, + offerings.stream().map(DiskOfferingVO::getId).collect(Collectors.toList()), + vrIds); + } + + protected long calculateVmCountForAccount(long accountId, String tag) { + if (StringUtils.isEmpty(tag)) { + return _userVmDao.countAllocatedVMsForAccount(accountId, VirtualMachineManager.ResourceCountRunningVMsonly.value()); + } + List vms = getVmsWithAccountAndTag(accountId, tag); + return vms.size(); + } + + protected long calculateVolumeCountForAccount(long accountId, String tag) { + if (StringUtils.isEmpty(tag)) { + long virtualRouterCount = _vmDao.findIdsOfAllocatedVirtualRoutersForAccount(accountId).size(); + return _volumeDao.countAllocatedVolumesForAccount(accountId) - virtualRouterCount; // don't count the volumes of virtual router + } + List volumes = getVolumesWithAccountAndTag(accountId, tag); + return volumes.size(); + } + + protected long calculateVmCpuCountForAccount(long accountId, String tag) { + if (StringUtils.isEmpty(tag)) { + return countCpusForAccount(accountId); + } + long cputotal = 0; + List vms = getVmsWithAccountAndTag(accountId, tag); + for (UserVmJoinVO vm : vms) { + cputotal += vm.getCpu(); + } + return cputotal; + } + + protected long calculateVmMemoryCountForAccount(long accountId, String tag) { + if (StringUtils.isEmpty(tag)) { + return calculateMemoryForAccount(accountId); + } + long memory = 0; + List vms = getVmsWithAccountAndTag(accountId, tag); + for (UserVmJoinVO vm : vms) { + memory += vm.getRamSize(); + } + return memory; + } + public long countCpusForAccount(long accountId) { long cputotal = 0; - // user vms - SearchBuilder userVmSearch = _userVmJoinDao.createSearchBuilder(); - userVmSearch.and("accountId", userVmSearch.entity().getAccountId(), Op.EQ); - userVmSearch.and("state", userVmSearch.entity().getState(), SearchCriteria.Op.NIN); - userVmSearch.and("displayVm", userVmSearch.entity().isDisplayVm(), Op.EQ); - userVmSearch.groupBy(userVmSearch.entity().getId()); // select distinct - userVmSearch.done(); - - SearchCriteria sc1 = userVmSearch.create(); - sc1.setParameters("accountId", accountId); - if (VirtualMachineManager.ResourceCountRunningVMsonly.value()) - sc1.setParameters("state", new Object[] {State.Destroyed, State.Error, State.Expunging, State.Stopped}); - else - sc1.setParameters("state", new Object[] {State.Destroyed, State.Error, State.Expunging}); - sc1.setParameters("displayVm", 1); - List userVms = _userVmJoinDao.search(sc1,null); + List userVms = getVmsWithAccount(accountId); for (UserVmJoinVO vm : userVms) { - cputotal += Long.valueOf(vm.getCpu()); + cputotal += vm.getCpu(); } return cputotal; } public long calculateMemoryForAccount(long accountId) { long ramtotal = 0; - // user vms - SearchBuilder userVmSearch = _userVmJoinDao.createSearchBuilder(); - userVmSearch.and("accountId", userVmSearch.entity().getAccountId(), Op.EQ); - userVmSearch.and("state", userVmSearch.entity().getState(), SearchCriteria.Op.NIN); - userVmSearch.and("displayVm", userVmSearch.entity().isDisplayVm(), Op.EQ); - userVmSearch.groupBy(userVmSearch.entity().getId()); // select distinct - userVmSearch.done(); - - SearchCriteria sc1 = userVmSearch.create(); - sc1.setParameters("accountId", accountId); - if (VirtualMachineManager.ResourceCountRunningVMsonly.value()) - sc1.setParameters("state", new Object[] {State.Destroyed, State.Error, State.Expunging, State.Stopped}); - else - sc1.setParameters("state", new Object[] {State.Destroyed, State.Error, State.Expunging}); - sc1.setParameters("displayVm", 1); - List userVms = _userVmJoinDao.search(sc1,null); + List userVms = getVmsWithAccount(accountId); for (UserVmJoinVO vm : userVms) { - ramtotal += Long.valueOf(vm.getRamSize()); + ramtotal += vm.getRamSize(); } return ramtotal; } @@ -1101,9 +1383,22 @@ private long calculatePublicIpForAccount(long accountId) { } } + protected long calculatePrimaryStorageForAccount(long accountId, String tag) { + if (StringUtils.isEmpty(tag)) { + List virtualRouters = _vmDao.findIdsOfAllocatedVirtualRoutersForAccount(accountId); + return _volumeDao.primaryStorageUsedForAccount(accountId, virtualRouters); + } + long storage = 0; + List volumes = getVolumesWithAccountAndTag(accountId, tag); + for (VolumeVO volume : volumes) { + storage += volume.getSize() == null ? 0L : volume.getSize(); + } + return storage; + } + @Override - public long getResourceCount(Account account, ResourceType type) { - return _resourceCountDao.getResourceCount(account.getId(), ResourceOwnerType.Account, type); + public long getResourceCount(Account account, ResourceType type, String tag) { + return _resourceCountDao.getResourceCount(account.getId(), ResourceOwnerType.Account, type, tag); } private boolean isDisplayFlagOn(Boolean displayResource) { @@ -1154,11 +1449,365 @@ public void changeResourceCount(long accountId, ResourceType type, Boolean displ } @Override - public ResourceReservation getReservation(final Account account, final Boolean displayResource, final Resource.ResourceType type, final Long delta) throws ResourceAllocationException { - if (! Boolean.FALSE.equals(displayResource)) { - return new CheckedReservation(account, type, delta, reservationDao, resourceLimitService); + public List getResourceLimitHostTags() { + if (StringUtils.isEmpty(ResourceLimitService.ResourceLimitHostTags.value())) { + return new ArrayList<>(); + } + return Arrays.asList(ResourceLimitService.ResourceLimitHostTags.value().split(",")); + } + + @Override + public List getResourceLimitStorageTags() { + if (StringUtils.isEmpty(ResourceLimitService.ResourceLimitStorageTags.value())) { + return new ArrayList<>(); + } + return Arrays.asList(ResourceLimitService.ResourceLimitStorageTags.value().split(",")); + } + + protected TaggedResourceLimitAndCountResponse getTaggedResourceLimitAndCountResponse(Account account, + Domain domain, ResourceOwnerType ownerType, ResourceType type, String tag) { + Long limit = ResourceOwnerType.Account.equals(ownerType) ? + findCorrectResourceLimitForAccount(account, type, tag) : + findCorrectResourceLimitForDomain(domain, type, tag); + Long count = 0L; + ResourceCountVO countVO = _resourceCountDao.findByOwnerAndTypeAndTag( + ResourceOwnerType.Account.equals(ownerType) ? account.getId() : domain.getId(), ownerType, type, tag); + if (countVO != null) { + count = countVO.getCount(); + } + TaggedResourceLimitAndCountResponse taggedResourceLimitAndCountResponse = new TaggedResourceLimitAndCountResponse(); + taggedResourceLimitAndCountResponse.setResourceType(type); + taggedResourceLimitAndCountResponse.setTag(tag); + taggedResourceLimitAndCountResponse.setLimit(limit); + taggedResourceLimitAndCountResponse.setTotal(count); + taggedResourceLimitAndCountResponse.setAvailable(limit == Resource.RESOURCE_UNLIMITED ? Resource.RESOURCE_UNLIMITED : (limit - count)); + return taggedResourceLimitAndCountResponse; + } + + protected void updateTaggedResourceLimitsAndCounts(String uuid, ResourceOwnerType ownerType, List hostTags, + List storageTags, ResourceLimitAndCountResponse response) { + Account account = null; + if (ResourceOwnerType.Account.equals(ownerType)) { + account = _accountDao.findByUuid(uuid); + } + Domain domain = null; + if (ResourceOwnerType.Domain.equals(ownerType)) { + domain = _domainDao.findByUuid(uuid); + } + List taggedResponses = new ArrayList<>(); + for (String tag : hostTags) { + for (ResourceType type : HostTagsSupportingTypes) { + taggedResponses.add(getTaggedResourceLimitAndCountResponse(account, domain, ownerType, type, tag)); + } + } + for (String tag : storageTags) { + for (ResourceType type : StorageTagsSupportingTypes) { + taggedResponses.add(getTaggedResourceLimitAndCountResponse(account, domain, ownerType, type, tag)); + } + } + response.setTaggedResourceLimitsAndCounts(taggedResponses); + } + + protected void updateTaggedResourceLimitsAndCountsForAccountsOrDomains(List accountResponses, List domainResponses, String tag) { + List hostTags = new ArrayList<>(getResourceLimitHostTags()); + List storageTags = new ArrayList<>(getResourceLimitStorageTags()); + if (StringUtils.isNotEmpty(tag)) { + hostTags.retainAll(List.of(tag)); + storageTags.retainAll(List.of(tag)); + } + if (CollectionUtils.isEmpty(hostTags) && CollectionUtils.isEmpty(storageTags)) { + return; + } + if (CollectionUtils.isNotEmpty(accountResponses)) { + for (AccountResponse response : accountResponses) { + updateTaggedResourceLimitsAndCounts(response.getObjectId(), ResourceOwnerType.Account, hostTags, storageTags, response); + } + } + if (CollectionUtils.isNotEmpty(domainResponses)) { + for (DomainResponse response : domainResponses) { + updateTaggedResourceLimitsAndCounts(response.getId(), ResourceOwnerType.Domain, hostTags, storageTags, response); + } + } + } + + @Override + public void updateTaggedResourceLimitsAndCountsForAccounts(List responses, String tag) { + updateTaggedResourceLimitsAndCountsForAccountsOrDomains(responses, null, tag); + } + + @Override + public void updateTaggedResourceLimitsAndCountsForDomains(List responses, String tag) { + updateTaggedResourceLimitsAndCountsForAccountsOrDomains(null, responses, tag); + } + + @Override + public List getResourceLimitHostTags(ServiceOffering serviceOffering, VirtualMachineTemplate template) { + if (StringUtils.isEmpty(serviceOffering.getHostTag()) && StringUtils.isEmpty(template.getTemplateTag())) { + return new ArrayList<>(); + } + List resourceLimitTagsFromConfig = getResourceLimitHostTags(); + if (CollectionUtils.isEmpty(resourceLimitTagsFromConfig)) { + return new ArrayList<>(); + } + List tags = new ArrayList<>(); + if (StringUtils.isNotEmpty(serviceOffering.getHostTag())) { + List offeringTags = com.cloud.utils.StringUtils.csvTagsToList(serviceOffering.getHostTag()); + for (String tag : offeringTags) { + if (StringUtils.isNotEmpty(tag) && resourceLimitTagsFromConfig.contains(tag)) { + tags.add(tag); + } + } + } + if (StringUtils.isNotEmpty(template.getTemplateTag()) + && resourceLimitTagsFromConfig.contains(template.getTemplateTag()) + && !tags.contains(template.getTemplateTag())) { + tags.add(template.getTemplateTag()); + } + return tags; + } + + @Override + public List getResourceLimitStorageTags(DiskOffering diskOffering) { + if (diskOffering == null || StringUtils.isEmpty(diskOffering.getTags())) { + return new ArrayList<>(); + } + List resourceLimitTagsFromConfig = getResourceLimitStorageTags(); + if (CollectionUtils.isEmpty(resourceLimitTagsFromConfig)) { + return new ArrayList<>(); + } + String[] offeringTags = diskOffering.getTagsArray(); + List tags = new ArrayList<>(); + for (String tag : offeringTags) { + if (StringUtils.isNotEmpty(tag) && resourceLimitTagsFromConfig.contains(tag)) { + tags.add(tag); + } + } + return tags; + } + + protected List getResourceLimitStorageTagsForResourceCountOperation(Boolean display, DiskOffering diskOffering) { + if (Boolean.FALSE.equals(display)) { + return new ArrayList<>(); + } + List tags = getResourceLimitStorageTags(diskOffering); + if (tags.isEmpty()) { + tags.add(null); + } else { + tags.add(0, null); + } + return tags; + } + + @Override + public void checkVolumeResourceLimit(Account owner, Boolean display, Long size, DiskOffering diskOffering) throws ResourceAllocationException { + List tags = getResourceLimitStorageTagsForResourceCountOperation(display, diskOffering); + if (CollectionUtils.isEmpty(tags)) { + return; + } + for (String tag : tags) { + checkResourceLimitWithTag(owner, ResourceType.volume, tag); + if (size != null) { + checkResourceLimitWithTag(owner, ResourceType.primary_storage, tag, size); + } + } + } + + @Override + public void incrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering) { + List tags = getResourceLimitStorageTagsForResourceCountOperation(display, diskOffering); + if (CollectionUtils.isEmpty(tags)) { + return; + } + for (String tag : tags) { + incrementResourceCountWithTag(accountId, ResourceType.volume, tag); + if (size != null) { + incrementResourceCountWithTag(accountId, ResourceType.primary_storage, tag, size); + } + } + } + + @Override + public void decrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering) { + List tags = getResourceLimitStorageTagsForResourceCountOperation(display, diskOffering); + if (CollectionUtils.isEmpty(tags)) { + return; + } + for (String tag : tags) { + decrementResourceCountWithTag(accountId, ResourceType.volume, tag); + if (size != null) { + decrementResourceCountWithTag(accountId, ResourceType.primary_storage, tag, size); + } + } + } + + @Override + public void incrementVolumePrimaryStorageResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering) { + if (size == null) { + return; + } + List tags = getResourceLimitStorageTagsForResourceCountOperation(display, diskOffering); + if (CollectionUtils.isEmpty(tags)) { + return; + } + for (String tag : tags) { + incrementResourceCountWithTag(accountId, ResourceType.primary_storage, tag, size); + } + } + + @Override + public void decrementVolumePrimaryStorageResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering) { + if (size == null) { + return; + } + List tags = getResourceLimitStorageTagsForResourceCountOperation(display, diskOffering); + if (CollectionUtils.isEmpty(tags)) { + return; + } + for (String tag : tags) { + decrementResourceCountWithTag(accountId, ResourceType.primary_storage, tag, size); + } + } + + protected List getResourceLimitHostTagsForResourceCountOperation(Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template) { + if (Boolean.FALSE.equals(display)) { + return new ArrayList<>(); + } + List tags = getResourceLimitHostTags(serviceOffering, template); + if (tags.isEmpty()) { + tags.add(null); + } else { + tags.add(0, null); + } + return tags; + } + + @Override + public void checkVmResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template) throws ResourceAllocationException { + List tags = getResourceLimitHostTagsForResourceCountOperation(display, serviceOffering, template); + if (CollectionUtils.isEmpty(tags)) { + return; + } + Long cpu = serviceOffering.getCpu() != null ? Long.valueOf(serviceOffering.getCpu()) : 0L; + Long ram = serviceOffering.getRamSize() != null ? Long.valueOf(serviceOffering.getRamSize()) : 0L; + for (String tag : tags) { + checkResourceLimitWithTag(owner, ResourceType.user_vm, tag); + checkResourceLimitWithTag(owner, ResourceType.cpu, tag, cpu); + checkResourceLimitWithTag(owner, ResourceType.memory, tag, ram); + } + } + + @Override + public void incrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template) { + List tags = getResourceLimitHostTagsForResourceCountOperation(display, serviceOffering, template); + if (CollectionUtils.isEmpty(tags)) { + return; + } + Long cpu = serviceOffering.getCpu() != null ? Long.valueOf(serviceOffering.getCpu()) : 0L; + Long ram = serviceOffering.getRamSize() != null ? Long.valueOf(serviceOffering.getRamSize()) : 0L; + for (String tag : tags) { + incrementResourceCountWithTag(accountId, ResourceType.user_vm, tag); + incrementResourceCountWithTag(accountId, ResourceType.cpu, tag, cpu); + incrementResourceCountWithTag(accountId, ResourceType.memory, tag, ram); + } + } + + @Override + public void decrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template) { + List tags = getResourceLimitHostTagsForResourceCountOperation(display, serviceOffering, template); + if (CollectionUtils.isEmpty(tags)) { + return; + } + Long cpu = serviceOffering.getCpu() != null ? Long.valueOf(serviceOffering.getCpu()) : 0L; + Long ram = serviceOffering.getRamSize() != null ? Long.valueOf(serviceOffering.getRamSize()) : 0L; + for (String tag : tags) { + decrementResourceCountWithTag(accountId, ResourceType.user_vm, tag); + decrementResourceCountWithTag(accountId, ResourceType.cpu, tag, cpu); + decrementResourceCountWithTag(accountId, ResourceType.memory, tag, ram); + } + } + + @Override + public void checkVmCpuResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long cpu) throws ResourceAllocationException { + List tags = getResourceLimitHostTagsForResourceCountOperation(display, serviceOffering, template); + if (CollectionUtils.isEmpty(tags)) { + return; + } + if (cpu == null) { + cpu = serviceOffering.getCpu() != null ? Long.valueOf(serviceOffering.getCpu()) : 0L; + } + for (String tag : tags) { + checkResourceLimitWithTag(owner, ResourceType.cpu, tag, cpu); + } + } + + @Override + public void incrementVmCpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long cpu) { + List tags = getResourceLimitHostTagsForResourceCountOperation(display, serviceOffering, template); + if (CollectionUtils.isEmpty(tags)) { + return; + } + if (cpu == null) { + cpu = serviceOffering.getCpu() != null ? Long.valueOf(serviceOffering.getCpu()) : 0L; + } + for (String tag : tags) { + incrementResourceCountWithTag(accountId, ResourceType.cpu, tag, cpu); + } + } + + @Override + public void decrementVmCpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long cpu) { + List tags = getResourceLimitHostTagsForResourceCountOperation(display, serviceOffering, template); + if (CollectionUtils.isEmpty(tags)) { + return; + } + if (cpu == null) { + cpu = serviceOffering.getCpu() != null ? Long.valueOf(serviceOffering.getCpu()) : 0L; + } + for (String tag : tags) { + decrementResourceCountWithTag(accountId, ResourceType.cpu, tag, cpu); + } + } + + @Override + public void checkVmMemoryResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory) throws ResourceAllocationException { + List tags = getResourceLimitHostTagsForResourceCountOperation(display, serviceOffering, template); + if (CollectionUtils.isEmpty(tags)) { + return; + } + if (memory == null) { + memory = serviceOffering.getRamSize() != null ? Long.valueOf(serviceOffering.getRamSize()) : 0L; + } + for (String tag : tags) { + checkResourceLimitWithTag(owner, ResourceType.memory, tag, memory); + } + } + + @Override + public void incrementVmMemoryResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory) { + List tags = getResourceLimitHostTagsForResourceCountOperation(display, serviceOffering, template); + if (CollectionUtils.isEmpty(tags)) { + return; + } + if (memory == null) { + memory = serviceOffering.getRamSize() != null ? Long.valueOf(serviceOffering.getRamSize()) : 0L; + } + for (String tag : tags) { + incrementResourceCountWithTag(accountId, ResourceType.memory, tag, memory); + } + } + + @Override + public void decrementVmMemoryResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory) { + List tags = getResourceLimitHostTagsForResourceCountOperation(display, serviceOffering, template); + if (CollectionUtils.isEmpty(tags)) { + return; + } + if (memory == null) { + memory = serviceOffering.getRamSize() != null ? Long.valueOf(serviceOffering.getRamSize()) : 0L; + } + for (String tag : tags) { + decrementResourceCountWithTag(accountId, ResourceType.memory, tag, memory); } - throw new CloudRuntimeException("no reservation needed for resources that display as false"); } @Override @@ -1168,7 +1817,13 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[] {ResourceCountCheckInterval, MaxAccountSecondaryStorage, MaxProjectSecondaryStorage}; + return new ConfigKey[] { + ResourceCountCheckInterval, + MaxAccountSecondaryStorage, + MaxProjectSecondaryStorage, + ResourceLimitHostTags, + ResourceLimitStorageTags + }; } protected class ResourceCountCheckTask extends ManagedContextRunnable { @@ -1220,34 +1875,21 @@ private void runResourceCheckTaskInternal() { for (ResourceType type : ResourceType.values()) { if (CollectionUtils.isEmpty(domains)) { - recalculateDomainResourceCountInContext(Domain.ROOT_DOMAIN, type); + recalculateDomainResourceCount(Domain.ROOT_DOMAIN, type, null); + recalculateDomainTaggedResourceCount(Domain.ROOT_DOMAIN, type, getResourceLimitHostTags(), getResourceLimitStorageTags()); } else { for (Domain domain : domains) { - recalculateDomainResourceCount(domain.getId(), type); + recalculateDomainResourceCount(domain.getId(), type, null); + recalculateDomainTaggedResourceCount(domain.getId(), type, getResourceLimitHostTags(), getResourceLimitStorageTags()); } } - // run through the accounts in the root domain for (AccountVO account : accounts) { - recalculateAccountResourceCountInContext(account.getId(), type); + recalculateAccountResourceCount(account.getId(), type, null); + recalculateAccountTaggedResourceCount(account.getId(), type, getResourceLimitHostTags(), getResourceLimitStorageTags()); } } s_logger.info("Finished resource counters recalculation periodic task."); } - - private void recalculateDomainResourceCountInContext(long domainId, ResourceType type) { - try { - recalculateDomainResourceCount(domainId, type); - } catch (Exception e) { - s_logger.warn("Resource counters recalculation periodic task failed for the domain " + domainId + " and the resource type " + type + " .", e); - } - } - private void recalculateAccountResourceCountInContext(long accountId, ResourceType type) { - try { - recalculateAccountResourceCount(accountId, type); - } catch (Exception e) { - s_logger.warn("Resource counters recalculation periodic task failed for the account " + accountId + " and the resource type " + type + " .", e); - } - } } } diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index f794736a4d56..87d1a21b1be0 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -43,8 +43,6 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.network.dao.PublicIpQuarantineDao; -import com.cloud.hypervisor.HypervisorGuru; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.affinity.AffinityGroupProcessor; @@ -225,8 +223,8 @@ import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListStorageProvidersCmd; import org.apache.cloudstack.api.command.admin.storage.ListStorageTagsCmd; -import org.apache.cloudstack.api.command.admin.storage.MigrateSecondaryStorageDataCmd; import org.apache.cloudstack.api.command.admin.storage.MigrateResourcesToAnotherSecondaryStorageCmd; +import org.apache.cloudstack.api.command.admin.storage.MigrateSecondaryStorageDataCmd; import org.apache.cloudstack.api.command.admin.storage.PreparePrimaryStorageForMaintenanceCmd; import org.apache.cloudstack.api.command.admin.storage.SyncStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateCloudToUseObjectStoreCmd; @@ -716,6 +714,7 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.HypervisorCapabilities; import com.cloud.hypervisor.HypervisorCapabilitiesVO; +import com.cloud.hypervisor.HypervisorGuru; import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao; import com.cloud.hypervisor.kvm.dpdk.DpdkHelper; import com.cloud.info.ConsoleProxyInfo; @@ -735,6 +734,7 @@ import com.cloud.network.dao.NetworkDomainDao; import com.cloud.network.dao.NetworkDomainVO; import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.PublicIpQuarantineDao; import com.cloud.network.vpc.dao.VpcDao; import com.cloud.org.Cluster; import com.cloud.org.Grouping.AllocationState; @@ -766,6 +766,7 @@ import com.cloud.storage.dao.GuestOSCategoryDao; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.GuestOSHypervisorDao; +import com.cloud.storage.dao.StoragePoolTagsDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.secondary.SecondaryStorageVmManager; @@ -995,6 +996,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe protected AnnotationDao annotationDao; @Inject UserDataManager userDataManager; + @Inject + StoragePoolTagsDao storagePoolTagsDao; @Inject private PublicIpQuarantineDao publicIpQuarantineDao; @@ -3199,6 +3202,35 @@ public boolean deleteAlerts(final DeleteAlertsCmd cmd) { return result; } + Pair> getHostIdsForCapacityListing(Long zoneId, Long podId, Long clusterId, Integer capacityType, String tag) { + if (StringUtils.isEmpty(tag)) { + return new Pair<>(true, null); + } + Short type = capacityType == null ? null : capacityType.shortValue(); + if (type != null && Capacity.STORAGE_CAPACITY_TYPES.contains(type)) { + return new Pair<>(false, null); + } + List hostIds = null; + try { + List hosts = _hostDao.listByHostTag(Type.Routing, clusterId, podId, zoneId, tag); + hostIds = hosts.stream().map(HostVO::getId).collect(Collectors.toList()); + } catch (CloudRuntimeException ignored) { + } + return new Pair<>(CollectionUtils.isNotEmpty(hostIds), hostIds); + } + + Pair> getStoragePoolIdsForCapacityListing(Integer capacityType, String tag) { + if (StringUtils.isEmpty(tag)) { + return new Pair<>(true, null); + } + Short type = capacityType == null ? null : capacityType.shortValue(); + if (type != null && !Capacity.STORAGE_CAPACITY_TYPES.contains(type)) { + return new Pair<>(false, null); + } + List storagePoolIds = storagePoolTagsDao.listPoolIdsByTag(tag); + return new Pair<>(CollectionUtils.isNotEmpty(storagePoolIds), storagePoolIds); + } + @Override public List listTopConsumedResources(final ListCapacityCmd cmd) { @@ -3207,9 +3239,10 @@ public List listTopConsumedResources(final ListCapacityCmd cmd) { final Long podId = cmd.getPodId(); final Long clusterId = cmd.getClusterId(); final Boolean fetchLatest = cmd.getFetchLatest(); + final String tag = cmd.getTag(); if (clusterId != null) { - throw new InvalidParameterValueException("Currently clusterId param is not suppoerted"); + throw new InvalidParameterValueException("Currently clusterId param is not supported"); } zoneId = _accountMgr.checkAccessAndSpecifyAuthority(CallContext.current().getCallingAccount(), zoneId); @@ -3218,42 +3251,48 @@ public List listTopConsumedResources(final ListCapacityCmd cmd) { } List summedCapacities = new ArrayList(); + int level = 3; if (zoneId == null && podId == null) {// Group by Zone, capacity type - final List summedCapacitiesAtZone = _capacityDao.listCapacitiesGroupedByLevelAndType(capacityType, zoneId, podId, clusterId, 1, cmd.getPageSizeVal()); - if (summedCapacitiesAtZone != null) { - summedCapacities.addAll(summedCapacitiesAtZone); - } + level = 1; } else if (podId == null) {// Group by Pod, capacity type - final List summedCapacitiesAtPod = _capacityDao.listCapacitiesGroupedByLevelAndType(capacityType, zoneId, podId, clusterId, 2, cmd.getPageSizeVal()); - if (summedCapacitiesAtPod != null) { - summedCapacities.addAll(summedCapacitiesAtPod); - } - } else { // Group by Cluster, capacity type - final List summedCapacitiesAtCluster = _capacityDao.listCapacitiesGroupedByLevelAndType(capacityType, zoneId, podId, clusterId, 3, cmd.getPageSizeVal()); - if (summedCapacitiesAtCluster != null) { - summedCapacities.addAll(summedCapacitiesAtCluster); + level = 2; + } + + final List capacities = new ArrayList<>(); + Pair> hostIdsForCapacity = getHostIdsForCapacityListing(zoneId, podId, clusterId, capacityType, tag); + Pair> storagePoolIdsForCapacity = getStoragePoolIdsForCapacityListing(capacityType, tag); + if (StringUtils.isNotEmpty(tag) && !hostIdsForCapacity.first() && !storagePoolIdsForCapacity.first()) { + return capacities; + } + + if (hostIdsForCapacity.first() || storagePoolIdsForCapacity.first()) { + final List summedHostCapacities = _capacityDao.listCapacitiesGroupedByLevelAndType( + capacityType, zoneId, podId, clusterId, level, hostIdsForCapacity.second(), + storagePoolIdsForCapacity.second(), cmd.getPageSizeVal()); + if (summedHostCapacities != null) { + summedCapacities.addAll(summedHostCapacities); } } - List summedCapacitiesForSecStorage = getStorageUsed(clusterId, podId, zoneId, capacityType); - if (summedCapacitiesForSecStorage != null) { - summedCapacities.addAll(summedCapacitiesForSecStorage); + + if (storagePoolIdsForCapacity.first()) { + List summedStorageCapacities = getStorageCapacities(clusterId, podId, zoneId, + storagePoolIdsForCapacity.second(), capacityType == null ? null : capacityType.shortValue()); + if (summedStorageCapacities != null) { + summedCapacities.addAll(summedStorageCapacities); + } } // Sort Capacities - Collections.sort(summedCapacities, new Comparator() { - @Override - public int compare(final SummedCapacity arg0, final SummedCapacity arg1) { - if (arg0.getPercentUsed() < arg1.getPercentUsed()) { - return 1; - } else if (arg0.getPercentUsed().equals(arg1.getPercentUsed())) { - return 0; - } - return -1; + summedCapacities.sort((arg0, arg1) -> { + if (arg0.getPercentUsed() < arg1.getPercentUsed()) { + return 1; + } else if (arg0.getPercentUsed().equals(arg1.getPercentUsed())) { + return 0; } + return -1; }); - final List capacities = new ArrayList(); Integer pageSize = null; try { @@ -3273,48 +3312,53 @@ public int compare(final SummedCapacity arg0, final SummedCapacity arg1) { return capacities; } - List getStorageUsed(Long clusterId, Long podId, Long zoneId, Integer capacityType) { - if (capacityType == null || capacityType == Capacity.CAPACITY_TYPE_SECONDARY_STORAGE) { - final List list = new ArrayList(); - if (zoneId != null) { - final DataCenterVO zone = ApiDBUtils.findZoneById(zoneId); - if (zone == null || zone.getAllocationState() == AllocationState.Disabled) { - return null; - } - List capacities = new ArrayList(); - capacities.add(_storageMgr.getSecondaryStorageUsedStats(null, zoneId)); - capacities.add(_storageMgr.getStoragePoolUsedStats(null, clusterId, podId, zoneId)); - for (CapacityVO capacity : capacities) { - if (capacity.getTotalCapacity() != 0) { - capacity.setUsedPercentage((float)capacity.getUsedCapacity() / capacity.getTotalCapacity()); - } else { - capacity.setUsedPercentage(0); - } - final SummedCapacity summedCapacity = new SummedCapacity(capacity.getUsedCapacity(), capacity.getTotalCapacity(), capacity.getUsedPercentage(), capacity.getCapacityType(), - capacity.getDataCenterId(), capacity.getPodId(), capacity.getClusterId()); - list.add(summedCapacity); + List getStorageCapacities(Long clusterId, Long podId, Long zoneId, List poolIds, Short capacityType) { + List capacityTypes = Arrays.asList(Capacity.CAPACITY_TYPE_STORAGE, Capacity.CAPACITY_TYPE_SECONDARY_STORAGE); + if (capacityType != null && !capacityTypes.contains(capacityType)) { + return null; + } + if (capacityType != null) { + capacityTypes = capacityTypes.stream().filter(x -> x.equals(capacityType)).collect(Collectors.toList()); + } + if (CollectionUtils.isNotEmpty(poolIds)) { + capacityTypes = capacityTypes.stream().filter(x -> x != Capacity.CAPACITY_TYPE_SECONDARY_STORAGE).collect(Collectors.toList()); + } + if (CollectionUtils.isEmpty(capacityTypes)) { + return null; + } + final List list = new ArrayList<>(); + List dcList = new ArrayList<>(); + if (zoneId != null) { + final DataCenterVO zone = ApiDBUtils.findZoneById(zoneId); + if (zone == null || zone.getAllocationState() == AllocationState.Disabled) { + return null; + } + dcList.add(zone); + } else { + dcList = _dcDao.listEnabledZones(); + podId = null; + clusterId = null; + } + for (DataCenterVO dc : dcList) { + List capacities = new ArrayList<>(); + if (capacityTypes.contains(Capacity.CAPACITY_TYPE_SECONDARY_STORAGE)) { + capacities.add(_storageMgr.getSecondaryStorageUsedStats(null, dc.getId())); + } + if (capacityTypes.contains(Capacity.CAPACITY_TYPE_STORAGE)) { + capacities.add(_storageMgr.getStoragePoolUsedStats(dc.getId(), podId, clusterId, poolIds)); + } + for (CapacityVO capacity : capacities) { + if (capacity.getTotalCapacity() != 0) { + capacity.setUsedPercentage((float)capacity.getUsedCapacity() / capacity.getTotalCapacity()); + } else { + capacity.setUsedPercentage(0); } - } else { - List dcList = _dcDao.listEnabledZones(); - for (DataCenterVO dc : dcList) { - List capacities = new ArrayList(); - capacities.add(_storageMgr.getSecondaryStorageUsedStats(null, dc.getId())); - capacities.add(_storageMgr.getStoragePoolUsedStats(null, null, null, dc.getId())); - for (CapacityVO capacity : capacities) { - if (capacity.getTotalCapacity() != 0) { - capacity.setUsedPercentage((float)capacity.getUsedCapacity() / capacity.getTotalCapacity()); - } else { - capacity.setUsedPercentage(0); - } - SummedCapacity summedCapacity = new SummedCapacity(capacity.getUsedCapacity(), capacity.getTotalCapacity(), capacity.getUsedPercentage(), capacity.getCapacityType(), - capacity.getDataCenterId(), capacity.getPodId(), capacity.getClusterId()); - list.add(summedCapacity); - } - }// End of for + SummedCapacity summedCapacity = new SummedCapacity(capacity.getUsedCapacity(), capacity.getTotalCapacity(), capacity.getUsedPercentage(), capacity.getCapacityType(), + capacity.getDataCenterId(), capacity.getPodId(), capacity.getClusterId()); + list.add(summedCapacity); } - return list; - } - return null; + }// End of for + return list; } @Override @@ -3325,26 +3369,34 @@ public List listCapacities(final ListCapacityCmd cmd) { final Long podId = cmd.getPodId(); final Long clusterId = cmd.getClusterId(); final Boolean fetchLatest = cmd.getFetchLatest(); + final String tag = cmd.getTag(); zoneId = _accountMgr.checkAccessAndSpecifyAuthority(CallContext.current().getCallingAccount(), zoneId); if (fetchLatest != null && fetchLatest) { _alertMgr.recalculateCapacity(); } - - final List summedCapacities = _capacityDao.findCapacityBy(capacityType, zoneId, podId, clusterId); - final List capacities = new ArrayList(); - - for (final SummedCapacity summedCapacity : summedCapacities) { - final CapacityVO capacity = new CapacityVO(null, summedCapacity.getDataCenterId(), summedCapacity.getPodId(), summedCapacity.getClusterId(), - summedCapacity.getUsedCapacity() + summedCapacity.getReservedCapacity(), summedCapacity.getTotalCapacity(), summedCapacity.getCapacityType()); - capacity.setAllocatedCapacity(summedCapacity.getAllocatedCapacity()); - capacities.add(capacity); + final List capacities = new ArrayList<>(); + Pair> hostIdsForCapacity = getHostIdsForCapacityListing(zoneId, podId, clusterId, capacityType, tag); + Pair> storagePoolIdsForCapacity = getStoragePoolIdsForCapacityListing(capacityType, tag); + if (StringUtils.isNotEmpty(tag) && !hostIdsForCapacity.first() && !storagePoolIdsForCapacity.first()) { + return capacities; + } + if (hostIdsForCapacity.first() || storagePoolIdsForCapacity.first()) { + final List summedCapacities = _capacityDao.findFilteredCapacityBy(capacityType, + zoneId, podId, clusterId, hostIdsForCapacity.second(), storagePoolIdsForCapacity.second()); + + for (final SummedCapacity summedCapacity : summedCapacities) { + final CapacityVO capacity = new CapacityVO(null, summedCapacity.getDataCenterId(), summedCapacity.getPodId(), summedCapacity.getClusterId(), + summedCapacity.getUsedCapacity() + summedCapacity.getReservedCapacity(), summedCapacity.getTotalCapacity(), summedCapacity.getCapacityType()); + capacity.setAllocatedCapacity(summedCapacity.getAllocatedCapacity()); + capacities.add(capacity); + } } // op_host_Capacity contains only allocated stats and the real time // stats are stored "in memory". // Show Sec. Storage only when the api is invoked for the zone layer. - List dcList = new ArrayList(); + List dcList = new ArrayList<>(); if (zoneId == null && podId == null && clusterId == null) { dcList = ApiDBUtils.listZones(); } else if (zoneId != null) { @@ -3355,18 +3407,18 @@ public List listCapacities(final ListCapacityCmd cmd) { } else { zoneId = ApiDBUtils.findPodById(podId).getDataCenterId(); } - if (capacityType == null || capacityType == Capacity.CAPACITY_TYPE_STORAGE) { - capacities.add(_storageMgr.getStoragePoolUsedStats(null, clusterId, podId, zoneId)); + if ((capacityType == null || capacityType == Capacity.CAPACITY_TYPE_STORAGE) && storagePoolIdsForCapacity.first()) { + capacities.add(_storageMgr.getStoragePoolUsedStats(zoneId, podId, clusterId, storagePoolIdsForCapacity.second())); } } for (final DataCenterVO zone : dcList) { zoneId = zone.getId(); - if ((capacityType == null || capacityType == Capacity.CAPACITY_TYPE_SECONDARY_STORAGE) && podId == null && clusterId == null) { + if ((capacityType == null || capacityType == Capacity.CAPACITY_TYPE_SECONDARY_STORAGE) && podId == null && clusterId == null && StringUtils.isEmpty(tag)) { capacities.add(_storageMgr.getSecondaryStorageUsedStats(null, zoneId)); } - if (capacityType == null || capacityType == Capacity.CAPACITY_TYPE_STORAGE) { - capacities.add(_storageMgr.getStoragePoolUsedStats(null, clusterId, podId, zoneId)); + if ((capacityType == null || capacityType == Capacity.CAPACITY_TYPE_STORAGE) && storagePoolIdsForCapacity.first()) { + capacities.add(_storageMgr.getStoragePoolUsedStats(zoneId, podId, clusterId, storagePoolIdsForCapacity.second())); } } return capacities; diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 04889d0b2a83..0fabf3a2bc27 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -1602,7 +1602,8 @@ public void cleanupStorage(boolean recurring) { if (_serverId == host.getManagementServerId().longValue()) { volService.destroyVolume(volume.getId()); // decrement volume resource count - _resourceLimitMgr.decrementResourceCount(volume.getAccountId(), ResourceType.volume, volume.isDisplayVolume()); + _resourceLimitMgr.decrementVolumeResourceCount(volume.getAccountId(), volume.isDisplayVolume(), + null, _diskOfferingDao.findByIdIncludingRemoved(volume.getDiskOfferingId())); // expunge volume from secondary if volume is on image store VolumeInfo volOnSecondary = volFactory.getVolume(volume.getId(), DataStoreRole.Image); if (volOnSecondary != null) { @@ -2416,25 +2417,24 @@ public CapacityVO getSecondaryStorageUsedStats(Long hostId, Long zoneId) { return capacity; } - @Override - public CapacityVO getStoragePoolUsedStats(Long poolId, Long clusterId, Long podId, Long zoneId) { + private CapacityVO getStoragePoolUsedStatsInternal(Long zoneId, Long podId, Long clusterId, List poolIds, Long poolId) { SearchCriteria sc = _storagePoolDao.createSearchCriteria(); - List pools = new ArrayList(); + List pools = new ArrayList<>(); if (zoneId != null) { sc.addAnd("dataCenterId", SearchCriteria.Op.EQ, zoneId); } - if (podId != null) { sc.addAnd("podId", SearchCriteria.Op.EQ, podId); } - if (clusterId != null) { sc.addAnd("clusterId", SearchCriteria.Op.EQ, clusterId); } - + if (CollectionUtils.isNotEmpty(poolIds)) { + sc.addAnd("id", SearchCriteria.Op.IN, poolIds.toArray()); + } if (poolId != null) { - sc.addAnd("hostOrPoolId", SearchCriteria.Op.EQ, poolId); + sc.addAnd("id", SearchCriteria.Op.EQ, poolId); } sc.addAnd("parent", SearchCriteria.Op.EQ, 0L); if (poolId != null) { @@ -2444,8 +2444,8 @@ public CapacityVO getStoragePoolUsedStats(Long poolId, Long clusterId, Long podI } CapacityVO capacity = new CapacityVO(poolId, zoneId, podId, clusterId, 0, 0, Capacity.CAPACITY_TYPE_STORAGE); - for (StoragePoolVO PrimaryDataStoreVO : pools) { - StorageStats stats = ApiDBUtils.getStoragePoolStatistics(PrimaryDataStoreVO.getId()); + for (StoragePoolVO pool : pools) { + StorageStats stats = ApiDBUtils.getStoragePoolStatistics(pool.getId()); if (stats == null) { continue; } @@ -2453,6 +2453,17 @@ public CapacityVO getStoragePoolUsedStats(Long poolId, Long clusterId, Long podI capacity.setTotalCapacity(stats.getCapacityBytes() + capacity.getTotalCapacity()); } return capacity; + + } + + @Override + public CapacityVO getStoragePoolUsedStats(Long poolId, Long clusterId, Long podId, Long zoneId) { + return getStoragePoolUsedStatsInternal(zoneId, podId, clusterId, null, poolId); + } + + @Override + public CapacityVO getStoragePoolUsedStats(Long zoneId, Long podId, Long clusterId, List poolIds) { + return getStoragePoolUsedStatsInternal(zoneId, podId, clusterId, poolIds, null); } @Override @@ -2665,13 +2676,7 @@ private long getUsedSize(StoragePool pool) { return 0; } - @Override - public boolean storagePoolHasEnoughIops(List> requestedVolumes, StoragePool pool) { - if (requestedVolumes == null || requestedVolumes.isEmpty() || pool == null) { - s_logger.debug(String.format("Cannot check if storage [%s] has enough IOPS to allocate volumes [%s].", pool, requestedVolumes)); - return false; - } - + protected boolean checkIfPoolIopsCapacityNull(StoragePool pool) { // Only IOPS-guaranteed primary storage like SolidFire is using/setting IOPS. // This check returns true for storage that does not specify IOPS. if (pool.getCapacityIops() == null) { @@ -2679,12 +2684,29 @@ public boolean storagePoolHasEnoughIops(List> requeste return true; } + return false; + } + protected boolean storagePoolHasEnoughIops(long requestedIops, StoragePool pool, boolean skipPoolCheck) { + if (!skipPoolCheck && checkIfPoolIopsCapacityNull(pool)) { + return true; + } StoragePoolVO storagePoolVo = _storagePoolDao.findById(pool.getId()); long currentIops = _capacityMgr.getUsedIops(storagePoolVo); + long futureIops = currentIops + requestedIops; + return futureIops <= pool.getCapacityIops(); + } + @Override + public boolean storagePoolHasEnoughIops(List> requestedVolumes, StoragePool pool) { + if (requestedVolumes == null || requestedVolumes.isEmpty() || pool == null) { + s_logger.debug(String.format("Cannot check if storage [%s] has enough IOPS to allocate volumes [%s].", pool, requestedVolumes)); + return false; + } + if (checkIfPoolIopsCapacityNull(pool)) { + return true; + } long requestedIops = 0; - for (Pair volumeDiskProfilePair : requestedVolumes) { Volume requestedVolume = volumeDiskProfilePair.first(); DiskProfile diskProfile = volumeDiskProfilePair.second(); @@ -2697,12 +2719,28 @@ public boolean storagePoolHasEnoughIops(List> requeste requestedIops += minIops; } } + return storagePoolHasEnoughIops(requestedIops, pool, true); + } - long futureIops = currentIops + requestedIops; - boolean hasEnoughIops = futureIops <= pool.getCapacityIops(); - String hasCapacity = hasEnoughIops ? "has" : "does not have"; - s_logger.debug(String.format("Pool [%s] %s enough IOPS to allocate volumes [%s].", pool, hasCapacity, requestedVolumes)); - return hasEnoughIops; + @Override + public boolean storagePoolHasEnoughIops(Long requestedIops, StoragePool pool) { + if (pool == null) { + return false; + } + if (requestedIops == null || requestedIops == 0) { + return true; + } + return storagePoolHasEnoughIops(requestedIops, pool, false); + } + + @Override + public boolean storagePoolHasEnoughSpace(Long size, StoragePool pool) { + if (size == null || size == 0) { + return true; + } + final StoragePoolVO poolVO = _storagePoolDao.findById(pool.getId()); + long allocatedSizeWithTemplate = _capacityMgr.getAllocatedPoolCapacity(poolVO, null); + return checkPoolforSpace(pool, allocatedSizeWithTemplate, size); } @Override @@ -2792,6 +2830,38 @@ public boolean storagePoolHasEnoughSpaceForResize(StoragePool pool, long current } } + protected Answer getCheckDatastorePolicyComplianceAnswer(String storagePolicyId, StoragePool pool) throws StorageUnavailableException { + if (StringUtils.isEmpty(storagePolicyId)) { + return null; + } + VsphereStoragePolicyVO storagePolicyVO = _vsphereStoragePolicyDao.findById(Long.parseLong(storagePolicyId)); + List hostIds = getUpHostsInPool(pool.getId()); + Collections.shuffle(hostIds); + + if (CollectionUtils.isEmpty(hostIds)) { + throw new StorageUnavailableException("Unable to send command to the pool " + pool.getName() + " due to there is no enabled hosts up in this cluster", pool.getId()); + } + try { + StorageFilerTO storageFilerTO = new StorageFilerTO(pool); + CheckDataStoreStoragePolicyComplainceCommand cmd = new CheckDataStoreStoragePolicyComplainceCommand(storagePolicyVO.getPolicyId(), storageFilerTO); + long targetHostId = _hvGuruMgr.getGuruProcessedCommandTargetHost(hostIds.get(0), cmd); + return _agentMgr.send(targetHostId, cmd); + } catch (AgentUnavailableException e) { + s_logger.debug("Unable to send storage pool command to " + pool + " via " + hostIds.get(0), e); + throw new StorageUnavailableException("Unable to send command to the pool ", pool.getId()); + } catch (OperationTimedoutException e) { + s_logger.debug("Failed to process storage pool command to " + pool + " via " + hostIds.get(0), e); + throw new StorageUnavailableException("Failed to process storage command to the pool ", pool.getId()); + } + } + + @Override + public boolean isStoragePoolCompliantWithStoragePolicy(long diskOfferingId, StoragePool pool) throws StorageUnavailableException { + String storagePolicyId = _diskOfferingDetailsDao.getDetail(diskOfferingId, ApiConstants.STORAGE_POLICY); + Answer answer = getCheckDatastorePolicyComplianceAnswer(storagePolicyId, pool); + return answer == null || answer.getResult(); + } + @Override public boolean isStoragePoolCompliantWithStoragePolicy(List> volumes, StoragePool pool) throws StorageUnavailableException { if (CollectionUtils.isEmpty(volumes)) { @@ -2812,27 +2882,9 @@ public boolean isStoragePoolCompliantWithStoragePolicy(List hostIds = getUpHostsInPool(pool.getId()); - Collections.shuffle(hostIds); - - if (hostIds == null || hostIds.isEmpty()) { - throw new StorageUnavailableException("Unable to send command to the pool " + pool.getName() + " due to there is no enabled hosts up in this cluster", pool.getId()); - } - try { - StorageFilerTO storageFilerTO = new StorageFilerTO(pool); - CheckDataStoreStoragePolicyComplainceCommand cmd = new CheckDataStoreStoragePolicyComplainceCommand(storagePolicyVO.getPolicyId(), storageFilerTO); - long targetHostId = _hvGuruMgr.getGuruProcessedCommandTargetHost(hostIds.get(0), cmd); - Answer answer = _agentMgr.send(targetHostId, cmd); - answers.add(new Pair<>(volume, answer)); - } catch (AgentUnavailableException e) { - s_logger.debug("Unable to send storage pool command to " + pool + " via " + hostIds.get(0), e); - throw new StorageUnavailableException("Unable to send command to the pool ", pool.getId()); - } catch (OperationTimedoutException e) { - s_logger.debug("Failed to process storage pool command to " + pool + " via " + hostIds.get(0), e); - throw new StorageUnavailableException("Failed to process storage command to the pool ", pool.getId()); - } + Answer answer = getCheckDatastorePolicyComplianceAnswer(storagePolicyId, pool); + if (answer != null) { + answers.add(new Pair<>(volume, answer)); } } // check cummilative result for all volumes diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 2a0821c5c0a7..ff6e96fa8660 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -350,7 +350,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic protected Gson _gson; private static final List SupportedHypervisorsForVolResize = Arrays.asList(HypervisorType.KVM, HypervisorType.XenServer, - HypervisorType.VMware, HypervisorType.Any, HypervisorType.None); + HypervisorType.VMware, HypervisorType.Simulator, HypervisorType.Any, HypervisorType.None); private List _storagePoolAllocators; private List supportingDefaultHV; @@ -506,7 +506,7 @@ public GetUploadParamsResponse doInTransaction(TransactionStatus status) throws Account account = _accountDao.findById(accountId); Domain domain = domainDao.findById(account.getDomainId()); - command.setDefaultMaxSecondaryStorageInGB(_resourceLimitMgr.findCorrectResourceLimitForAccountAndDomain(account, domain, ResourceType.secondary_storage)); + command.setDefaultMaxSecondaryStorageInGB(_resourceLimitMgr.findCorrectResourceLimitForAccountAndDomain(account, domain, ResourceType.secondary_storage, null)); command.setAccountId(accountId); Gson gson = new GsonBuilder().create(); String metadata = EncryptionUtil.encodeData(gson.toJson(command), key); @@ -525,10 +525,14 @@ private boolean validateVolume(Account caller, long ownerId, Long zoneId, String // permission check Account volumeOwner = _accountMgr.getActiveAccountById(ownerId); + DiskOfferingVO diskOffering = null; + if (diskOfferingId != null) { + diskOffering = _diskOfferingDao.findById(diskOfferingId); + } _accountMgr.checkAccess(caller, null, true, volumeOwner); // Check that the resource limit for volumes won't be exceeded - _resourceLimitMgr.checkResourceLimit(volumeOwner, ResourceType.volume); + _resourceLimitMgr.checkVolumeResourceLimit(volumeOwner, true, null, diskOffering); // Verify that zone exists DataCenterVO zone = _dcDao.findById(zoneId); @@ -561,7 +565,6 @@ private boolean validateVolume(Account caller, long ownerId, Long zoneId, String // Check that the disk offering specified is valid if (diskOfferingId != null) { - DiskOfferingVO diskOffering = _diskOfferingDao.findById(diskOfferingId); if ((diskOffering == null) || diskOffering.getRemoved() != null || diskOffering.isComputeOnly()) { throw new InvalidParameterValueException("Please specify a valid disk offering."); } @@ -651,7 +654,7 @@ public VolumeVO doInTransaction(TransactionStatus status) { // Increment resource count during allocation; if actual creation fails, // decrement it - _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume); + _resourceLimitMgr.incrementVolumeResourceCount(volume.getAccountId(), true, null, diskOfferingVO); //url can be null incase of postupload if (url != null) { _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); @@ -709,7 +712,7 @@ public VolumeVO allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationExcept } // Check that the resource limit for volumes won't be exceeded - _resourceLimitMgr.checkResourceLimit(owner, ResourceType.volume, displayVolume); +// _resourceLimitMgr.checkResourceLimit(owner, ResourceType.volume, displayVolume); Long zoneId = cmd.getZoneId(); Long diskOfferingId = null; @@ -893,8 +896,8 @@ public VolumeVO allocVolume(CreateVolumeCmd cmd) throws ResourceAllocationExcept Storage.ProvisioningType provisioningType = diskOffering.getProvisioningType(); - // Check that the resource limit for primary storage won't be exceeded - _resourceLimitMgr.checkResourceLimit(owner, ResourceType.primary_storage, displayVolume, new Long(size)); + // Check that the resource limit for volume & primary storage won't be exceeded + _resourceLimitMgr.checkVolumeResourceLimit(owner,displayVolume, size, diskOffering); // Verify that zone exists DataCenterVO zone = _dcDao.findById(zoneId); @@ -980,8 +983,8 @@ public VolumeVO doInTransaction(TransactionStatus status) { CallContext.current().putContextParameter(Volume.class, volume.getId()); // Increment resource count during allocation; if actual creation fails, // decrement it - _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume, displayVolume); - _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.primary_storage, displayVolume, new Long(volume.getSize())); + _resourceLimitMgr.incrementVolumeResourceCount(volume.getAccountId(), displayVolume, volume.getSize(), + _diskOfferingDao.findById(volume.getDiskOfferingId())); return volume; } }); @@ -1040,8 +1043,8 @@ public VolumeVO createVolume(CreateVolumeCmd cmd) { } finally { if (!created) { s_logger.trace("Decrementing volume resource count for account id=" + volume.getAccountId() + " as volume failed to create on the backend"); - _resourceLimitMgr.decrementResourceCount(volume.getAccountId(), ResourceType.volume, cmd.getDisplayVolume()); - _resourceLimitMgr.decrementResourceCount(volume.getAccountId(), ResourceType.primary_storage, cmd.getDisplayVolume(), new Long(volume.getSize())); + _resourceLimitMgr.decrementVolumeResourceCount(volume.getAccountId(), cmd.getDisplayVolume(), + volume.getSize(), _diskOfferingDao.findById(volume.getDiskOfferingId())); } } } @@ -1529,10 +1532,11 @@ private VolumeVO orchestrateResizeVolume(long volumeId, long currentSize, long n } /* Update resource count for the account on primary storage resource */ + DiskOffering diskOffering = _diskOfferingDao.findById(volume.getDiskOfferingId()); if (!shrinkOk) { - _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.primary_storage, volume.isDisplayVolume(), newSize - currentSize); + _resourceLimitMgr.incrementVolumePrimaryStorageResourceCount(volume.getAccountId(), volume.isDisplayVolume(), newSize - currentSize, diskOffering); } else { - _resourceLimitMgr.decrementResourceCount(volume.getAccountId(), ResourceType.primary_storage, volume.isDisplayVolume(), currentSize - newSize); + _resourceLimitMgr.decrementVolumePrimaryStorageResourceCount(volume.getAccountId(), volume.isDisplayVolume(), currentSize - newSize, diskOffering); } UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_RESIZE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), @@ -1734,8 +1738,8 @@ public Volume destroyVolume(long volumeId, Account caller, boolean expunge, bool s_logger.debug("Failed to destroy volume" + volume.getId(), e); return null; } - _resourceLimitMgr.decrementResourceCount(volume.getAccountId(), ResourceType.volume, volume.isDisplay()); - _resourceLimitMgr.decrementResourceCount(volume.getAccountId(), ResourceType.primary_storage, volume.isDisplay(), new Long(volume.getSize())); + _resourceLimitMgr.decrementVolumeResourceCount(volume.getAccountId(), volume.isDisplay(), + volume.getSize(), _diskOfferingDao.findById(volume.getDiskOfferingId())); return volume; } if (!deleteVolumeFromStorage(volume, caller)) { @@ -1792,8 +1796,8 @@ public Volume recoverVolume(long volumeId) { s_logger.debug("Failed to recover volume" + volume.getId(), e); throw new CloudRuntimeException("Failed to recover volume" + volume.getId(), e); } - _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume, volume.isDisplay()); - _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.primary_storage, volume.isDisplay(), new Long(volume.getSize())); + _resourceLimitMgr.incrementVolumeResourceCount(volume.getAccountId(), volume.isDisplay(), + volume.getSize(), _diskOfferingDao.findById(volume.getDiskOfferingId())); publishVolumeCreationUsageEvent(volume); @@ -2025,9 +2029,7 @@ private void validateVolumeReadyStateAndHypervisorChecks(VolumeVO volume, long c /* Only works for KVM/XenServer/VMware (or "Any") for now, and volumes with 'None' since they're just allocated in DB */ HypervisorType hypervisorType = _volsDao.getHypervisorType(volume.getId()); - if (hypervisorType != HypervisorType.KVM && hypervisorType != HypervisorType.XenServer - && hypervisorType != HypervisorType.VMware && hypervisorType != HypervisorType.Any - && hypervisorType != HypervisorType.None) { + if (!SupportedHypervisorsForVolResize.contains(hypervisorType)) { throw new InvalidParameterValueException("Hypervisor " + hypervisorType + " does not support volume resize"); } @@ -2650,8 +2652,11 @@ public void updateDisplay(Volume volume, Boolean displayVolume) { private void updateResourceCount(Volume volume, Boolean displayVolume) { // Update only when the flag has changed. if (displayVolume != null && displayVolume != volume.isDisplayVolume()) { - _resourceLimitMgr.changeResourceCount(volume.getAccountId(), ResourceType.volume, displayVolume); - _resourceLimitMgr.changeResourceCount(volume.getAccountId(), ResourceType.primary_storage, displayVolume, new Long(volume.getSize())); + if (Boolean.FALSE.equals(displayVolume)) { + _resourceLimitMgr.decrementVolumeResourceCount(volume.getAccountId(), true, volume.getSize(), _diskOfferingDao.findById(volume.getDiskOfferingId())); + } else { + _resourceLimitMgr.incrementVolumeResourceCount(volume.getAccountId(), true, volume.getSize(), _diskOfferingDao.findById(volume.getDiskOfferingId())); + } } } @@ -3870,8 +3875,7 @@ public Volume assignVolumeToAccount(AssignVolumeCmd command) throws ResourceAllo _accountMgr.checkAccess(caller, null, true, oldAccount); _accountMgr.checkAccess(caller, null, true, newAccount); - _resourceLimitMgr.checkResourceLimit(newAccount, ResourceType.volume); - _resourceLimitMgr.checkResourceLimit(newAccount, ResourceType.primary_storage, volume.getSize()); + _resourceLimitMgr.checkVolumeResourceLimit(newAccount, true, volume.getSize(), _diskOfferingDao.findById(volume.getDiskOfferingId())); Transaction.execute(new TransactionCallbackNoReturn() { @Override @@ -3886,16 +3890,15 @@ public void doInTransactionWithoutResult(TransactionStatus status) { protected void updateVolumeAccount(Account oldAccount, VolumeVO volume, Account newAccount) { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_DELETE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), Volume.class.getName(), volume.getUuid(), volume.isDisplayVolume()); - _resourceLimitMgr.decrementResourceCount(oldAccount.getAccountId(), ResourceType.volume); - _resourceLimitMgr.decrementResourceCount(oldAccount.getAccountId(), ResourceType.primary_storage, volume.getSize()); + DiskOfferingVO diskOfferingVO = _diskOfferingDao.findById(volume.getDiskOfferingId()); + _resourceLimitMgr.decrementVolumeResourceCount(oldAccount.getAccountId(), true, volume.getSize(), + diskOfferingVO); volume.setAccountId(newAccount.getAccountId()); volume.setDomainId(newAccount.getDomainId()); _volsDao.persist(volume); - - _resourceLimitMgr.incrementResourceCount(newAccount.getAccountId(), ResourceType.volume); - _resourceLimitMgr.incrementResourceCount(newAccount.getAccountId(), ResourceType.primary_storage, volume.getSize()); - + _resourceLimitMgr.incrementVolumeResourceCount(newAccount.getAccountId(), true, volume.getSize(), + diskOfferingVO); UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), volume.getDiskOfferingId(), volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), volume.isDisplayVolume()); diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index 940860dd04d7..9f59a5e0fd2d 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -1038,8 +1038,8 @@ public SnapshotPolicyVO createPolicy(CreateSnapshotPolicyCmd cmd, Account policy // Verify that max doesn't exceed domain and account snapshot limits in case display is on if (display) { - long accountLimit = _resourceLimitMgr.findCorrectResourceLimitForAccount(owner, ResourceType.snapshot); - long domainLimit = _resourceLimitMgr.findCorrectResourceLimitForDomain(_domainMgr.getDomain(owner.getDomainId()), ResourceType.snapshot); + long accountLimit = _resourceLimitMgr.findCorrectResourceLimitForAccount(owner, ResourceType.snapshot, null); + long domainLimit = _resourceLimitMgr.findCorrectResourceLimitForDomain(_domainMgr.getDomain(owner.getDomainId()), ResourceType.snapshot, null); if (!_accountMgr.isRootAdmin(owner.getId()) && ((accountLimit != -1 && maxSnaps > accountLimit) || (domainLimit != -1 && maxSnaps > domainLimit))) { String message = "domain/account"; if (owner.getType() == Account.Type.PROJECT) { diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index b886f0868f67..ca9ebb27891a 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -423,7 +423,6 @@ public List doInTransaction(TransactionStatus postUploadAllocation(imageStores, template, payloads); } else { postUploadAllocation(List.of(imageStore), template, payloads); - } if(payloads.isEmpty()) { @@ -472,7 +471,7 @@ private void postUploadAllocation(List imageStores, VMTemplateVO temp Account account = _accountDao.findById(accountId); Domain domain = _domainDao.findById(account.getDomainId()); - payload.setDefaultMaxSecondaryStorageInGB(_resourceLimitMgr.findCorrectResourceLimitForAccountAndDomain(account, domain, ResourceType.secondary_storage)); + payload.setDefaultMaxSecondaryStorageInGB(_resourceLimitMgr.findCorrectResourceLimitForAccountAndDomain(account, domain, ResourceType.secondary_storage, null)); payload.setAccountId(accountId); payload.setRemoteEndPoint(ep.getPublicAddr()); payload.setRequiresHvm(template.requiresHvm()); diff --git a/server/src/main/java/com/cloud/user/DomainManagerImpl.java b/server/src/main/java/com/cloud/user/DomainManagerImpl.java index 1551309890cc..7b5625cb564a 100644 --- a/server/src/main/java/com/cloud/user/DomainManagerImpl.java +++ b/server/src/main/java/com/cloud/user/DomainManagerImpl.java @@ -984,25 +984,43 @@ public void doInTransactionWithoutResult(TransactionStatus status) { return domainToBeMoved; } - protected void validateNewParentDomainResourceLimits(DomainVO domainToBeMoved, DomainVO newParentDomain) throws ResourceAllocationException { + protected void validateNewParentDomainResourceLimit(DomainVO domainToBeMoved, DomainVO newParentDomain, + Resource.ResourceType resourceType, String tag) throws ResourceAllocationException { long domainToBeMovedId = domainToBeMoved.getId(); long newParentDomainId = newParentDomain.getId(); - for (Resource.ResourceType resourceType : Resource.ResourceType.values()) { - long currentDomainResourceCount = _resourceCountDao.getResourceCount(domainToBeMovedId, ResourceOwnerType.Domain, resourceType); - long newParentDomainResourceCount = _resourceCountDao.getResourceCount(newParentDomainId, ResourceOwnerType.Domain, resourceType); - long newParentDomainResourceLimit = resourceLimitService.findCorrectResourceLimitForDomain(newParentDomain, resourceType); + long currentDomainResourceCount = _resourceCountDao.getResourceCount(domainToBeMovedId, ResourceOwnerType.Domain, resourceType, tag); + long newParentDomainResourceCount = _resourceCountDao.getResourceCount(newParentDomainId, ResourceOwnerType.Domain, resourceType, tag); + long newParentDomainResourceLimit = resourceLimitService.findCorrectResourceLimitForDomain(newParentDomain, resourceType, tag); + + if (newParentDomainResourceLimit == Resource.RESOURCE_UNLIMITED) { + return; + } + + if (currentDomainResourceCount + newParentDomainResourceCount > newParentDomainResourceLimit) { + String message = String.format("Cannot move domain [%s] to parent domain [%s] as maximum domain resource limit of type [%s] would be exceeded. The current resource " + + "count for domain [%s] is [%s], the resource count for the new parent domain [%s] is [%s], and the limit is [%s].", domainToBeMoved.getUuid(), + newParentDomain.getUuid(), resourceType, domainToBeMoved.getUuid(), currentDomainResourceCount, newParentDomain.getUuid(), newParentDomainResourceCount, + newParentDomainResourceLimit); + s_logger.error(message); + throw new ResourceAllocationException(message, resourceType); + } + } - if (newParentDomainResourceLimit == Resource.RESOURCE_UNLIMITED) { - return; - } - if (currentDomainResourceCount + newParentDomainResourceCount > newParentDomainResourceLimit) { - String message = String.format("Cannot move domain [%s] to parent domain [%s] as maximum domain resource limit of type [%s] would be exceeded. The current resource " - + "count for domain [%s] is [%s], the resource count for the new parent domain [%s] is [%s], and the limit is [%s].", domainToBeMoved.getUuid(), - newParentDomain.getUuid(), resourceType, domainToBeMoved.getUuid(), currentDomainResourceCount, newParentDomain.getUuid(), newParentDomainResourceCount, - newParentDomainResourceLimit); - s_logger.error(message); - throw new ResourceAllocationException(message, resourceType); + protected void validateNewParentDomainResourceLimits(DomainVO domainToBeMoved, DomainVO newParentDomain) throws ResourceAllocationException { + List hostTags = resourceLimitService.getResourceLimitHostTags(); + List storageTags = resourceLimitService.getResourceLimitStorageTags(); + for (Resource.ResourceType resourceType : Resource.ResourceType.values()) { + validateNewParentDomainResourceLimit(domainToBeMoved, newParentDomain, resourceType, null); + if (ResourceLimitService.HostTagsSupportingTypes.contains(resourceType)) { + for (String tag : hostTags) { + validateNewParentDomainResourceLimit(domainToBeMoved, newParentDomain, resourceType, tag); + } + } + if (ResourceLimitService.StorageTagsSupportingTypes.contains(resourceType)) { + for (String tag : storageTags) { + validateNewParentDomainResourceLimit(domainToBeMoved, newParentDomain, resourceType, tag); + } } } } diff --git a/server/src/main/java/com/cloud/vm/UserVmManager.java b/server/src/main/java/com/cloud/vm/UserVmManager.java index 4f1396913cce..3c9922528577 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManager.java +++ b/server/src/main/java/com/cloud/vm/UserVmManager.java @@ -130,4 +130,6 @@ UserVm updateVirtualMachine(long id, String displayName, String group, Boolean h Boolean getDestroyRootVolumeOnVmDestruction(Long domainId); + Map getDiskOfferingSuitabilityForVm(long vmId, List diskOfferingIds); + } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 121ca95e365f..74da760906d5 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -682,25 +682,15 @@ public List getVirtualMachines(long hostId) { return _vmDao.listByHostId(hostId); } - private void resourceLimitCheck(Account owner, Boolean displayVm, Long cpu, Long memory) throws ResourceAllocationException { - _resourceLimitMgr.checkResourceLimit(owner, ResourceType.user_vm, displayVm); - _resourceLimitMgr.checkResourceLimit(owner, ResourceType.cpu, displayVm, cpu); - _resourceLimitMgr.checkResourceLimit(owner, ResourceType.memory, displayVm, memory); - } - - protected void resourceCountIncrement(long accountId, Boolean displayVm, Long cpu, Long memory) { - if (! VirtualMachineManager.ResourceCountRunningVMsonly.value()) { - _resourceLimitMgr.incrementResourceCount(accountId, ResourceType.user_vm, displayVm); - _resourceLimitMgr.incrementResourceCount(accountId, ResourceType.cpu, displayVm, cpu); - _resourceLimitMgr.incrementResourceCount(accountId, ResourceType.memory, displayVm, memory); + protected void resourceCountIncrement(long accountId, Boolean displayVm, ServiceOffering serviceOffering, VirtualMachineTemplate template) { + if (!VirtualMachineManager.ResourceCountRunningVMsonly.value()) { + _resourceLimitMgr.incrementVmResourceCount(accountId, displayVm, serviceOffering, template); } } - protected void resourceCountDecrement(long accountId, Boolean displayVm, Long cpu, Long memory) { - if (! VirtualMachineManager.ResourceCountRunningVMsonly.value()) { - _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.user_vm, displayVm); - _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.cpu, displayVm, cpu); - _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.memory, displayVm, memory); + protected void resourceCountDecrement(long accountId, Boolean displayVm, ServiceOffering serviceOffering, VirtualMachineTemplate template) { + if (!VirtualMachineManager.ResourceCountRunningVMsonly.value()) { + _resourceLimitMgr.decrementVmResourceCount(accountId, displayVm, serviceOffering, template); } } @@ -1306,12 +1296,13 @@ private UserVm upgradeStoppedVirtualMachine(Long vmId, Long svcOffId, Map currentCpu) { - _resourceLimitMgr.checkResourceLimit(owner, ResourceType.cpu, newCpu - currentCpu); + _resourceLimitMgr.checkVmCpuResourceLimit(owner, vmInstance.isDisplay(), newServiceOffering, template, (long)(newCpu - currentCpu)); } if (newMemory > currentMemory) { - _resourceLimitMgr.checkResourceLimit(owner, ResourceType.memory, newMemory - currentMemory); + _resourceLimitMgr.checkVmMemoryResourceLimit(owner, vmInstance.isDisplay(), newServiceOffering, template, (long)(newMemory - currentMemory)); } } @@ -1330,14 +1321,14 @@ private UserVm upgradeStoppedVirtualMachine(Long vmId, Long svcOffId, Map currentCpu) { - _resourceLimitMgr.incrementResourceCount(owner.getAccountId(), ResourceType.cpu, new Long(newCpu - currentCpu)); + _resourceLimitMgr.incrementVmCpuResourceCount(owner.getAccountId(), vmInstance.isDisplay(), newServiceOffering, template, (long)(newCpu - currentCpu)); } else if (currentCpu > newCpu) { - _resourceLimitMgr.decrementResourceCount(owner.getAccountId(), ResourceType.cpu, new Long(currentCpu - newCpu)); + _resourceLimitMgr.decrementVmMemoryResourceCount(owner.getAccountId(), vmInstance.isDisplay(), newServiceOffering, template, (long)(currentCpu - newCpu)); } if (newMemory > currentMemory) { - _resourceLimitMgr.incrementResourceCount(owner.getAccountId(), ResourceType.memory, new Long(newMemory - currentMemory)); + _resourceLimitMgr.incrementVmMemoryResourceCount(owner.getAccountId(), vmInstance.isDisplay(), newServiceOffering, template, (long)(newMemory - currentMemory)); } else if (currentMemory > newMemory) { - _resourceLimitMgr.decrementResourceCount(owner.getAccountId(), ResourceType.memory, new Long(currentMemory - newMemory)); + _resourceLimitMgr.decrementVmMemoryResourceCount(owner.getAccountId(), vmInstance.isDisplay(), newServiceOffering, template, (long)(currentMemory - newMemory)); } } @@ -1923,10 +1914,11 @@ public boolean upgradeVirtualMachine(Long vmId, Long newServiceOfferingId, Map supportedHypervisorTypes = new HashSet<>(); supportedHypervisorTypes.add(HypervisorType.XenServer); @@ -2011,13 +2004,15 @@ private boolean upgradeRunningVirtualMachine(Long vmId, Long newServiceOfferingI throw new InvalidParameterValueException(String.format("Dynamic scaling of vGPU type is not supported. VM has vGPU Type: [%s].", currentVgpuType)); } + VMTemplateVO template = _templateDao.findByIdIncludingRemoved(vmInstance.getTemplateId()); + // Check resource limits if (newCpu > currentCpu) { - _resourceLimitMgr.checkResourceLimit(caller, ResourceType.cpu, newCpu - currentCpu); + _resourceLimitMgr.checkVmCpuResourceLimit(owner, vmInstance.isDisplay(), newServiceOffering, template, (long)(newCpu - currentCpu)); } if (newMemory > currentMemory) { - _resourceLimitMgr.checkResourceLimit(caller, ResourceType.memory, memoryDiff); + _resourceLimitMgr.checkVmMemoryResourceLimit(caller, vmInstance.isDisplay(), newServiceOffering, template, (long)memoryDiff); } // Dynamically upgrade the running vms @@ -2049,11 +2044,11 @@ private boolean upgradeRunningVirtualMachine(Long vmId, Long newServiceOfferingI // Increment CPU and Memory count accordingly. if (newCpu > currentCpu) { - _resourceLimitMgr.incrementResourceCount(caller.getAccountId(), ResourceType.cpu, new Long(newCpu - currentCpu)); + _resourceLimitMgr.incrementVmCpuResourceCount(caller.getAccountId(), vmInstance.isDisplay(), newServiceOffering, template, (long)(newCpu - currentCpu)); } if (memoryDiff > 0) { - _resourceLimitMgr.incrementResourceCount(caller.getAccountId(), ResourceType.memory, new Long(memoryDiff)); + _resourceLimitMgr.incrementVmMemoryResourceCount(caller.getAccountId(), vmInstance.isDisplay(), newServiceOffering, template, (long)memoryDiff); } // #1 Check existing host has capacity @@ -2085,11 +2080,11 @@ private boolean upgradeRunningVirtualMachine(Long vmId, Long newServiceOfferingI if (!success) { // Decrement CPU and Memory count accordingly. if (newCpu > currentCpu) { - _resourceLimitMgr.decrementResourceCount(caller.getAccountId(), ResourceType.cpu, new Long(newCpu - currentCpu)); + _resourceLimitMgr.decrementVmCpuResourceCount(caller.getAccountId(), vmInstance.isDisplay(), newServiceOffering, template, (long)(newCpu - currentCpu)); } if (memoryDiff > 0) { - _resourceLimitMgr.decrementResourceCount(caller.getAccountId(), ResourceType.memory, new Long(memoryDiff)); + _resourceLimitMgr.decrementVmMemoryResourceCount(caller.getAccountId(), vmInstance.isDisplay(), newServiceOffering, template, (long)memoryDiff); } } } @@ -2278,11 +2273,12 @@ public UserVm recoverVirtualMachine(RecoverVMCmd cmd) throws ResourceAllocationE // Get serviceOffering for Virtual Machine ServiceOfferingVO serviceOffering = serviceOfferingDao.findById(vm.getId(), vm.getServiceOfferingId()); + VMTemplateVO tempalte = _templateDao.findByIdIncludingRemoved(vm.getTemplateId()); // First check that the maximum number of UserVMs, CPU and Memory limit for the given // accountId will not be exceeded if (! VirtualMachineManager.ResourceCountRunningVMsonly.value()) { - resourceLimitCheck(account, vm.isDisplayVm(), new Long(serviceOffering.getCpu()), new Long(serviceOffering.getRamSize())); + resourceLimitService.checkVmResourceLimit(account, vm.isDisplayVm(), serviceOffering, tempalte); } _haMgr.cancelDestroy(vm, vm.getHostId()); @@ -2306,7 +2302,7 @@ public UserVm recoverVirtualMachine(RecoverVMCmd cmd) throws ResourceAllocationE } //Update Resource Count for the given account - resourceCountIncrement(account.getId(), vm.isDisplayVm(), new Long(serviceOffering.getCpu()), new Long(serviceOffering.getRamSize())); + resourceCountIncrement(account.getId(), vm.isDisplayVm(), serviceOffering, tempalte); } }); @@ -2595,11 +2591,12 @@ private void updateVmStateForFailedVmCreation(Long vmId, Long hostId) { String msg = "Failed to deploy Vm with Id: " + vmId + ", on Host with Id: " + hostId; _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_USERVM, vm.getDataCenterId(), vm.getPodIdToDeployIn(), msg, msg); - // Get serviceOffering for Virtual Machine + // Get serviceOffering and template for Virtual Machine ServiceOfferingVO offering = serviceOfferingDao.findById(vm.getId(), vm.getServiceOfferingId()); + VMTemplateVO tempalte = _templateDao.findByIdIncludingRemoved(vm.getTemplateId()); // Update Resource Count for the given account - resourceCountDecrement(vm.getAccountId(), vm.isDisplayVm(), new Long(offering.getCpu()), new Long(offering.getRamSize())); + resourceCountDecrement(vm.getAccountId(), vm.isDisplayVm(), offering, tempalte); } } } @@ -2721,13 +2718,13 @@ private void verifyVmLimits(UserVmVO vmInstance, Map details) { } long currentCpu = currentServiceOffering.getCpu(); long currentMemory = currentServiceOffering.getRamSize(); - + VMTemplateVO template = _templateDao.findByIdIncludingRemoved(vmInstance.getTemplateId()); try { if (newCpu > currentCpu) { - _resourceLimitMgr.checkResourceLimit(owner, ResourceType.cpu, newCpu - currentCpu); + _resourceLimitMgr.checkVmCpuResourceLimit(owner, vmInstance.isDisplay(), svcOffering, template, newCpu - currentCpu); } if (newMemory > currentMemory) { - _resourceLimitMgr.checkResourceLimit(owner, ResourceType.memory, newMemory - currentMemory); + _resourceLimitMgr.checkVmMemoryResourceLimit(owner, vmInstance.isDisplay(), svcOffering, template, newMemory - currentMemory); } } catch (ResourceAllocationException e) { s_logger.error(String.format("Failed to updated VM due to: %s", e.getLocalizedMessage())); @@ -2735,14 +2732,14 @@ private void verifyVmLimits(UserVmVO vmInstance, Map details) { } if (newCpu > currentCpu) { - _resourceLimitMgr.incrementResourceCount(owner.getAccountId(), ResourceType.cpu, newCpu - currentCpu); + _resourceLimitMgr.incrementVmCpuResourceCount(owner.getAccountId(), vmInstance.isDisplay(), svcOffering, template, newCpu - currentCpu); } else if (newCpu > 0 && currentCpu > newCpu){ - _resourceLimitMgr.decrementResourceCount(owner.getAccountId(), ResourceType.cpu, currentCpu - newCpu); + _resourceLimitMgr.decrementVmCpuResourceCount(owner.getAccountId(), vmInstance.isDisplay(), svcOffering, template, currentCpu - newCpu); } if (newMemory > currentMemory) { - _resourceLimitMgr.incrementResourceCount(owner.getAccountId(), ResourceType.memory, newMemory - currentMemory); + _resourceLimitMgr.incrementVmMemoryResourceCount(owner.getAccountId(), vmInstance.isDisplay(), svcOffering, template, newMemory - currentMemory); } else if (newMemory > 0 && currentMemory > newMemory){ - _resourceLimitMgr.decrementResourceCount(owner.getAccountId(), ResourceType.memory, currentMemory - newMemory); + _resourceLimitMgr.decrementVmMemoryResourceCount(owner.getAccountId(), vmInstance.isDisplay(), svcOffering, template, currentMemory - newMemory); } } @@ -2863,10 +2860,11 @@ protected void updateDisplayVmFlag(Boolean isDisplayVm, Long id, UserVmVO vmInst // Resource limit changes ServiceOffering offering = serviceOfferingDao.findByIdIncludingRemoved(vmInstance.getId(), vmInstance.getServiceOfferingId()); + VMTemplateVO template = _templateDao.findByIdIncludingRemoved(vmInstance.getTemplateId()); if (isDisplayVm) { - resourceCountIncrement(vmInstance.getAccountId(), true, Long.valueOf(offering.getCpu()), Long.valueOf(offering.getRamSize())); + resourceCountIncrement(vmInstance.getAccountId(), true, offering, template); } else { - resourceCountDecrement(vmInstance.getAccountId(), true, Long.valueOf(offering.getCpu()), Long.valueOf(offering.getRamSize())); + resourceCountDecrement(vmInstance.getAccountId(), true, offering, template); } // Usage @@ -3998,11 +3996,22 @@ private UserVm createVirtualMachine(DataCenter zone, ServiceOffering serviceOffe CallContext.current().putContextParameter(VirtualMachine.class, vm.getUuid()); return vm; } - private UserVm getCheckedUserVmResource(DataCenter zone, String hostName, String displayName, Account owner, Long diskOfferingId, Long diskSize, List networkList, List securityGroupIdList, String group, HTTPMethod httpmethod, String userData, Long userDataId, String userDataDetails, List sshKeyPairs, Account caller, Map requestedIps, IpAddresses defaultIps, Boolean isDisplayVm, String keyboard, List affinityGroupIdList, Map customParameters, String customId, Map> dhcpOptionMap, Map datadiskTemplateToDiskOfferringMap, Map userVmOVFPropertiesMap, boolean dynamicScalingEnabled, String vmType, VMTemplateVO template, HypervisorType hypervisorType, long accountId, ServiceOfferingVO offering, boolean isIso, Long rootDiskOfferingId, long volumesSize) throws ResourceAllocationException, StorageUnavailableException, InsufficientCapacityException { + + private UserVm getCheckedUserVmResource(DataCenter zone, String hostName, String displayName, Account owner, + Long diskOfferingId, Long diskSize, List networkList, List securityGroupIdList, String group, + HTTPMethod httpmethod, String userData, Long userDataId, String userDataDetails, List sshKeyPairs, + Account caller, Map requestedIps, IpAddresses defaultIps, Boolean isDisplayVm, + String keyboard, List affinityGroupIdList, Map customParameters, String customId, + Map> dhcpOptionMap, Map datadiskTemplateToDiskOfferringMap, + Map userVmOVFPropertiesMap, boolean dynamicScalingEnabled, String vmType, VMTemplateVO template, + HypervisorType hypervisorType, long accountId, ServiceOfferingVO offering, boolean isIso, + Long rootDiskOfferingId, long volumesSize) throws ResourceAllocationException, StorageUnavailableException, + InsufficientCapacityException { if (!VirtualMachineManager.ResourceCountRunningVMsonly.value()) { - try (CheckedReservation vmReservation = new CheckedReservation(owner, ResourceType.user_vm, 1l, reservationDao, resourceLimitService); - CheckedReservation cpuReservation = new CheckedReservation(owner, ResourceType.cpu, Long.valueOf(offering.getCpu()), reservationDao, resourceLimitService); - CheckedReservation memReservation = new CheckedReservation(owner, ResourceType.memory, Long.valueOf(offering.getRamSize()), reservationDao, resourceLimitService); + List resourceLimitHostTags = resourceLimitService.getResourceLimitHostTags(offering, template); + try (CheckedReservation vmReservation = new CheckedReservation(owner, ResourceType.user_vm, resourceLimitHostTags, 1l, reservationDao, resourceLimitService); + CheckedReservation cpuReservation = new CheckedReservation(owner, ResourceType.cpu, resourceLimitHostTags, Long.valueOf(offering.getCpu()), reservationDao, resourceLimitService); + CheckedReservation memReservation = new CheckedReservation(owner, ResourceType.memory, resourceLimitHostTags, Long.valueOf(offering.getRamSize()), reservationDao, resourceLimitService); ) { return getUncheckedUserVmResource(zone, hostName, displayName, owner, diskOfferingId, diskSize, networkList, securityGroupIdList, group, httpmethod, userData, userDataId, userDataDetails, sshKeyPairs, caller, requestedIps, defaultIps, isDisplayVm, keyboard, affinityGroupIdList, customParameters, customId, dhcpOptionMap, datadiskTemplateToDiskOfferringMap, userVmOVFPropertiesMap, dynamicScalingEnabled, vmType, template, hypervisorType, accountId, offering, isIso, rootDiskOfferingId, volumesSize); } catch (ResourceAllocationException | CloudRuntimeException e) { @@ -4017,9 +4026,24 @@ private UserVm getCheckedUserVmResource(DataCenter zone, String hostName, String } } - private UserVm getUncheckedUserVmResource(DataCenter zone, String hostName, String displayName, Account owner, Long diskOfferingId, Long diskSize, List networkList, List securityGroupIdList, String group, HTTPMethod httpmethod, String userData, Long userDataId, String userDataDetails, List sshKeyPairs, Account caller, Map requestedIps, IpAddresses defaultIps, Boolean isDisplayVm, String keyboard, List affinityGroupIdList, Map customParameters, String customId, Map> dhcpOptionMap, Map datadiskTemplateToDiskOfferringMap, Map userVmOVFPropertiesMap, boolean dynamicScalingEnabled, String vmType, VMTemplateVO template, HypervisorType hypervisorType, long accountId, ServiceOfferingVO offering, boolean isIso, Long rootDiskOfferingId, long volumesSize) throws ResourceAllocationException, StorageUnavailableException, InsufficientCapacityException { - try (CheckedReservation volumeReservation = new CheckedReservation(owner, ResourceType.volume, (isIso || diskOfferingId == null ? 1l : 2), reservationDao, resourceLimitService); - CheckedReservation primaryStorageReservation = new CheckedReservation(owner, ResourceType.primary_storage, volumesSize, reservationDao, resourceLimitService)) { + protected List getResourceLimitStorageTags(long diskOfferingId) { + DiskOfferingVO diskOfferingVO = _diskOfferingDao.findById(diskOfferingId); + return resourceLimitService.getResourceLimitStorageTags(diskOfferingVO); + } + + private UserVm getUncheckedUserVmResource(DataCenter zone, String hostName, String displayName, Account owner, + Long diskOfferingId, Long diskSize, List networkList, List securityGroupIdList, String group, + HTTPMethod httpmethod, String userData, Long userDataId, String userDataDetails, List sshKeyPairs, + Account caller, Map requestedIps, IpAddresses defaultIps, Boolean isDisplayVm, + String keyboard, List affinityGroupIdList, Map customParameters, String customId, + Map> dhcpOptionMap, Map datadiskTemplateToDiskOfferringMap, + Map userVmOVFPropertiesMap, boolean dynamicScalingEnabled, String vmType, VMTemplateVO template, + HypervisorType hypervisorType, long accountId, ServiceOfferingVO offering, boolean isIso, + Long rootDiskOfferingId, long volumesSize) throws ResourceAllocationException, StorageUnavailableException, + InsufficientCapacityException { + List resourceLimitStorageTags = getResourceLimitStorageTags(diskOfferingId != null ? diskOfferingId : offering.getDiskOfferingId()); + try (CheckedReservation volumeReservation = new CheckedReservation(owner, ResourceType.volume, resourceLimitStorageTags, (isIso || diskOfferingId == null ? 1l : 2), reservationDao, resourceLimitService); + CheckedReservation primaryStorageReservation = new CheckedReservation(owner, ResourceType.primary_storage, resourceLimitStorageTags, volumesSize, reservationDao, resourceLimitService)) { // verify security group ids if (securityGroupIdList != null) { @@ -4061,8 +4085,7 @@ private UserVm getUncheckedUserVmResource(DataCenter zone, String hostName, Stri dataDiskTemplateId + ". Disk offering size should be greater than or equal to the template size"); } _templateDao.loadDetails(dataDiskTemplate); - _resourceLimitMgr.checkResourceLimit(owner, ResourceType.volume, 1); - _resourceLimitMgr.checkResourceLimit(owner, ResourceType.primary_storage, dataDiskOffering.getDiskSize()); + resourceLimitService.checkVolumeResourceLimit(owner, true, dataDiskOffering.getDiskSize(), dataDiskOffering); } } @@ -4572,7 +4595,7 @@ private UserVmVO commitUserVm(final boolean isImport, final DataCenter zone, fin } //Update Resource Count for the given account - resourceCountIncrement(accountId, isDisplayVm, new Long(offering.getCpu()), new Long(offering.getRamSize())); + resourceCountIncrement(accountId, isDisplayVm, offering, template); } return vm; } @@ -5346,10 +5369,11 @@ public Pair> startVirtualMach if (owner.getState() == Account.State.DISABLED) { throw new PermissionDeniedException("The owner of " + vm + " is disabled: " + vm.getAccountId()); } + VMTemplateVO template = _templateDao.findByIdIncludingRemoved(vm.getTemplateId()); if (VirtualMachineManager.ResourceCountRunningVMsonly.value()) { // check if account/domain is with in resource limits to start a new vm ServiceOfferingVO offering = serviceOfferingDao.findById(vm.getId(), vm.getServiceOfferingId()); - resourceLimitCheck(owner, vm.isDisplayVm(), Long.valueOf(offering.getCpu()), Long.valueOf(offering.getRamSize())); + resourceLimitService.checkVmResourceLimit(owner, vm.isDisplayVm(), offering, template); } // check if vm is security group enabled if (_securityGroupMgr.isVmSecurityGroupEnabled(vmId) && _securityGroupMgr.getSecurityGroupsForVm(vmId).isEmpty() @@ -5407,11 +5431,8 @@ public Pair> startVirtualMach // Set parameters Map params = null; - VMTemplateVO template = null; if (vm.isUpdateParameters()) { _vmDao.loadDetails(vm); - // Check that the password was passed in and is valid - template = _templateDao.findByIdIncludingRemoved(vm.getTemplateId()); String password = getCurrentVmPasswordOrDefineNewPassword(String.valueOf(additionalParams.getOrDefault(VirtualMachineProfile.Param.VmPassword, "")), vm, template); @@ -5606,11 +5627,11 @@ public UserVm destroyVm(long vmId, boolean expunge) throws ResourceUnavailableEx } if (vmState != State.Error) { - // Get serviceOffering for Virtual Machine + // Get serviceOffering and template for Virtual Machine ServiceOfferingVO offering = serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()); - + VMTemplateVO template = _templateDao.findByIdIncludingRemoved(vm.getTemplateId()); //Update Resource Count for the given account - resourceCountDecrement(vm.getAccountId(), vm.isDisplayVm(), new Long(offering.getCpu()), new Long(offering.getRamSize())); + resourceCountDecrement(vm.getAccountId(), vm.isDisplayVm(),offering, template); } return _vmDao.findById(vmId); } else { @@ -7105,6 +7126,36 @@ public VirtualMachine migrateVirtualMachineWithVolume(Long vmId, Host destinatio return findMigratedVm(vm.getId(), vm.getType()); } + protected void checkVolumesLimits(Account account, List volumes) throws ResourceAllocationException { + Long totalVolumes = 0L; + Long totalVolumesSize = 0L; + Map diskOfferingVolumeCountMap = new HashMap<>(); + Map diskOfferingSizeMap = new HashMap<>(); + for (VolumeVO volume : volumes) { + if (!volume.isDisplay()) { + continue; + } + totalVolumes++; + totalVolumesSize += volume.getSize(); + if (diskOfferingVolumeCountMap.containsKey(volume.getDiskOfferingId())) { + diskOfferingVolumeCountMap.put(volume.getDiskOfferingId(), diskOfferingVolumeCountMap.get(volume.getDiskOfferingId()) + 1); + diskOfferingSizeMap.put(volume.getDiskOfferingId(), diskOfferingSizeMap.get(volume.getDiskOfferingId()) + volume.getSize()); + } else { + diskOfferingVolumeCountMap.put(volume.getDiskOfferingId(), 1L); + diskOfferingSizeMap.put(volume.getDiskOfferingId(), volume.getSize()); + } + } + _resourceLimitMgr.checkResourceLimit(account, ResourceType.volume, totalVolumes); + _resourceLimitMgr.checkResourceLimit(account, ResourceType.primary_storage, totalVolumesSize); + for (Long diskOfferingId : diskOfferingVolumeCountMap.keySet()) { + List tags = resourceLimitService.getResourceLimitStorageTags(_diskOfferingDao.findById(diskOfferingId)); + for (String tag : tags) { + resourceLimitService.checkResourceLimitWithTag(account, ResourceType.volume, tag, diskOfferingVolumeCountMap.get(diskOfferingId)); + resourceLimitService.checkResourceLimitWithTag(account, ResourceType.primary_storage, tag, diskOfferingSizeMap.get(diskOfferingId)); + } + } + } + @DB @Override @ActionEvent(eventType = EventTypes.EVENT_VM_MOVE, eventDescription = "move VM to another user", async = false) @@ -7197,6 +7248,7 @@ public UserVm moveVMToUser(final AssignVMCmd cmd) throws ResourceAllocationExcep DataCenterVO zone = _dcDao.findById(vm.getDataCenterId()); + VirtualMachineTemplate template = _templateDao.findByIdIncludingRemoved(vm.getTemplateId()); // Get serviceOffering and Volumes for Virtual Machine final ServiceOfferingVO offering = serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()); @@ -7204,20 +7256,14 @@ public UserVm moveVMToUser(final AssignVMCmd cmd) throws ResourceAllocationExcep removeInstanceFromInstanceGroup(cmd.getVmId()); // VV 2: check if account/domain is with in resource limits to create a new vm - if (! VirtualMachineManager.ResourceCountRunningVMsonly.value()) { - resourceLimitCheck(newAccount, vm.isDisplayVm(), new Long(offering.getCpu()), new Long(offering.getRamSize())); + if (!VirtualMachineManager.ResourceCountRunningVMsonly.value()) { + resourceLimitService.checkVmResourceLimit(newAccount, vm.isDisplayVm(), offering, template); } // VV 3: check if volumes and primary storage space are with in resource limits - _resourceLimitMgr.checkResourceLimit(newAccount, ResourceType.volume, _volsDao.findByInstance(cmd.getVmId()).size()); - Long totalVolumesSize = (long)0; - for (VolumeVO volume : volumes) { - totalVolumesSize += volume.getSize(); - } - _resourceLimitMgr.checkResourceLimit(newAccount, ResourceType.primary_storage, totalVolumesSize); + checkVolumesLimits(newAccount, volumes); // VV 4: Check if new owner can use the vm template - VirtualMachineTemplate template = _templateDao.findByIdIncludingRemoved(vm.getTemplateId()); if (template == null) { throw new InvalidParameterValueException(String.format("Template for VM: %s cannot be found", vm.getUuid())); } @@ -7238,7 +7284,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { vm.getId(), vm.getHostName(), vm.getServiceOfferingId(), vm.getTemplateId(), vm.getHypervisorType().toString(), VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplayVm()); // update resource counts for old account - resourceCountDecrement(oldAccount.getAccountId(), vm.isDisplayVm(), new Long(offering.getCpu()), new Long(offering.getRamSize())); + resourceCountDecrement(oldAccount.getAccountId(), vm.isDisplayVm(), offering, template); // OWNERSHIP STEP 1: update the vm owner vm.setAccountId(newAccount.getAccountId()); @@ -7249,22 +7295,19 @@ public void doInTransactionWithoutResult(TransactionStatus status) { for (VolumeVO volume : volumes) { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_DELETE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), Volume.class.getName(), volume.getUuid(), volume.isDisplayVolume()); - _resourceLimitMgr.decrementResourceCount(oldAccount.getAccountId(), ResourceType.volume); - _resourceLimitMgr.decrementResourceCount(oldAccount.getAccountId(), ResourceType.primary_storage, new Long(volume.getSize())); + DiskOfferingVO diskOfferingVO = _diskOfferingDao.findById(volume.getDiskOfferingId()); + _resourceLimitMgr.decrementVolumeResourceCount(oldAccount.getAccountId(), volume.isDisplay(), volume.getSize(), diskOfferingVO); volume.setAccountId(newAccount.getAccountId()); volume.setDomainId(newAccount.getDomainId()); _volsDao.persist(volume); - _resourceLimitMgr.incrementResourceCount(newAccount.getAccountId(), ResourceType.volume); - _resourceLimitMgr.incrementResourceCount(newAccount.getAccountId(), ResourceType.primary_storage, new Long(volume.getSize())); + _resourceLimitMgr.incrementVolumeResourceCount(newAccount.getAccountId(), volume.isDisplay(), volume.getSize(), diskOfferingVO); UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), volume.getDiskOfferingId(), volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), volume.isDisplayVolume()); } //update resource count of new account - if (! VirtualMachineManager.ResourceCountRunningVMsonly.value()) { - resourceCountIncrement(newAccount.getAccountId(), vm.isDisplayVm(), new Long(offering.getCpu()), new Long(offering.getRamSize())); - } + resourceCountIncrement(newAccount.getAccountId(), vm.isDisplayVm(), offering, template); //generate usage events to account for this change UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_CREATE, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), @@ -7803,8 +7846,7 @@ public UserVm restoreVirtualMachine(final Account caller, final long vmId, final } // 1. Save usage event and update resource count for user vm volumes - _resourceLimitMgr.incrementResourceCount(newVol.getAccountId(), ResourceType.volume, newVol.isDisplay()); - _resourceLimitMgr.incrementResourceCount(newVol.getAccountId(), ResourceType.primary_storage, newVol.isDisplay(), new Long(newVol.getSize())); + _resourceLimitMgr.incrementVolumeResourceCount(newVol.getAccountId(), newVol.isDisplay(), newVol.getSize(), _diskOfferingDao.findById(newVol.getDiskOfferingId())); // 2. Create Usage event for the newly created volume UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_VOLUME_CREATE, newVol.getAccountId(), newVol.getDataCenterId(), newVol.getId(), newVol.getName(), newVol.getDiskOfferingId(), template.getId(), newVol.getSize()); _usageEventDao.persist(usageEvent); @@ -8318,21 +8360,24 @@ private void removeVMFromAffinityGroups(long vmId) { */ private void postProcessingUnmanageVM(UserVmVO vm) { ServiceOfferingVO offering = serviceOfferingDao.findById(vm.getServiceOfferingId()); - Long cpu = offering.getCpu() != null ? new Long(offering.getCpu()) : 0L; - Long ram = offering.getRamSize() != null ? new Long(offering.getRamSize()) : 0L; + VMTemplateVO template = _templateDao.findByIdIncludingRemoved(vm.getTemplateId()); // First generate a VM stop event if the VM was not stopped already + boolean resourceNotDecremented = true; if (vm.getState() != State.Stopped) { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_STOP, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), vm.getHostName(), vm.getServiceOfferingId(), vm.getTemplateId(), vm.getHypervisorType().toString(), VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplayVm()); - resourceCountDecrement(vm.getAccountId(), vm.isDisplayVm(), cpu, ram); + resourceCountDecrement(vm.getAccountId(), vm.isDisplayVm(), offering, template); + resourceNotDecremented = false; } // VM destroy usage event UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_DESTROY, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), vm.getHostName(), vm.getServiceOfferingId(), vm.getTemplateId(), vm.getHypervisorType().toString(), VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplayVm()); - resourceCountDecrement(vm.getAccountId(), vm.isDisplayVm(), cpu, ram); + if (resourceNotDecremented) { + resourceCountDecrement(vm.getAccountId(), vm.isDisplayVm(), offering, template); + } } /* @@ -8346,8 +8391,7 @@ private void postProcessingUnmanageVMVolumes(List volumes, UserVmVO vm UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_DELETE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), Volume.class.getName(), volume.getUuid(), volume.isDisplayVolume()); } - _resourceLimitMgr.decrementResourceCount(vm.getAccountId(), ResourceType.volume); - _resourceLimitMgr.decrementResourceCount(vm.getAccountId(), ResourceType.primary_storage, new Long(volume.getSize())); + _resourceLimitMgr.decrementVolumeResourceCount(vm.getAccountId(), volume.isDisplayVolume(), volume.getSize(), _diskOfferingDao.findById(volume.getDiskOfferingId())); } } @@ -8431,10 +8475,15 @@ public Boolean getDestroyRootVolumeOnVmDestruction(Long domainId){ return DestroyRootVolumeOnVmDestruction.valueIn(domainId); } - private void setVncPasswordForKvmIfAvailable(Map customParameters, UserVmVO vm){ + private void setVncPasswordForKvmIfAvailable(Map customParameters, UserVmVO vm) { if (customParameters.containsKey(VmDetailConstants.KVM_VNC_PASSWORD) && StringUtils.isNotEmpty(customParameters.get(VmDetailConstants.KVM_VNC_PASSWORD))) { vm.setVncPassword(customParameters.get(VmDetailConstants.KVM_VNC_PASSWORD)); } } + + @Override + public Map getDiskOfferingSuitabilityForVm(long vmId, List diskOfferingIds) { + return _itMgr.getDiskOfferingSuitabilityForVm(vmId, diskOfferingIds); + } } diff --git a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java index b7190f4ff217..dd83e60c5dbe 100644 --- a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java @@ -438,9 +438,12 @@ private List getHostsManagedVms(List hosts) { return managedVms; } - private boolean hostSupportsServiceOffering(HostVO host, ServiceOffering serviceOffering) { + private boolean hostSupportsServiceOfferingAndTemplate(HostVO host, ServiceOffering serviceOffering, VirtualMachineTemplate template) { + if (StringUtils.isAllEmpty(serviceOffering.getHostTag(), template.getTemplateTag())) { + return true; + } hostDao.loadHostTags(host); - return host.checkHostServiceOfferingTags(serviceOffering); + return host.checkHostServiceOfferingAndTemplateTags(serviceOffering, template); } private boolean storagePoolSupportsDiskOffering(StoragePool pool, DiskOffering diskOffering) { @@ -501,8 +504,6 @@ private ServiceOfferingVO getUnmanagedInstanceServiceOffering(final UnmanagedIns throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Service offering (%s) %dMHz CPU speed does not match VM CPU speed %dMHz and VM is not in powered off state (Power state: %s)", serviceOffering.getUuid(), serviceOffering.getSpeed(), cpuSpeed, instance.getPowerState())); } } - resourceLimitService.checkResourceLimit(owner, Resource.ResourceType.cpu, Long.valueOf(serviceOffering.getCpu())); - resourceLimitService.checkResourceLimit(owner, Resource.ResourceType.memory, Long.valueOf(serviceOffering.getRamSize())); return serviceOffering; } @@ -604,7 +605,7 @@ private void checkUnmanagedDiskAndOfferingForImport(String instanceName, Unmanag if (diskOffering != null) { accountService.checkAccess(owner, diskOffering, zone); } - resourceLimitService.checkResourceLimit(owner, Resource.ResourceType.volume); + resourceLimitService.checkVolumeResourceLimit(owner, true, null, diskOffering); if (disk.getCapacity() == null || disk.getCapacity() == 0) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Size of disk(ID: %s) is found invalid during VM import", disk.getDiskId())); } @@ -891,7 +892,7 @@ private UserVm migrateImportedVM(HostVO sourceHost, VirtualMachineTemplate templ cleanupFailedImportVM(vm); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to check migrations need during import, VM: %s", userVm.getInstanceName())); } - if (!hostSupportsServiceOffering(sourceHost, serviceOffering)) { + if (!hostSupportsServiceOfferingAndTemplate(sourceHost, serviceOffering, template)) { LOGGER.debug(String.format("VM %s needs to be migrated", vm.getUuid())); final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm, template, serviceOffering, owner, null); DeploymentPlanner.ExcludeList excludeList = new DeploymentPlanner.ExcludeList(); @@ -1008,7 +1009,7 @@ private UserVm migrateImportedVM(HostVO sourceHost, VirtualMachineTemplate templ return userVm; } - private void publishVMUsageUpdateResourceCount(final UserVm userVm, ServiceOfferingVO serviceOfferingVO) { + private void publishVMUsageUpdateResourceCount(final UserVm userVm, ServiceOfferingVO serviceOfferingVO, VirtualMachineTemplate templateVO) { if (userVm == null || serviceOfferingVO == null) { LOGGER.error(String.format("Failed to publish usage records during VM import because VM [%s] or ServiceOffering [%s] is null.", userVm, serviceOfferingVO)); cleanupFailedImportVM(userVm); @@ -1031,9 +1032,7 @@ private void publishVMUsageUpdateResourceCount(final UserVm userVm, ServiceOffer cleanupFailedImportVM(userVm); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("VM import failed for unmanaged vm %s during publishing usage records", userVm.getInstanceName())); } - resourceLimitService.incrementResourceCount(userVm.getAccountId(), Resource.ResourceType.user_vm, userVm.isDisplayVm()); - resourceLimitService.incrementResourceCount(userVm.getAccountId(), Resource.ResourceType.cpu, userVm.isDisplayVm(), Long.valueOf(serviceOfferingVO.getCpu())); - resourceLimitService.incrementResourceCount(userVm.getAccountId(), Resource.ResourceType.memory, userVm.isDisplayVm(), Long.valueOf(serviceOfferingVO.getRamSize())); + resourceLimitService.incrementVmResourceCount(userVm.getAccountId(), userVm.isDisplayVm(), serviceOfferingVO, templateVO); // Save usage event and update resource count for user vm volumes List volumes = volumeDao.findByInstance(userVm.getId()); for (VolumeVO volume : volumes) { @@ -1043,8 +1042,8 @@ private void publishVMUsageUpdateResourceCount(final UserVm userVm, ServiceOffer } catch (Exception e) { LOGGER.error(String.format("Failed to publish volume ID: %s usage records during VM import", volume.getUuid()), e); } - resourceLimitService.incrementResourceCount(userVm.getAccountId(), Resource.ResourceType.volume, volume.isDisplayVolume()); - resourceLimitService.incrementResourceCount(userVm.getAccountId(), Resource.ResourceType.primary_storage, volume.isDisplayVolume(), volume.getSize()); + resourceLimitService.incrementVolumeResourceCount(userVm.getAccountId(), volume.isDisplayVolume(), + volume.getSize(), diskOfferingDao.findById(volume.getDiskOfferingId())); } List nics = nicDao.listByVmId(userVm.getId()); @@ -1059,6 +1058,40 @@ private void publishVMUsageUpdateResourceCount(final UserVm userVm, ServiceOffer } } + protected void checkUnmanagedDiskLimits(Account account, UnmanagedInstanceTO.Disk rootDisk, ServiceOffering serviceOffering, + List dataDisks, Map dataDiskOfferingMap) throws ResourceAllocationException { + Long totalVolumes = 0L; + Long totalVolumesSize = 0L; + List disks = new ArrayList<>(); + disks.add(rootDisk); + disks.addAll(dataDisks); + Map diskOfferingMap = new HashMap<>(dataDiskOfferingMap); + diskOfferingMap.put(rootDisk.getDiskId(), serviceOffering.getDiskOfferingId()); + Map diskOfferingVolumeCountMap = new HashMap<>(); + Map diskOfferingSizeMap = new HashMap<>(); + for (UnmanagedInstanceTO.Disk disk : disks) { + totalVolumes++; + totalVolumesSize += disk.getCapacity(); + Long diskOfferingId = diskOfferingMap.get(disk.getDiskId()); + if (diskOfferingVolumeCountMap.containsKey(diskOfferingId)) { + diskOfferingVolumeCountMap.put(diskOfferingId, diskOfferingVolumeCountMap.get(diskOfferingId) + 1); + diskOfferingSizeMap.put(diskOfferingId, diskOfferingSizeMap.get(diskOfferingId) + disk.getCapacity()); + } else { + diskOfferingVolumeCountMap.put(diskOfferingId, 1L); + diskOfferingSizeMap.put(diskOfferingId, disk.getCapacity()); + } + } + resourceLimitService.checkResourceLimit(account, Resource.ResourceType.volume, totalVolumes); + resourceLimitService.checkResourceLimit(account, Resource.ResourceType.primary_storage, totalVolumesSize); + for (Long diskOfferingId : diskOfferingVolumeCountMap.keySet()) { + List tags = resourceLimitService.getResourceLimitStorageTags(diskOfferingDao.findById(diskOfferingId)); + for (String tag : tags) { + resourceLimitService.checkResourceLimitWithTag(account, Resource.ResourceType.volume, tag, diskOfferingVolumeCountMap.get(diskOfferingId)); + resourceLimitService.checkResourceLimitWithTag(account, Resource.ResourceType.primary_storage, tag, diskOfferingSizeMap.get(diskOfferingId)); + } + } + } + private UserVm importVirtualMachineInternal(final UnmanagedInstanceTO unmanagedInstance, final String instanceName, final DataCenter zone, final Cluster cluster, final HostVO host, final VirtualMachineTemplate template, final String displayName, final String hostName, final Account caller, final Account owner, final Long userId, final ServiceOfferingVO serviceOffering, final Map dataDiskOfferingMap, @@ -1089,8 +1122,8 @@ private UserVm importVirtualMachineInternal(final UnmanagedInstanceTO unmanagedI } } - if (!migrateAllowed && host != null && !hostSupportsServiceOffering(host, validatedServiceOffering)) { - throw new InvalidParameterValueException(String.format("Service offering: %s is not compatible with host: %s of unmanaged VM: %s", serviceOffering.getUuid(), host.getUuid(), instanceName)); + if (!migrateAllowed && !hostSupportsServiceOfferingAndTemplate(host, validatedServiceOffering, template)) { + throw new InvalidParameterValueException(String.format("Service offering: %s or template: %s is not compatible with host: %s of unmanaged VM: %s", serviceOffering.getUuid(), template.getUuid(), host.getUuid(), instanceName)); } // Check disks and supplied disk offerings List unmanagedInstanceDisks = unmanagedInstance.getDisks(); @@ -1119,7 +1152,7 @@ private UserVm importVirtualMachineInternal(final UnmanagedInstanceTO unmanagedI checkUnmanagedDiskAndOfferingForImport(unmanagedInstance.getName(), dataDisks, dataDiskOfferingMap, owner, zone, cluster, migrateAllowed); allDetails.put(VmDetailConstants.DATA_DISK_CONTROLLER, dataDisks.get(0).getController()); } - resourceLimitService.checkResourceLimit(owner, Resource.ResourceType.volume, unmanagedInstanceDisks.size()); + checkUnmanagedDiskLimits(owner, rootDisk, serviceOffering, dataDisks, dataDiskOfferingMap); } catch (ResourceAllocationException e) { LOGGER.error(String.format("Volume resource allocation error for owner: %s", owner.getUuid()), e); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Volume resource allocation error for owner: %s. %s", owner.getUuid(), StringUtils.defaultString(e.getMessage()))); @@ -1203,7 +1236,7 @@ private UserVm importVirtualMachineInternal(final UnmanagedInstanceTO unmanagedI if (migrateAllowed) { userVm = migrateImportedVM(host, template, validatedServiceOffering, userVm, owner, diskProfileStoragePoolList); } - publishVMUsageUpdateResourceCount(userVm, validatedServiceOffering); + publishVMUsageUpdateResourceCount(userVm, validatedServiceOffering, template); return userVm; } @@ -1305,7 +1338,6 @@ private UserVmResponse baseImportInstance(ImportUnmanagedInstanceCmd cmd) { String hostName = getHostNameForImportInstance(cmd.getHostName(), cluster.getHypervisorType(), instanceName, displayName); checkVmwareInstanceNameForImportInstance(cluster.getHypervisorType(), instanceName, hostName, zone); - final Map nicNetworkMap = cmd.getNicNetworkList(); final Map nicIpAddressMap = cmd.getNicIpAddressList(); final Map dataDiskOfferingMap = cmd.getDataDiskToDiskOfferingList(); @@ -2237,7 +2269,7 @@ private UserVm importExternalKvmVirtualMachine(final UnmanagedInstanceTO unmanag cleanupFailedImportVM(userVm); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to import NICs while importing vm: %s. %s", instanceName, StringUtils.defaultString(e.getMessage()))); } - publishVMUsageUpdateResourceCount(userVm, serviceOffering); + publishVMUsageUpdateResourceCount(userVm, serviceOffering, template); return userVm; } @@ -2371,7 +2403,7 @@ private UserVm importKvmVirtualMachineFromDisk(final ImportSource importSource, throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to import volumes while importing vm: %s. %s", instanceName, StringUtils.defaultString(e.getMessage()))); } networkOrchestrationService.importNic(macAddress,0,network, true, userVm, requestedIpPair, true); - publishVMUsageUpdateResourceCount(userVm, serviceOffering); + publishVMUsageUpdateResourceCount(userVm, serviceOffering, template); return userVm; } diff --git a/server/src/test/java/com/cloud/resourcelimit/CheckedReservationTest.java b/server/src/test/java/com/cloud/resourcelimit/CheckedReservationTest.java index b8b35e2ce872..1acc5fc2727b 100644 --- a/server/src/test/java/com/cloud/resourcelimit/CheckedReservationTest.java +++ b/server/src/test/java/com/cloud/resourcelimit/CheckedReservationTest.java @@ -41,6 +41,8 @@ import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.when; +import java.util.List; + @RunWith(MockitoJUnitRunner.class) public class CheckedReservationTest { @@ -76,7 +78,9 @@ public void getId() { lenient().when(reservationDao.persist(Mockito.any())).thenReturn(reservation); lenient().when(reservation.getId()).thenReturn(1L); try (CheckedReservation cr = new CheckedReservation(account, Resource.ResourceType.user_vm,1l, reservationDao, resourceLimitService) ) { - long id = cr.getId(); + List ids = cr.getIds(); + assertEquals(1, cr.getIds().size()); + long id = ids.get(0); assertEquals(1L, id); } catch (NullPointerException npe) { fail("NPE caught"); diff --git a/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java b/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java index 30085193e55e..954c85ec3f20 100644 --- a/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java +++ b/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java @@ -16,14 +16,16 @@ // under the License. package com.cloud.resourcelimit; -import com.cloud.configuration.ResourceLimit; -import com.cloud.vpc.MockResourceLimitManagerImpl; -import junit.framework.TestCase; import org.apache.log4j.Logger; import org.junit.After; import org.junit.Before; import org.junit.Test; +import com.cloud.configuration.ResourceLimit; +import com.cloud.vpc.MockResourceLimitManagerImpl; + +import junit.framework.TestCase; + public class ResourceLimitManagerImplTest extends TestCase { private static final Logger s_logger = Logger.getLogger(ResourceLimitManagerImplTest.class); @@ -92,7 +94,7 @@ private void resourceLimitServiceCall(Long accountId, Long domainId, Integer res String msg = "Update Resource Limit: TEST FAILED"; ResourceLimit result = null; try { - result = _resourceLimitService.updateResourceLimit(accountId, domainId, resourceType, max); + result = _resourceLimitService.updateResourceLimit(accountId, domainId, resourceType, max, null); assertFalse(msg, (result != null || (result == null && max != null && max.longValue() == -1L))); } catch (Exception ex) { fail(msg); diff --git a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java index a41cbb82704e..0bdbe55422b5 100644 --- a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java +++ b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java @@ -893,8 +893,7 @@ public void destroyVolumeIfPossibleTestVolumeStateReady() { private void verifyMocksForTestDestroyVolumeWhenVolumeIsNotInRightState() { Mockito.verify(volumeServiceMock, Mockito.times(0)).destroyVolume(volumeMockId); - Mockito.verify(resourceLimitServiceMock, Mockito.times(0)).decrementResourceCount(accountMockId, ResourceType.volume, true); - Mockito.verify(resourceLimitServiceMock, Mockito.times(0)).decrementResourceCount(accountMockId, ResourceType.primary_storage, true, volumeSizeMock); + Mockito.verify(resourceLimitServiceMock, Mockito.times(0)).decrementVolumeResourceCount(accountMockId, true, volumeSizeMock, newDiskOfferingMock); } private void configureMocksForTestDestroyVolumeWhenVolume() { @@ -902,8 +901,7 @@ private void configureMocksForTestDestroyVolumeWhenVolume() { Mockito.lenient().doReturn(true).when(volumeVoMock).isDisplayVolume(); Mockito.lenient().doNothing().when(volumeServiceMock).destroyVolume(volumeMockId); - Mockito.lenient().doNothing().when(resourceLimitServiceMock).decrementResourceCount(accountMockId, ResourceType.volume, true); - Mockito.lenient().doNothing().when(resourceLimitServiceMock).decrementResourceCount(accountMockId, ResourceType.primary_storage, true, volumeSizeMock); + Mockito.lenient().doNothing().when(resourceLimitServiceMock).decrementVolumeResourceCount(accountMockId, true, volumeSizeMock, newDiskOfferingMock); } @Test @@ -1449,22 +1447,21 @@ public void updateVolumeAccountTest() { Account newAccountMock = new AccountVO(accountMockId + 1); Mockito.doReturn(volumeVoMock).when(volumeDaoMock).persist(volumeVoMock); + Mockito.when(_diskOfferingDao.findById(Mockito.anyLong())).thenReturn(newDiskOfferingMock); volumeApiServiceImpl.updateVolumeAccount(accountMock, volumeVoMock, newAccountMock); usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_DELETE, volumeVoMock.getAccountId(), volumeVoMock.getDataCenterId(), volumeVoMock.getId(), volumeVoMock.getName(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.isDisplayVolume())); - Mockito.verify(resourceLimitServiceMock).decrementResourceCount(accountMock.getAccountId(), ResourceType.volume); - Mockito.verify(resourceLimitServiceMock).decrementResourceCount(accountMock.getAccountId(), ResourceType.primary_storage, volumeVoMock.getSize()); + Mockito.verify(resourceLimitServiceMock).decrementVolumeResourceCount(accountMock.getAccountId(), true, volumeVoMock.getSize(), newDiskOfferingMock); Mockito.verify(volumeVoMock).setAccountId(newAccountMock.getAccountId()); Mockito.verify(volumeVoMock).setDomainId(newAccountMock.getDomainId()); Mockito.verify(volumeDaoMock).persist(volumeVoMock); - Mockito.verify(resourceLimitServiceMock).incrementResourceCount(newAccountMock.getAccountId(), ResourceType.volume); - Mockito.verify(resourceLimitServiceMock).incrementResourceCount(newAccountMock.getAccountId(), ResourceType.primary_storage, volumeVoMock.getSize()); + Mockito.verify(resourceLimitServiceMock).incrementVolumeResourceCount(newAccountMock.getAccountId(), true, volumeVoMock.getSize(), newDiskOfferingMock); usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_DELETE, volumeVoMock.getAccountId(), volumeVoMock.getDataCenterId(), volumeVoMock.getId(), volumeVoMock.getName(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.isDisplayVolume())); diff --git a/server/src/test/java/com/cloud/user/AccountManagerImplVolumeDeleteEventTest.java b/server/src/test/java/com/cloud/user/AccountManagerImplVolumeDeleteEventTest.java index 21474a53ee71..b71285988ed4 100644 --- a/server/src/test/java/com/cloud/user/AccountManagerImplVolumeDeleteEventTest.java +++ b/server/src/test/java/com/cloud/user/AccountManagerImplVolumeDeleteEventTest.java @@ -145,8 +145,6 @@ protected void defineMocksBehavior() throws AgentUnavailableException, Concurren ServiceOfferingVO offering = mock(ServiceOfferingVO.class); lenient().when(offering.getCpu()).thenReturn(500); lenient().when(offering.getId()).thenReturn(1l); - when(offering.getCpu()).thenReturn(500); - when(offering.getRamSize()).thenReturn(500); when(serviceOfferingDao.findByIdIncludingRemoved(nullable(Long.class), nullable(Long.class))).thenReturn(offering); lenient().when(_domainMgr.getDomain(nullable(Long.class))).thenReturn(domain); diff --git a/server/src/test/java/com/cloud/vpc/MockResourceLimitManagerImpl.java b/server/src/test/java/com/cloud/vpc/MockResourceLimitManagerImpl.java index 3b29b3b7229e..e633816a1e20 100644 --- a/server/src/test/java/com/cloud/vpc/MockResourceLimitManagerImpl.java +++ b/server/src/test/java/com/cloud/vpc/MockResourceLimitManagerImpl.java @@ -16,31 +16,41 @@ // under the License. package com.cloud.vpc; + +import java.util.List; +import java.util.Map; + +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.api.response.AccountResponse; +import org.apache.cloudstack.api.response.DomainResponse; +import org.springframework.stereotype.Component; + import com.cloud.configuration.Resource.ResourceType; import com.cloud.configuration.ResourceCount; import com.cloud.configuration.ResourceLimit; import com.cloud.domain.Domain; import com.cloud.exception.ResourceAllocationException; +import com.cloud.offering.DiskOffering; +import com.cloud.offering.ServiceOffering; +import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; import com.cloud.user.ResourceLimitService; import com.cloud.utils.component.ManagerBase; -import com.cloud.utils.exception.CloudRuntimeException; -import org.apache.cloudstack.user.ResourceReservation; -import org.springframework.stereotype.Component; - -import javax.naming.ConfigurationException; -import java.util.List; -import java.util.Map; @Component public class MockResourceLimitManagerImpl extends ManagerBase implements ResourceLimitService { /* (non-Javadoc) - * @see com.cloud.user.ResourceLimitService#updateResourceLimit(java.lang.Long, java.lang.Long, java.lang.Integer, java.lang.Long) + * @see com.cloud.user.ResourceLimitService#updateResourceLimit(java.lang.Long, java.lang.Long, java.lang.Integer, java.lang.Long, java.lang.String) */ @Override - public ResourceLimit updateResourceLimit(Long accountId, Long domainId, Integer resourceType, Long max) { - // TODO Auto-generated method stub + public ResourceLimit updateResourceLimit(Long accountId, Long domainId, Integer resourceType, Long max, String tag) { + return null; + } + + @Override + public List recalculateResourceCount(Long accountId, Long domainId, Integer typeId, String tag) { return null; } @@ -57,16 +67,16 @@ public List recalculateResourceCount(Long accountId, Lo * @see com.cloud.user.ResourceLimitService#searchForLimits(java.lang.Long, java.lang.Long, java.lang.Long, com.cloud.user.ResourceLimitService, java.lang.Long, java.lang.Long) */ @Override - public List searchForLimits(Long id, Long accountId, Long domainId, ResourceType resourceType, Long startIndex, Long pageSizeVal) { + public List searchForLimits(Long id, Long accountId, Long domainId, ResourceType resourceType, String tag, Long startIndex, Long pageSizeVal) { // TODO Auto-generated method stub return null; } /* (non-Javadoc) - * @see com.cloud.user.ResourceLimitService#findCorrectResourceLimitForAccount(com.cloud.user.Account, com.cloud.configuration.Resource.ResourceType) + * @see com.cloud.user.ResourceLimitService#findCorrectResourceLimitForAccount(com.cloud.user.Account, com.cloud.configuration.Resource.ResourceType, java.lang.String) */ @Override - public long findCorrectResourceLimitForAccount(Account account, ResourceType type) { + public long findCorrectResourceLimitForAccount(Account account, ResourceType type, String tag) { // TODO Auto-generated method stub return 0; } @@ -78,10 +88,10 @@ public long findCorrectResourceLimitForAccount(long accountId, Long limit, Resou } /* (non-Javadoc) - * @see com.cloud.user.ResourceLimitService#findCorrectResourceLimitForDomain(com.cloud.domain.Domain, com.cloud.configuration.Resource.ResourceType) + * @see com.cloud.user.ResourceLimitService#findCorrectResourceLimitForDomain(com.cloud.domain.Domain, com.cloud.configuration.Resource.ResourceType, java.lang.String) */ @Override - public long findCorrectResourceLimitForDomain(Domain domain, ResourceType type) { + public long findCorrectResourceLimitForDomain(Domain domain, ResourceType type, String tag) { // TODO Auto-generated method stub return 0; } @@ -92,7 +102,7 @@ public long findDefaultResourceLimitForDomain(ResourceType resourceType) { } @Override - public long findCorrectResourceLimitForAccountAndDomain(Account account, Domain domain, ResourceType type) { + public long findCorrectResourceLimitForAccountAndDomain(Account account, Domain domain, ResourceType type, String tag) { return 0; } @@ -148,10 +158,10 @@ public long calculateSecondaryStorageForAccount(long accountId) { } /* (non-Javadoc) - * @see com.cloud.user.ResourceLimitService#getResourceCount(com.cloud.user.Account, com.cloud.configuration.Resource.ResourceType) + * @see com.cloud.user.ResourceLimitService#getResourceCount(com.cloud.user.Account, com.cloud.configuration.Resource.ResourceType, java.lang.String) */ @Override - public long getResourceCount(Account account, ResourceType type) { + public long getResourceCount(Account account, ResourceType type, String tag) { // TODO Auto-generated method stub return 0; } @@ -176,11 +186,6 @@ public void decrementResourceCount(long accountId, ResourceType type, Boolean di //To change body of implemented methods use File | Settings | File Templates. } - @Override - public ResourceReservation getReservation(Account account, Boolean displayResource, ResourceType type, Long delta) { - throw new CloudRuntimeException("no reservation implemented for mock resource management."); - } - /* (non-Javadoc) * @see com.cloud.utils.component.Manager#configure(java.lang.String, java.util.Map) */ @@ -217,4 +222,118 @@ public String getName() { return null; } + @Override + public void incrementResourceCountWithTag(long accountId, ResourceType type, String tag, Long... delta) { + + } + + @Override + public void decrementResourceCountWithTag(long accountId, ResourceType type, String tag, Long... delta) { + + } + + @Override + public void checkResourceLimitWithTag(Account account, ResourceType type, String tag, long... count) throws ResourceAllocationException { + + } + + @Override + public List getResourceLimitHostTags() { + return null; + } + + @Override + public List getResourceLimitStorageTags() { + return null; + } + + @Override + public void updateTaggedResourceLimitsAndCountsForAccounts(List responses, String tag) { + + } + + @Override + public void updateTaggedResourceLimitsAndCountsForDomains(List responses, String tag) { + + } + + @Override + public List getResourceLimitHostTags(ServiceOffering serviceOffering, VirtualMachineTemplate template) { + return null; + } + + @Override + public List getResourceLimitStorageTags(DiskOffering diskOffering) { + return null; + } + + @Override + public void checkVolumeResourceLimit(Account owner, Boolean display, Long size, DiskOffering diskOffering) throws ResourceAllocationException { + + } + + @Override + public void incrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering) { + + } + + @Override + public void decrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering) { + + } + + @Override + public void incrementVolumePrimaryStorageResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering) { + + } + + @Override + public void decrementVolumePrimaryStorageResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering) { + + } + + @Override + public void checkVmResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template) throws ResourceAllocationException { + + } + + @Override + public void incrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template) { + + } + + @Override + public void decrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template) { + + } + + @Override + public void checkVmCpuResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long cpu) throws ResourceAllocationException { + + } + + @Override + public void incrementVmCpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long cpu) { + + } + + @Override + public void decrementVmCpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long cpu) { + + } + + @Override + public void checkVmMemoryResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory) throws ResourceAllocationException { + + } + + @Override + public void incrementVmMemoryResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory) { + + } + + @Override + public void decrementVmMemoryResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory) { + + } } diff --git a/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java index e78319983531..9c3fcc40c944 100644 --- a/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java @@ -300,7 +300,6 @@ public void setUp() throws Exception { HostVO hostVO = Mockito.mock(HostVO.class); when(hostVO.isInMaintenanceStates()).thenReturn(false); hosts.add(hostVO); - when(hostVO.checkHostServiceOfferingTags(Mockito.any())).thenReturn(true); when(resourceManager.listHostsInClusterByStatus(Mockito.anyLong(), Mockito.any(Status.class))).thenReturn(hosts); when(resourceManager.listAllUpAndEnabledHostsInOneZoneByHypervisor(any(Hypervisor.HypervisorType.class), Mockito.anyLong())).thenReturn(hosts); List templates = new ArrayList<>(); diff --git a/test/integration/component/test_resource_limit_tags.py b/test/integration/component/test_resource_limit_tags.py new file mode 100644 index 000000000000..feb5c7820e22 --- /dev/null +++ b/test/integration/component/test_resource_limit_tags.py @@ -0,0 +1,648 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" BVT tests for resource limit tags functionalities +""" +# Import Local Modules +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.cloudstackAPI import (listCapacity, + listResourceLimits, + updateResourceLimit, + updateResourceCount) +from marvin.lib.base import (Host, + StoragePool, + Account, + Domain, + Zone, + ServiceOffering, + DiskOffering, + VirtualMachine, + Volume, + Configurations) +from marvin.lib.common import (get_domain, + get_zone, + get_template) +from marvin.codes import FAILED +from marvin.cloudstackException import CloudstackAPIException +from nose.plugins.attrib import attr +import logging +# Import System modules +import math +import random + + +_multiprocess_shared_ = True +MAX_VM_LIMIT = 2 +MAX_RAM_VM_LIMIT = 3 +MAX_DATA_VOLUME_LIMIT = 3 +MAX_PS_DATA_VOLUME_LIMIT = 2 + +class TestResourceLimitTags(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + testClient = super(TestResourceLimitTags, cls).getClsTestClient() + cls.apiclient = testClient.getApiClient() + cls.services = testClient.getParsedTestDataConfig() + + # Get Zone, Domain and templates + cls.domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests()) + cls.services['mode'] = cls.zone.networktype + + cls._cleanup = [] + cls.logger = logging.getLogger('TestResourceLimitTags') + + template = get_template( + cls.apiclient, + cls.zone.id, + cls.services["ostype"]) + if template == FAILED: + assert False, "get_template() failed to return template with description %s" % cls.services["ostype"] + + # Set Zones and disk offerings + cls.services["small"]["zoneid"] = cls.zone.id + cls.services["small"]["template"] = template.id + + cls.host_tags = ['htag1', 'htag2', 'htag3'] + cls.host_tags_supporting_types = [0, 8, 9] + Configurations.update(cls.apiclient, + "resource.limit.host.tags", + value=','.join(cls.host_tags) + ) + cls.storage_tags = ['stag1', 'stag2', 'stag3'] + cls.storage_tags_supporting_types = [2, 10] + Configurations.update(cls.apiclient, + "resource.limit.storage.tags", + value=','.join(cls.storage_tags) + ) + + hosts = Host.list(cls.apiclient, type='Routing') + cls.original_host_tag_map = {} + for idx, host in enumerate(hosts): + cls.original_host_tag_map[host.id] = host.hosttags + if idx % 2 == 0: + Host.update(cls.apiclient, id=host.id, hosttags=cls.host_tags[1]) + else: + Host.update(cls.apiclient, id=host.id, hosttags='') + + pools = StoragePool.list(cls.apiclient) + cls.original_storage_pool_tag_map = {} + for idx, pool in enumerate(pools): + cls.original_storage_pool_tag_map[pool.id] = pool.tags + if idx % 2 == 0: + StoragePool.update(cls.apiclient, id=pool.id, tags=cls.storage_tags[1]) + else: + StoragePool.update(cls.apiclient, id=pool.id, tags='') + + + cls.untagged_compute_offering = ServiceOffering.create( + cls.apiclient, + cls.services["service_offerings"]["tiny"]) + cls._cleanup.append(cls.untagged_compute_offering) + + host_tagged_compute_offering_service = cls.services["service_offerings"]["tiny"].copy() + host_tagged_compute_offering_service["hosttags"] = cls.host_tags[1] + cls.host_tagged_compute_offering = ServiceOffering.create( + cls.apiclient, + host_tagged_compute_offering_service) + cls._cleanup.append(cls.host_tagged_compute_offering) + + host_storage_tagged_compute_offering_service = cls.services["service_offerings"]["tiny"].copy() + host_storage_tagged_compute_offering_service["hosttags"] = cls.host_tags[1] + host_storage_tagged_compute_offering_service["tags"] = cls.storage_tags[1] + cls.host_storage_tagged_compute_offering = ServiceOffering.create( + cls.apiclient, + host_storage_tagged_compute_offering_service) + cls._cleanup.append(cls.host_storage_tagged_compute_offering) + + cls.untagged_disk_offering = DiskOffering.create( + cls.apiclient, + cls.services["disk_offering"] + ) + cls._cleanup.append(cls.untagged_disk_offering) + + tagged_disk_offering_service = cls.services["disk_offering"].copy() + tagged_disk_offering_service["tags"] = cls.storage_tags[1] + cls.tagged_disk_offering = DiskOffering.create( + cls.apiclient, + tagged_disk_offering_service + ) + cls._cleanup.append(cls.tagged_disk_offering) + + cls.services["virtual_machine"]["zoneid"] = cls.zone.id + cls.services["virtual_machine"]["template"] = template.id + + @classmethod + def tearDownClass(cls): + for host_id in cls.original_host_tag_map: + tag = cls.original_host_tag_map[host_id] + if tag is None: + tag = '' + Host.update(cls.apiclient, id=host_id, hosttags=tag) + for pool_id in cls.original_storage_pool_tag_map: + tag = cls.original_storage_pool_tag_map[pool_id] + if tag is None: + tag = '' + StoragePool.update(cls.apiclient, id=pool_id, tags=tag) + super(TestResourceLimitTags, cls).tearDownClass() + + def setUp(self): + self.cleanup = [] + self.tag_type = random.choice(['host', 'storage']) + self.domain1 = Domain.create( + self.apiclient, + self.services["domain"]) + self.cleanup.append(self.domain1) + self.account = Account.create( + self.apiclient, + self.services["account"], + domainid=self.domain1.id) + self.cleanup.append(self.account) + self.userapiclient = self.testClient.getUserApiClient( + UserName=self.account.name, + DomainName=self.account.domain + ) + + def tearDown(self): + super(TestResourceLimitTags, self).tearDown() + + def check_entity_tagged_resource_count(self, taggedresources): + self.assertNotEqual( + taggedresources, + None, + "Check tagged resources list" + ) + for type in self.host_tags_supporting_types: + filtered_limits = list(filter(lambda x: x.resourcetype == type, taggedresources)) + self.assertEqual( + len(filtered_limits), + len(self.host_tags) + ) + for limit in filtered_limits: + self.assertTrue(limit.tag in self.host_tags) + for type in self.storage_tags_supporting_types: + filtered_limits = list(filter(lambda x: x.resourcetype == type, taggedresources)) + self.assertEqual( + len(filtered_limits), + len(self.storage_tags) + ) + for limit in filtered_limits: + self.assertTrue(limit.tag in self.storage_tags) + + def verify_entity_resource_limits(self, limits, resource_type, tag, max): + if type(limits) is list: + if len(limits) == 0: + self.fail("Empty limits list") + limits = limits[0] + self.assertNotEqual( + limits, + None, + "Check tagged limits list" + ) + self.assertEqual(max, + limits.max, + "Max value not equal" + ) + self.assertEqual(tag, + limits.tag, + "Tag value not equal" + ) + self.assertEqual(str(resource_type), + limits.resourcetype, + "Resource type value not equal" + ) + + def update_domain_account_tagged_limit(self, resource_type, tag, max, for_account=False): + cmd = updateResourceLimit.updateResourceLimitCmd() + cmd.domainid = self.domain1.id + if for_account: + cmd.account = self.account.name + cmd.resourcetype = resource_type + cmd.tag = tag + cmd.max = max + response = self.apiclient.updateResourceLimit(cmd) + return response + + def list_domain_account_tagged_limit(self, resource_type, tag, for_account=False): + cmd = listResourceLimits.listResourceLimitsCmd() + cmd.domainid = self.domain1.id + if for_account: + cmd.account = self.account.name + cmd.resourcetype = resource_type + cmd.tag = tag + response = self.apiclient.listResourceLimits(cmd) + return response + + def run_test_update_domain_account_tagged_limit(self, for_account=False): + tags = self.host_tags + resource_types = self.host_tags_supporting_types + if self.tag_type == 'storage': + tags = self.storage_tags + resource_types = self.storage_tags_supporting_types + tag = random.choice(tags) + resource_type = random.choice(resource_types) + max = random.randrange(5, 10) + + response = self.update_domain_account_tagged_limit(resource_type, tag, max, for_account) + self.verify_entity_resource_limits(response, resource_type, tag, max) + + response = self.list_domain_account_tagged_limit(resource_type, tag, for_account) + self.verify_entity_resource_limits(response, resource_type, tag, max) + + def run_test_domain_account_tagged_vm_limit(self, resource_type, max): + counter = 0 + increment = 1 + if resource_type == 8: + increment = 100 + elif resource_type == 9: + increment = 128 + vm_to_delete = None + while counter < max: + self.vm = VirtualMachine.create( + self.userapiclient, + self.services["virtual_machine"], + serviceofferingid=self.host_tagged_compute_offering.id, + mode=self.services["mode"] + ) + self.cleanup.append(self.vm) + counter = counter + increment + if vm_to_delete is None: + vm_to_delete = self.vm + # Tagged VM shouldn't be deployed + try: + self.vm2 = VirtualMachine.create( + self.userapiclient, + self.services["virtual_machine"], + serviceofferingid=self.host_tagged_compute_offering.id, + mode=self.services["mode"] + ) + self.cleanup.append(self.vm2) + self.fail("VM deployed over tagged limit for domain/account") + except CloudstackAPIException as e: + self.logger.debug("Over tagged limit VM for domain/account deployment failed with : %s" % e) + # Untagged VM should be deployed fine + self.vm3 = VirtualMachine.create( + self.userapiclient, + self.services["virtual_machine"], + serviceofferingid=self.untagged_compute_offering.id, + mode=self.services["mode"] + ) + self.cleanup.append(self.vm3) + # Delete one of the tagged VMs and check if a new tagged VM is deployed + vm_to_delete.delete(self.apiclient, expunge=True) + for idx, x in enumerate(self.cleanup): + if x.id == vm_to_delete.id: + self.cleanup.pop(idx) + break + self.vm = VirtualMachine.create( + self.userapiclient, + self.services["virtual_machine"], + serviceofferingid=self.host_tagged_compute_offering.id, + mode=self.services["mode"] + ) + self.cleanup.append(self.vm) + + def run_test_domain_account_tagged_volume_limit(self, resource_type, max): + increment = 1 + if resource_type == 10: + increment = 1*1024*1024*1024 + volume_to_delete = None + self.vm = VirtualMachine.create( + self.userapiclient, + self.services["virtual_machine"], + serviceofferingid=self.host_storage_tagged_compute_offering.id, + mode=self.services["mode"] + ) + self.cleanup.append(self.vm) + counter = 1 + if resource_type == 10: + counter = self.root_volume_size + while counter < max: + self.volume = Volume.create( + self.userapiclient, + self.services["volume"], + diskofferingid=self.tagged_disk_offering.id, + zoneid=self.zone.id + ) + self.cleanup.append(self.volume) + counter = counter + increment + if volume_to_delete is None: + volume_to_delete = self.volume + # Tagged VM shouldn't be deployed + try: + self.volume2 = Volume.create( + self.userapiclient, + self.services["volume"], + diskofferingid=self.tagged_disk_offering.id, + zoneid=self.zone.id + ) + self.cleanup.append(self.volume2) + self.fail("Volume created over tagged limit for domain/account") + except CloudstackAPIException as e: + self.logger.debug("Over tagged limit volume for domain/account deployment failed with : %s" % e) + # Untagged VM should be deployed fine + self.volume3 = Volume.create( + self.userapiclient, + self.services["volume"], + diskofferingid=self.untagged_disk_offering.id, + zoneid=self.zone.id + ) + self.cleanup.append(self.volume3) + # Delete one of the tagged volumes and check if a new tagged volume is deployed + volume_to_delete.delete(self.apiclient) + for idx, x in enumerate(self.cleanup): + if x.id == volume_to_delete.id: + self.cleanup.pop(idx) + break + self.volume = Volume.create( + self.userapiclient, + self.services["volume"], + diskofferingid=self.tagged_disk_offering.id, + zoneid=self.zone.id + ) + self.cleanup.append(self.volume) + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_01_check_list_capacity(self): + """Test to verify listing capacity with tags + """ + # Validate the following: + # 1. List capacity with a valid tag + # 2. Verify + + cmd = listCapacity.listCapacityCmd() + cmd.tag = self.host_tags[1] + response = self.apiclient.listCapacity(cmd) + self.assertNotEqual( + response, + None, + "Check capacity is listed" + ) + self.assertTrue(len(response) > 0) + cmd.tag = "INVALID" + response = self.apiclient.listCapacity(cmd) + self.assertEqual( + response, + None, + "Check capacity is listed incorrectly" + ) + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_02_check_list_domain_tagged_limit(self): + """Test to verify listing domain tagged resource counts + """ + + domain = Domain.list( + self.apiclient, + id = self.domain1.id + )[0] + self.check_entity_tagged_resource_count(domain.taggedresources) + return + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_03_check_list_account_tagged_limit(self): + """Test to verify listing account tagged resource counts + """ + + account = Account.list( + self.apiclient, + id = self.account.id + )[0] + self.check_entity_tagged_resource_count(account.taggedresources) + return + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_04_check_update_domain_tagged_limit(self): + """Test to verify listing domain tagged resource limits + """ + self.run_test_update_domain_account_tagged_limit() + return + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_05_check_update_account_tagged_limit(self): + """Test to verify listing domain tagged resource limits + """ + self.run_test_update_domain_account_tagged_limit(True) + return + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_06_verify_domain_tagged_vm_limit(self): + """Test to verify domain tagged resource limits working + 1. Check if VM(s) can be deployed within tagged limits for domain + 2. Check if VM(s) can not be deployed over tagged limits for domain + 3. Check if VM(s) can be deployed without tag for domain + """ + resource_type = 0 + tag = self.host_tags[1] + max = MAX_VM_LIMIT + self.update_domain_account_tagged_limit(resource_type, tag, max) + self.run_test_domain_account_tagged_vm_limit(resource_type, max) + return + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_07_verify_account_tagged_vm_limit(self): + """Test to verify account tagged resource limits working + 1. Check if VM(s) can be deployed within tagged limits for account + 2. Check if VM(s) can not be deployed over tagged limits for account + 3. Check if VM(s) can be deployed without tag for account + """ + resource_type = 9 + tag = self.host_tags[1] + max = self.host_tagged_compute_offering.memory * MAX_RAM_VM_LIMIT + self.update_domain_account_tagged_limit(resource_type, tag, max) + self.run_test_domain_account_tagged_vm_limit(resource_type, max) + return + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_08_verify_domain_tagged_volume_limit(self): + """Test to verify domain tagged resource limits working + 1. Check if volume(s) can be deployed within tagged limits for domain + 2. Check if volume(s) can not be deployed over tagged limits for domain + 3. Check if volume(s) can be deployed without tag for domain + """ + resource_type = 2 + tag = self.storage_tags[1] + max = MAX_DATA_VOLUME_LIMIT + self.update_domain_account_tagged_limit(resource_type, tag, max) + self.run_test_domain_account_tagged_volume_limit(resource_type, max) + return + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_09_verify_account_tagged_volume_limit(self): + """Test to verify account tagged resource limits working + 1. Check if volume(s) can be deployed within tagged limits for domain + 2. Check if volume(s) can not be deployed over tagged limits for domain + 3. Check if volume(s) can be deployed without tag for domain + """ + self.test_vm = VirtualMachine.create( + self.userapiclient, + self.services["virtual_machine"], + serviceofferingid=self.untagged_compute_offering.id, + mode=self.services["mode"] + ) + self.cleanup.append(self.test_vm) + volume = Volume.list( + self.userapiclient, + virtualmachineid=self.test_vm.id, + listall=True, + type='ROOT' + )[0] + self.root_volume_size = volume['size'] + resource_type = 10 + tag = self.storage_tags[1] + max = math.ceil(self.root_volume_size/(1024*1024*1024)) + MAX_PS_DATA_VOLUME_LIMIT + self.update_domain_account_tagged_limit(resource_type, tag, max, True) + self.run_test_domain_account_tagged_volume_limit(resource_type, (max*1024*1024*1024)) + return + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_10_verify_assign_vm_limit(self): + """Test to verify limits are updated on changing VM owner + """ + self.donor_account = Account.create( + self.apiclient, + self.services["account"], + domainid=self.domain1.id) + self.cleanup.append(self.donor_account) + # Pass mode=basic to avoid creation of PF rules for the VM + self.vm = VirtualMachine.create( + self.userapiclient, + self.services["virtual_machine"], + serviceofferingid=self.host_storage_tagged_compute_offering.id, + mode='basic' + ) + self.cleanup.append(self.vm) + self.vm.stop(self.userapiclient) + acc = Account.list( + self.userapiclient, + id=self.account.id + )[0] + tags = [self.host_storage_tagged_compute_offering.hosttags, self.host_storage_tagged_compute_offering.storagetags] + source_account_usage_before = list(filter(lambda x: x.tag in tags, acc['taggedresources'])) + self.vm.assign_virtual_machine(self.apiclient, self.donor_account.name ,self.domain1.id) + acc = Account.list( + self.userapiclient, + id=self.account.id + )[0] + source_account_usage_after = list(filter(lambda x: x.tag in tags, acc['taggedresources'])) + for usage in source_account_usage_after: + self.assertTrue(usage.total == 0, "Usage for %s with tag %s is not zero for source account" % (usage.resourcetypename, usage.tag)) + acc = Account.list( + self.apiclient, + id=self.donor_account.id + )[0] + target_account_usage = list(filter(lambda x: x.tag in tags, acc['taggedresources'])) + for idx, usage in enumerate(target_account_usage): + expected_usage = source_account_usage_before[idx] + self.assertTrue(usage.total == expected_usage.total, "Usage for %s with tag %s is not matching for target account" % (usage.resourcetypename, usage.tag)) + return + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_11_verify_scale_vm_limit(self): + """Test to verify limits are updated on scaling VM + """ + scale_compute_offering_service = self.services["service_offerings"]["tiny"].copy() + scale_compute_offering_service["cpunumber"] = 2 * self.host_tagged_compute_offering.cpunumber + scale_compute_offering_service["memory"] = 2 * self.host_tagged_compute_offering.memory + scale_compute_offering_service["hosttags"] = self.host_tagged_compute_offering.hosttags + self.scale_compute_offering = ServiceOffering.create( + self.apiclient, + scale_compute_offering_service) + self.cleanup.append(self.scale_compute_offering) + + self.vm = VirtualMachine.create( + self.userapiclient, + self.services["virtual_machine"], + serviceofferingid=self.host_tagged_compute_offering.id, + mode=self.services["mode"] + ) + self.cleanup.append(self.vm) + self.vm.stop(self.userapiclient) + acc = Account.list( + self.userapiclient, + id=self.account.id + )[0] + tags = [self.host_tagged_compute_offering.hosttags] + account_usage_before = list(filter(lambda x: x.tag in tags, acc['taggedresources'])) + self.vm.scale(self.userapiclient, self.scale_compute_offering.id) + acc = Account.list( + self.userapiclient, + id=self.account.id + )[0] + account_usage_after = list(filter(lambda x: x.tag in tags, acc['taggedresources'])) + for idx, usage in enumerate(account_usage_after): + expected_usage_total = account_usage_before[idx].total + if usage.resourcetype in [8, 9]: + expected_usage_total = 2 * expected_usage_total + self.assertTrue(usage.total == expected_usage_total, "Usage for %s with tag %s is not matching for target account" % (usage.resourcetypename, usage.tag)) + return + + @attr(tags=["devcloud", "advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + def test_12_verify_scale_volume_limit(self): + """Test to verify limits are updated on scaling volume + """ + scale_disk_offering_service = self.services["disk_offering"].copy() + scale_disk_offering_service["tags"] = self.tagged_disk_offering.tags + scale_disk_offering_service["disksize"] = 2 * self.tagged_disk_offering.disksize + self.scale_disk_offering = DiskOffering.create( + self.apiclient, + scale_disk_offering_service + ) + self.cleanup.append(self.scale_disk_offering) + + self.vm = VirtualMachine.create( + self.userapiclient, + self.services["virtual_machine"], + serviceofferingid=self.host_tagged_compute_offering.id, + mode=self.services["mode"] + ) + self.cleanup.append(self.vm) + self.volume = Volume.create( + self.userapiclient, + self.services["volume"], + diskofferingid=self.tagged_disk_offering.id, + zoneid=self.zone.id + ) + self.vm.attach_volume( + self.userapiclient, + volume=self.volume + ) + self.vm.detach_volume( + self.userapiclient, + volume=self.volume + ) + acc = Account.list( + self.userapiclient, + id=self.account.id + )[0] + tags = [self.tagged_disk_offering.tags] + account_usage_before = list(filter(lambda x: x.tag in tags, acc['taggedresources'])) + self.volume.resize( + self.userapiclient, + diskofferingid=self.scale_disk_offering.id + ) + acc = Account.list( + self.userapiclient, + id=self.account.id + )[0] + account_usage_after = list(filter(lambda x: x.tag in tags, acc['taggedresources'])) + for idx, usage in enumerate(account_usage_after): + expected_usage_total = account_usage_before[idx].total + if usage.resourcetype in [10]: + expected_usage_total = 2 * expected_usage_total + self.assertTrue(usage.total == expected_usage_total, "Usage for %s with tag %s is not matching for target account" % (usage.resourcetypename, usage.tag)) + return diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 7de4c7092256..be6ce95004f2 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -1260,6 +1260,7 @@ "label.management.servers": "Management servers", "label.managementservers": "Number of management servers", "label.matchall": "Match all", +"label.max": "Max.", "label.max.primary.storage": "Max. primary (GiB)", "label.max.secondary.storage": "Max. secondary (GiB)", "label.max.migrations": "Max. migrations", @@ -2044,6 +2045,7 @@ "label.tag.key": "Tag key", "label.tag.value": "Tag value", "label.tagged": "Tagged", +"label.tagged.limits": "Tagged limits", "label.tags": "Tags", "label.tag.type": "Tag Type", "label.tagtypeuuid": "Tag Type", @@ -2054,9 +2056,10 @@ "label.tariffvalue": "Tariff value", "label.tcp": "TCP", "label.tcp.proxy": "TCP proxy", -"label.template": "Select a Template", -"label.template.select.existing": "Select an existing Template", -"label.template.temporary.import": "Use a temporary Template for import", +"label.template": "Select a template", +"label.templatetag": "Tag", +"label.template.select.existing": "Select an existing template", +"label.template.temporary.import": "Use a temporary template for import", "label.templatebody": "Body", "label.templatefileupload": "Local file", "label.templateid": "Select a Template", @@ -3254,7 +3257,8 @@ "message.update.ipaddress.processing": "Updating IP Address...", "message.update.resource.count": "Please confirm that you want to update resource counts for this Account.", "message.update.resource.count.domain": "Please confirm that you want to update resource counts for this domain.", -"message.update.ssl": "Please submit a new X.509 compliant SSL certificate chain to be updated in each console proxy and secondary storage virtual Instance:", +"message.update.resource.limit.max.untagged.error": "%x untagged limit is %y. Please specify limit for tag '%z' less than or equal to that", +"message.update.ssl": "Please submit a new X.509 compliant SSL certificate chain to be updated in each console proxy and secondary storage virtual instance:", "message.upload.failed": "Upload Failed", "message.upload.file.limit": "Only one file can be uploaded at a time.", "message.upload.file.processing": "Please do not close this form or refresh your browser, file upload is in progress...", diff --git a/ui/src/components/view/ResourceCountUsage.vue b/ui/src/components/view/ResourceCountUsage.vue index ecd24a18291a..dc0b4cada33f 100644 --- a/ui/src/components/view/ResourceCountUsage.vue +++ b/ui/src/components/view/ResourceCountUsage.vue @@ -36,6 +36,37 @@ :percent="parseFloat(getPercentUsed(resource[item + 'total'], resource[item + 'limit']))" :format="p => resource[item + 'limit'] !== '-1' && resource[item + 'limit'] !== 'Unlimited' ? p.toFixed(2) + '%' : ''" /> + + + + + + + @@ -43,6 +74,8 @@