diff --git a/.gitignore b/.gitignore index d38171e..0ceb9f9 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ __pycache__ .odo/ .odo/env .odo/odo-file-index.json +venv diff --git a/Containerfile b/Containerfile index b418ccd..d5a3706 100644 --- a/Containerfile +++ b/Containerfile @@ -1,4 +1,4 @@ -FROM quay.io/rhpds/python-kopf-s2i:v1.38 +FROM quay.io/rhpds/python-kopf-s2i:v1.43 USER root diff --git a/Development.adoc b/Development.adoc index 0907909..f09dead 100644 --- a/Development.adoc +++ b/Development.adoc @@ -1,22 +1,11 @@ # Poolboy Development -Poolboy development can be performed in the `odo` OpenShift developer CLI or building with OpenShift build configs. -An OpenShift cluster with cluster-admin is required for `odo` development. -https://developers.redhat.com/products/codeready-containers/overview[CodeReady Containers] is recommended for local development. -An Ansible test suite is available for functional testing. - -## Development with `odo` - -Use of `odo` is recommended for fast iterative development. -`odo` simplifies the build/deploy process and avoids creating unnecessary build artifacts during the development process. - -. Install the `odo` developer CLI as described in the OpenShift documentation: -https://docs.openshift.com/container-platform/latest/cli_reference/developer_cli_odo/installing-odo.html[Installing odo] +## Local Development . Create poolboy local dev resources from the provided helm chart using `poolboy.dev.local` as the operator domain: + --------------------------------------------- -helm template helm \ +helm template helm --include-crds \ --set deploy=false \ --set admin.deploy=false \ --set nameOverride=poolboy-dev \ @@ -24,32 +13,28 @@ helm template helm \ | oc apply -f - --------------------------------------------- -. Create a project for development using `odo`: +. Create a project for development and set as active project: + ------------------------------ oc create namespace poolboy-dev ------------------------------- - -. Change project to `poolboy-dev` namespace: -+ ----------------------- oc project poolboy-dev ----------------------- +------------------------------ -. Grant privileges for cluster role `poolboy-dev` to default service account: +. Setup virtual environment: + -------------------------------------------------------------- -oc adm policy add-cluster-role-to-user poolboy-dev -z default -------------------------------------------------------------- +--------------------- +python -m venv ./venv +. ./venv/bin/activate +pip install git+https://github.com/rhpds/kopf.git@add-deprecated-finalizer-support +pip install -r dev-requirements.txt +pip install -r requirements.txt +--------------------- -. Run odo: +. Run locally: + ------- -odo dev +./run-local.sh ------- -+ -NOTE: The poolboy operator domain is specified in the devfile. -If you are developing with a different operator domain then you will need to update the `devfile.yaml`. . Run tests with Ansible: + @@ -109,10 +94,12 @@ oc start-build poolboy --from-dir=. --follow + -------------------------------------------------------------------------------- helm template helm \ +--set enablePrometheusMetrics=false \ --set nameOverride=poolboy-dev \ --set namespace.create=false \ --set manageClaimsInterval=5 \ --set manageHandlesInterval=5 \ +--set managePoolsInterval=5 \ --set operatorDomain.name=poolboy.dev.local \ --set resourceHandlerCount=2 \ --set image.tagOverride=- \ diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..5bfc1a5 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,3 @@ +ansible==13.4.0 +kubernetes==35.0.0 +kubernetes-asyncio==35.0.1 diff --git a/devfile.yaml b/devfile.yaml deleted file mode 100644 index 4d6cd16..0000000 --- a/devfile.yaml +++ /dev/null @@ -1,37 +0,0 @@ -commands: -- exec: - commandLine: >- - rm -rf /tmp/src && cp /tmp/projects -r /tmp/src && /tmp/src/.s2i/bin/assemble - component: s2i-builder - group: - isDefault: true - kind: build - hotReloadCapable: false - workingDir: ${PROJECT_SOURCE} - id: s2i-assemble -- exec: - commandLine: /usr/libexec/s2i/run - component: s2i-builder - group: - isDefault: true - kind: run - hotReloadCapable: false - workingDir: ${PROJECT_SOURCE} - id: s2i-run -components: -- container: - env: - - name: MANAGE_CLAIMS_INTERVAL - value: "5" - - name: MANAGE_HANDLES_INTERVAL - value: "5" - - name: OPERATOR_DOMAIN - value: poolboy.dev.local - image: quay.io/redhat-cop/python-kopf-s2i:v1.37 - mountSources: true - sourceMapping: /tmp/projects - name: s2i-builder -metadata: - name: poolboy - version: 1.0.0 -schemaVersion: 2.0.0 diff --git a/helm/crds/resourcepoolscalings.yaml b/helm/crds/resourcepoolscalings.yaml new file mode 100644 index 0000000..9522601 --- /dev/null +++ b/helm/crds/resourcepoolscalings.yaml @@ -0,0 +1,96 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: resourcepoolscalings.poolboy.gpte.redhat.com +spec: + group: poolboy.gpte.redhat.com + scope: Namespaced + names: + plural: resourcepoolscalings + singular: resourcepoolscaling + kind: ResourcePoolScaling + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - name: Pool + type: string + jsonPath: .spec.resourcePool.name + - name: At + type: string + jsonPath: .spec.at + - name: Count + type: integer + jsonPath: .spec.count + - name: Created + type: integer + jsonPath: .status.count + - name: State + type: string + jsonPath: .status.state + schema: + openAPIV3Schema: + description: >- + ResourcePoolScalings configure ResourceHandles to be added to a ResourcePool at a specific time. + type: object + required: + - apiVersion + - kind + - metadata + - spec + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + properties: + name: + type: string + maxLength: 63 + pattern: ^[a-z0-9A-Z]([a-z0-9A-Z\-._]*[a-z0-9A-Z])?$ + spec: + description: ResourcePoolScaling specification + type: object + properties: + at: + type: string + format: date-time + count: + type: integer + minimum: 0 + resourcePool: + type: object + properties: + name: + type: string + required: + - name + required: + - count + - resourcePool + status: + description: ResourcePoolScaling status + type: object + x-kubernetes-preserve-unknown-fields: true + properties: + count: + description: Number of ResourceHandles created + type: integer + diffBase: + description: Kopf diffbase + type: string + kopf: + description: Kopf status + type: object + x-kubernetes-preserve-unknown-fields: true + state: + type: string + enum: + - waiting + - scaling + - done diff --git a/helm/templates/crds/resourcepoolscalings.yaml b/helm/templates/crds/resourcepoolscalings.yaml new file mode 100644 index 0000000..431fba7 --- /dev/null +++ b/helm/templates/crds/resourcepoolscalings.yaml @@ -0,0 +1,98 @@ +{{- if .Values.crds.create -}} +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: resourcepoolscalings.{{ include "poolboy.operatorDomain" . }} +spec: + group: {{ include "poolboy.operatorDomain" . }} + scope: Namespaced + names: + plural: resourcepoolscalings + singular: resourcepoolscaling + kind: ResourcePoolScaling + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - name: Pool + type: string + jsonPath: .spec.resourcePool.name + - name: At + type: string + jsonPath: .spec.at + - name: Count + type: integer + jsonPath: .spec.count + - name: Created + type: integer + jsonPath: .status.count + - name: State + type: string + jsonPath: .status.state + schema: + openAPIV3Schema: + description: >- + ResourcePoolScalings configure ResourceHandles to be added to a ResourcePool at a specific time. + type: object + required: + - apiVersion + - kind + - metadata + - spec + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + properties: + name: + type: string + maxLength: 63 + pattern: ^[a-z0-9A-Z]([a-z0-9A-Z\-._]*[a-z0-9A-Z])?$ + spec: + description: ResourcePoolScaling specification + type: object + properties: + at: + type: string + format: date-time + count: + type: integer + minimum: 0 + resourcePool: + type: object + properties: + name: + type: string + required: + - name + required: + - count + - resourcePool + status: + description: ResourcePoolScaling status + type: object + x-kubernetes-preserve-unknown-fields: true + properties: + count: + description: Number of ResourceHandles created + type: integer + diffBase: + description: Kopf diffbase + type: string + kopf: + description: Kopf status + type: object + x-kubernetes-preserve-unknown-fields: true + state: + type: string + enum: + - waiting + - scaling + - done +{{- end -}} diff --git a/helm/templates/metrics-credentials.yaml b/helm/templates/metrics-credentials.yaml index 9172b6d..eb2aba9 100644 --- a/helm/templates/metrics-credentials.yaml +++ b/helm/templates/metrics-credentials.yaml @@ -1,4 +1,4 @@ -{{- if .Values.enablePrometheusMetrics -}} +{{- if (and .Values.deploy .Values.enablePrometheusMetrics) -}} apiVersion: secretgenerator.mittwald.de/v1alpha1 kind: StringSecret metadata: diff --git a/helm/templates/rbac.yaml b/helm/templates/rbac.yaml index 8a14ad0..07ee050 100644 --- a/helm/templates/rbac.yaml +++ b/helm/templates/rbac.yaml @@ -57,6 +57,8 @@ rules: - resourcehandles/status - resourcepools - resourcepools/status + - resourcepoolscalings + - resourcepoolscalings/status - resourcewatches - resourcewatches/status verbs: diff --git a/helm/templates/service-monitor.yaml b/helm/templates/service-monitor.yaml index 6273228..454df81 100644 --- a/helm/templates/service-monitor.yaml +++ b/helm/templates/service-monitor.yaml @@ -1,4 +1,4 @@ -{{- if .Values.enablePrometheusMetrics -}} +{{- if (and .Values.deploy .Values.enablePrometheusMetrics) -}} apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: diff --git a/operator/k8sobject.py b/operator/k8sobject.py new file mode 100644 index 0000000..6247da1 --- /dev/null +++ b/operator/k8sobject.py @@ -0,0 +1,223 @@ +from datetime import datetime, timezone + +from kubernetes_asyncio.client.rest import ApiException as k8sApiException +from kubernetes_asyncio.client.models import V1OwnerReference + +from poolboy import Poolboy + +class K8sObject: + @classmethod + async def create(cls, definition): + namespace = definition['metadata'].get('namespace') + if namespace is None: + return cls( + await Poolboy.custom_objects_api.create_cluster_custom_object( + group = cls.api_group, + plural = cls.plural, + version = cls.api_version, + body = definition, + ) + ) + + return cls( + await Poolboy.custom_objects_api.create_namespaced_custom_object( + group = cls.api_group, + namespace = namespace, + plural = cls.plural, + version = cls.api_version, + body = definition, + ) + ) + + @classmethod + async def fetch(cls, name, namespace=None): + return cls(await cls.fetch_definition(name, namespace)) + + @classmethod + async def fetch_definition(cls, name, namespace=None): + if namespace is None: + return await Poolboy.custom_objects_api.get_cluster_custom_object( + group = cls.api_group, + name = name, + plural = cls.plural, + version = cls.api_version, + ) + + return await Poolboy.custom_objects_api.get_namespaced_custom_object( + group = cls.api_group, + name = name, + namespace = namespace, + plural = cls.plural, + version = cls.api_version, + ) + + @classmethod + async def list(cls, label_selector=None, namespace=None): + _continue = None + while True: + if namespace is None: + obj_list = await Poolboy.custom_objects_api.list_cluster_custom_object( + group = cls.api_group, + label_selector = label_selector, + plural = cls.plural, + version = cls.api_version, + limit = 20, + _continue = _continue + ) + else: + obj_list = await Poolboy.custom_objects_api.list_namespaced_custom_object( + group = cls.api_group, + label_selector = label_selector, + namespace = namespace, + plural = cls.plural, + version = cls.api_version, + limit = 20, + _continue = _continue + ) + for definition in obj_list.get('items', []): + yield cls(definition=definition) + _continue = obj_list['metadata'].get('continue') + if not _continue: + return + + def __init__(self, definition): + self.definition = definition + + def __str__(self): + if self.namespace is None: + return f"{self.kind} {self.name}" + return f"{self.kind} {self.name} in {self.namespace}" + + @property + def annotations(self): + return self.metadata.get('annotations', {}) + + @property + def api_group_version(self): + return f"{self.api_group}/{self.api_version}" + + @property + def creation_datetime(self): + return datetime.strptime( + self.creation_timestamp, '%Y-%m-%dT%H:%M:%SZ' + ).replace(tzinfo=timezone.utc) + + @property + def creation_timestamp(self): + return self.metadata['creationTimestamp'] + + @property + def deletion_timestamp(self) -> str|None: + return self.metadata.get('deletionTimestamp') + + @property + def labels(self): + return self.metadata.get('labels', {}) + + @property + def metadata(self): + return self.definition['metadata'] + + @property + def name(self): + return self.metadata['name'] + + @property + def namespace(self): + return self.metadata.get('namespace') + + @property + def spec(self): + return self.definition['spec'] + + @property + def status(self): + return self.definition.get('status', {}) + + @property + def uid(self): + return self.metadata['uid'] + + def as_owner_reference(self): + return V1OwnerReference( + api_version=self.api_group_version, + controller=True, + kind=self.kind, + name=self.name, + uid=self.uid, + ) + + def as_owner_reference_dict(self): + return { + "apiVersion": self.api_group_version, + "controller": True, + "kind": self.kind, + "name": self.name, + "uid": self.uid, + } + + async def delete(self): + try: + if self.namespace is None: + self.definition = await Poolboy.custom_objects_api.delete_cluster_custom_object( + group = self.api_group, + name = self.name, + plural = self.plural, + version = self.api_version, + ) + else: + self.definition = await Poolboy.custom_objects_api.delete_namespaced_custom_object( + group = self.api_group, + name = self.name, + namespace = self.namespace, + plural = self.plural, + version = self.api_version, + ) + except k8sApiException as exception: + if exception.status != 404: + raise + + async def merge_patch(self, patch): + if self.namespace is None: + self.definition = await Poolboy.custom_objects_api.patch_cluster_custom_object( + group = self.api_group, + name = self.name, + plural = self.plural, + version = self.api_version, + body = patch, + _content_type = 'application/merge-patch+json', + ) + else: + self.definition = await Poolboy.custom_objects_api.patch_namespaced_custom_object( + group = self.api_group, + name = self.name, + namespace = self.namespace, + plural = self.plural, + version = self.api_version, + body = patch, + _content_type = 'application/merge-patch+json', + ) + + async def merge_patch_status(self, patch): + if self.namespace is None: + self.definition = await Poolboy.custom_objects_api.patch_cluster_custom_object_status( + group = self.api_group, + name = self.name, + plural = self.plural, + version = self.api_version, + body = {"status": patch}, + _content_type = 'application/merge-patch+json', + ) + else: + self.definition = await Poolboy.custom_objects_api.patch_namespaced_custom_object_status( + group = self.api_group, + name = self.name, + namespace = self.namespace, + plural = self.plural, + version = self.api_version, + body = {"status": patch}, + _content_type = 'application/merge-patch+json', + ) + + async def refetch(self): + self.definition = cls.fetch_definition(self.name, self.namespace) diff --git a/operator/operator.py b/operator/operator.py index b90d873..243e95c 100755 --- a/operator/operator.py +++ b/operator/operator.py @@ -34,7 +34,7 @@ async def startup(logger: kopf.ObjectLogger, settings: kopf.OperatorSettings, ** ) # Support deprecated resource handler finalizer - if Poolboy.operator_mode_resource_handler: + if Poolboy.operator_mode_resource_handler or Poolboy.operator_mode_all_in_one: settings.persistence.deprecated_finalizer = re.compile(re.escape(Poolboy.operator_domain) + '/handler-[0-9]+$') # Store progress in status. @@ -403,7 +403,7 @@ async def resource_pool_event( await resource_pool.manage(logger=logger) @kopf.on.delete( - Poolboy.operator_domain, Poolboy.operator_version, 'resourcepools', + ResourcePool.api_group, ResourcePool.api_version, ResourcePool.plural, label_selector=label_selector, ) async def resource_pool_delete( @@ -431,7 +431,8 @@ async def resource_pool_delete( ) await resource_pool.handle_delete(logger=logger) - @kopf.daemon(Poolboy.operator_domain, Poolboy.operator_version, 'resourcepools', + @kopf.daemon( + ResourcePool.api_group, ResourcePool.api_version, ResourcePool.plural, cancellation_timeout = 1, initial_delay = Poolboy.manage_pools_interval, label_selector=label_selector, diff --git a/operator/poolboy.py b/operator/poolboy.py index b1fddb3..c050423 100644 --- a/operator/poolboy.py +++ b/operator/poolboy.py @@ -38,6 +38,7 @@ class Poolboy(): resource_pool_name_label = f"{operator_domain}/resource-pool-name" resource_pool_namespace_annotation = f"{operator_domain}/resource-pool-namespace" resource_pool_namespace_label = f"{operator_domain}/resource-pool-namespace" + resource_pool_scaling_name_label = f"{operator_domain}/resource-pool-scaling-name" resource_provider_name_annotation = f"{operator_domain}/resource-provider-name" resource_provider_namespace_annotation = f"{operator_domain}/resource-provider-namespace" resource_requester_email_annotation = f"{operator_domain}/resource-requester-email" diff --git a/operator/resourceclaim.py b/operator/resourceclaim.py index 6c37ccc..67af3e2 100644 --- a/operator/resourceclaim.py +++ b/operator/resourceclaim.py @@ -524,6 +524,7 @@ async def update_status_from_handle(self, if 'lifespan' in resource_handle.spec \ and 'lifespan' not in self.status: lifespan_value = { + "start": datetime.now(timezone.utc).strftime('%FT%TZ'), "end": resource_handle.lifespan_end_timestamp, "maximum": lifespan_maximum, "relativeMaximum": lifespan_relative_maximum, diff --git a/operator/resourcehandle.py b/operator/resourcehandle.py index 2603bee..d7de632 100644 --- a/operator/resourcehandle.py +++ b/operator/resourcehandle.py @@ -25,6 +25,7 @@ ResourceClaimT = TypeVar('ResourceClaimT', bound='ResourceClaim') ResourceHandleT = TypeVar('ResourceHandleT', bound='ResourceHandle') ResourcePoolT = TypeVar('ResourcePoolT', bound='ResourcePool') +ResourcePoolScalingT = TypeVar('ResourcePoolScalingT', bound='ResourcePoolScaling') ResourceProviderT = TypeVar('ResourceProviderT', bound='ResourceProvider') class ResourceHandleMatch: @@ -178,7 +179,14 @@ async def bind_handle_to_claim( matched_resource_handle = None for match in matches: matched_resource_handle = match.resource_handle - patch = [ + patch = [] + if 'labels' not in matched_resource_handle.metadata: + patch.append({ + "op": "add", + "path": "/metadata/labels", + "value": {}, + }) + patch.extend([ { "op": "add", "path": f"/metadata/labels/{Poolboy.resource_claim_name_label.replace('/', '~1')}", @@ -197,7 +205,7 @@ async def bind_handle_to_claim( "namespace": resource_claim.namespace, } } - ] + ]) # Update ResourceProvider to match ResourceClaim if resource_claim.has_resource_provider: @@ -233,13 +241,23 @@ async def bind_handle_to_claim( # Set lifespan end from default on claim bind lifespan_default = matched_resource_handle.get_lifespan_default(resource_claim) if lifespan_default: - patch.append({ - "op": "add", - "path": "/spec/lifespan/end", - "value": ( - datetime.now(timezone.utc) + matched_resource_handle.get_lifespan_default_timedelta(resource_claim) - ).strftime('%FT%TZ'), - }) + lifespan_end_value = ( + datetime.now(timezone.utc) + matched_resource_handle.get_lifespan_default_timedelta(resource_claim) + ).strftime('%FT%TZ') + if 'lifespan' not in matched_resource_handle.spec: + patch.append({ + "op": "add", + "path": "/spec/lifespan", + "value": { + "end": lifespan_end_value, + }, + }) + else: + patch.append({ + "op": "add", + "path": "/spec/lifespan/end", + "value": lifespan_end_value, + }) try: await matched_resource_handle.json_patch(patch) @@ -412,6 +430,7 @@ async def create_for_pool( cls, logger: kopf.ObjectLogger, resource_pool: ResourcePoolT, + resource_pool_scaling: ResourcePoolScalingT|None, ): definition = { "apiVersion": Poolboy.operator_api_version, @@ -429,6 +448,9 @@ async def create_for_pool( } } + if resource_pool_scaling is not None: + definition['metadata']['labels'][Poolboy.resource_pool_scaling_name_label] = resource_pool_scaling.name + if resource_pool.has_resource_provider: definition['spec']['provider'] = resource_pool.spec['provider'] resource_provider = await resource_pool.get_resource_provider() @@ -838,6 +860,10 @@ def resource_pool_namespace(self) -> str|None: if 'resourcePool' in self.spec: return self.spec['resourcePool'].get('namespace', Poolboy.namespace) + @property + def resource_pool_scaling_name(self) -> str|None: + return self.labels.get(Poolboy.resource_pool_scaling_name_label) + @property def resource_provider_name(self) -> str|None: return self.spec.get('provider', {}).get('name') diff --git a/operator/resourcepool.py b/operator/resourcepool.py index 2ef7b47..06ab3d1 100644 --- a/operator/resourcepool.py +++ b/operator/resourcepool.py @@ -10,6 +10,9 @@ import resourcehandle import resourceprovider +from kubernetes_asyncio.client.rest import ApiException as k8sApiException +from resourcepoolscaling import ResourcePoolScaling + from kopfobject import KopfObject from poolboy import Poolboy @@ -121,6 +124,8 @@ def lifespan_unclaimed_timedelta(self): @property def max_unready(self) -> int|None: + """Maximum number of ResourceHandles which can be unready in the pool. + If exceeded then no further provisions will start.""" return self.spec.get('maxUnready') @property @@ -175,7 +180,7 @@ async def assign_resource_handler(self): } }) await self.json_patch(patch) - except kubernetes_asyncio.client.exceptions.ApiException as exception: + except k8sApiException as exception: pass async def get_resource_provider(self) -> ResourceProviderT: @@ -196,6 +201,16 @@ async def manage(self, logger: kopf.ObjectLogger): if self.delete_unhealthy_resource_handles and resource_handle.is_healthy == False: logger.info(f"Deleting {resource_handle} in {self} due to failed health check") await resource_handle.delete() + if resource_handle.resource_pool_scaling_name is not None: + try: + resource_pool_scaling = await ResourcePoolScaling.fetch( + name=resource_handle.resource_pool_scaling_name, + namespace=resource_handle.namespace, + ) + await resource_pool_scaling.decrement_created_count() + except k8sApiException as exception: + if exception.status != 404: + raise continue available_resource_handles.append(resource_handle) if resource_handle.is_ready: @@ -206,22 +221,35 @@ async def manage(self, logger: kopf.ObjectLogger): "ready": resource_handle.is_ready, }) - resource_handle_deficit = self.min_available - len(available_resource_handles) - - if self.max_unready is not None: - unready_count = len(available_resource_handles) - len(ready_resource_handles) - if resource_handle_deficit > self.max_unready - unready_count: - resource_handle_deficit = self.max_unready - unready_count - - if resource_handle_deficit > 0: - for i in range(resource_handle_deficit): - resource_handle = await resourcehandle.ResourceHandle.create_for_pool( - logger=logger, - resource_pool=self + min_available_deficit = self.min_available - len(available_resource_handles) + resource_pool_scaling = await ResourcePoolScaling.get_active_scaling_for_pool(self.name) + unready_count = len(available_resource_handles) - len(ready_resource_handles) + + while ( + (self.max_unready is None or unready_count < self.max_unready) and + ( + min_available_deficit > 0 or + ( + resource_pool_scaling is not None and + resource_pool_scaling.count > resource_pool_scaling.current_count ) - resource_handles_for_status.append({ - "name": resource_handle.name, - }) + ) + ): + resource_handle = await resourcehandle.ResourceHandle.create_for_pool( + logger=logger, + resource_pool=self, + resource_pool_scaling=resource_pool_scaling, + ) + resource_handles_for_status.append({ + "name": resource_handle.name, + }) + unready_count += 1 + min_available_deficit -= 1 + if resource_pool_scaling is not None: + logger.info("Created %s for %s with %s", resource_handle, self, resource_pool_scaling) + await resource_pool_scaling.increment_created_count() + else: + logger.info("Created %s for %s min available", resource_handle, self) patch = [] if not self.status: diff --git a/operator/resourcepoolscaling.py b/operator/resourcepoolscaling.py new file mode 100644 index 0000000..aa5941e --- /dev/null +++ b/operator/resourcepoolscaling.py @@ -0,0 +1,109 @@ +import asyncio + +from datetime import datetime, timezone +from typing import TypeVar +from uuid import UUID + +import kopf +import pytimeparse + +from kubernetes_asyncio.client.rest import ApiException as k8sApiException + +from k8sobject import K8sObject +from poolboy import Poolboy + +ResourcePoolScalingT = TypeVar('ResourcePoolScalingT', bound='ResourcePoolScaling') + +class ResourcePoolScaling(K8sObject): + api_group = Poolboy.operator_domain + api_version = Poolboy.operator_version + kind = "ResourcePoolScaling" + plural = "resourcepoolscalings" + + @classmethod + async def get_active_scaling_for_pool(cls, pool_name: str) -> ResourcePoolScalingT|None: + async for scaling in ResourcePoolScaling.list(namespace=Poolboy.namespace): + if ( + scaling.resource_pool_name == pool_name and + scaling.is_active and + not scaling.ignore + ): + return scaling + return None + + @property + def at_datetime(self) -> datetime: + if 'at' in self.spec: + return datetime.strptime(self.spec['at'], '%Y-%m-%dT%H:%M:%S%z') + return datetime.now(timezone.utc) + + @property + def count(self) -> int: + return self.spec['count'] + + @property + def ignore(self) -> bool: + """Return whether this ResourcePoolScaling should be ignored""" + return Poolboy.ignore_label in self.labels + + @property + def is_active(self) -> bool: + """ResourcePoolScaling is active if scheduled time is in the past and + count is less than current count.""" + return self.at_datetime <= datetime.now(timezone.utc) and self.count > self.current_count + + @property + def current_count(self) -> int: + """Count of ResourceHandles successfully created for this ResourcePoolScaling. + This value is incremented for each ResourceHandle created and decremented for + each unhealthy ResourceHandle removed from the pool.""" + return self.status.get('count', 0) + + @property + def resource_pool_name(self) -> int: + """Label value used to select which resource handler pod should manage this ResourcePool.""" + return self.spec['resourcePool']['name'] + + async def decrement_created_count(self) -> None: + """Decrement count in status.""" + while True: + try: + self.definition.setdefault('status', {})['count'] = self.current_count - 1 + if self.current_count >= self.count: + self.definition['state'] = 'done' + + self.definition = await Poolboy.custom_objects_api.replace_namespaced_custom_object_status( + body = self.definition, + group = self.api_group, + name = self.name, + namespace = self.namespace, + plural = self.plural, + version = self.api_version, + ) + return + except k8sApiException as exception: + if exception.status != 409: + raise + await self.refetch() + + async def increment_created_count(self) -> None: + """Increment count in status.""" + while True: + try: + self.definition.setdefault('status', {})['count'] = self.current_count + 1 + if self.current_count >= self.count: + self.definition['state'] = 'done' + + self.definition = await Poolboy.custom_objects_api.replace_namespaced_custom_object_status( + body = self.definition, + group = self.api_group, + name = self.name, + namespace = self.namespace, + plural = self.plural, + version = self.api_version, + ) + return + except k8sApiException as exception: + if exception.status != 409: + raise + await self.refetch() diff --git a/requirements.txt b/requirements.txt index 6b992ea..795ce54 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,20 @@ aioprometheus==23.12.0 inflection==0.5.1 -Jinja2==3.1.5 -jmespath-community==1.1.2 -jsonpointer==2.2 -jsonschema==3.2.0 -openapi-schema-validator==0.1.5 -prometheus-client==0.11.0 -pyasn1==0.4.8 -pyasn1-modules==0.2.8 -pydantic==1.10.13 +Jinja2==3.1.6 +jmespath-community==1.1.3 +jsonpointer==3.0.0 +jsonschema==4.26.0 +openapi-schema-validator==0.8.1 +prometheus-client==0.24.1 +pyasn1==0.6.2 +pyasn1-modules==0.4.2 +pydantic==2.12.5 pyOpenSSL==20.0.1 python-dateutil==2.8.1 python-string-utils==1.0.0 pytimeparse==1.1.8 PyYAML==6.0.1 -requests==2.32.0 +requests==2.32.5 str2bool==1.1 StringGenerator==0.4.4 -urllib3==1.26.19 +urllib3==2.6.3 diff --git a/run-local.sh b/run-local.sh new file mode 100755 index 0000000..0fb5e83 --- /dev/null +++ b/run-local.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -eo pipefail + +export OPERATOR_DOMAIN=poolboy.dev.local +export OPERATOR_NAMESPACE=poolboy-dev +export OPERATOR_MODE=all-in-one +export MANAGE_CLAIMS_INTERVAL=5 +export MANAGE_HANDLES_INTERVAL=5 +export MANAGE_POOLS_INTERVAL=5 + +helm template helm --include-crds \ +--set deploy=false \ +--set admin.deploy=false \ +--set nameOverride=poolboy-dev \ +--set operatorDomain.name=poolboy.dev.local \ +| oc apply -f - + +oc create namespace poolboy-dev || : +oc project poolboy-dev + +if [[ -d venv ]]; then + . ./venv/bin/activate +else + python -m venv ./venv + . ./venv/bin/activate + pip install git+https://github.com/rhpds/kopf.git@add-deprecated-finalizer-support + pip install -r dev-requirements.txt + pip install -r requirements.txt +fi + +cd ./operator + +exec kopf run \ + --standalone \ + --all-namespaces \ + --liveness=http://0.0.0.0:8080/healthz \ + operator.py diff --git a/run-test.sh b/run-test.sh new file mode 100755 index 0000000..405ec2e --- /dev/null +++ b/run-test.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -eo pipefail + +. ./venv/bin/activate + +ansible-playbook \ +-e ansible_python_interpreter='{{ ansible_playbook_python }}' \ +-e poolboy_domain=poolboy.dev.local \ +-e poolboy_namespace=poolboy-dev \ +-e poolboy_service_account=poolboy-dev \ +-e poolboy_test_namespace=poolboy-dev-test \ +-e '{"poolboy_tests": ["simple"]}' \ +test/playbook.yaml diff --git a/test/roles/poolboy_test_simple/tasks/cleanup-resources.yaml b/test/roles/poolboy_test_simple/tasks/cleanup-resources.yaml index ac7becf..b8ff605 100644 --- a/test/roles/poolboy_test_simple/tasks/cleanup-resources.yaml +++ b/test/roles/poolboy_test_simple/tasks/cleanup-resources.yaml @@ -20,6 +20,27 @@ label: "{{ _claim.metadata.name }}" loop_var: _claim +- name: Get test ResourcePoolScalings + kubernetes.core.k8s_info: + api_version: "{{ poolboy_domain }}/v1" + kind: ResourcePoolScaling + namespace: "{{ poolboy_namespace }}" + label_selectors: + - "{{ poolboy_domain }}/test=simple" + register: r_test_pools + +- name: Remove test ResourcePoolScalings + kubernetes.core.k8s: + api: "{{ poolboy_domain }}/v1" + kind: ResourcePoolScaling + namespace: "{{ _pool.metadata.namespace }}" + name: "{{ _pool.metadata.name }}" + state: absent + loop: "{{ r_test_pools.resources | default([]) }}" + loop_control: + label: "{{ _pool.metadata.name }}" + loop_var: _pool + - name: Get test ResourcePools kubernetes.core.k8s_info: api_version: "{{ poolboy_domain }}/v1" diff --git a/test/roles/poolboy_test_simple/tasks/test-finalizers-01.yaml b/test/roles/poolboy_test_simple/tasks/test-finalizers-01.yaml index fe4d2e8..c333a29 100644 --- a/test/roles/poolboy_test_simple/tasks/test-finalizers-01.yaml +++ b/test/roles/poolboy_test_simple/tasks/test-finalizers-01.yaml @@ -109,7 +109,10 @@ failed_when: >- r_get_resource_handle.resources | length != 1 or r_get_resource_handle.resources[0].metadata.finalizers is undefined or - r_get_resource_handle.resources[0].metadata.finalizers != [poolboy_domain ~ '/handler'] + ( + r_get_resource_handle.resources[0].metadata.finalizers != [poolboy_domain] and + r_get_resource_handle.resources[0].metadata.finalizers != [poolboy_domain ~ '/handler'] + ) - name: Set deprecated finalizer on ResourceHandle for test-finalizers-01-a kubernetes.core.k8s: diff --git a/test/roles/poolboy_test_simple/tasks/test-pool-scaling-01.yaml b/test/roles/poolboy_test_simple/tasks/test-pool-scaling-01.yaml new file mode 100644 index 0000000..0e57634 --- /dev/null +++ b/test/roles/poolboy_test_simple/tasks/test-pool-scaling-01.yaml @@ -0,0 +1,239 @@ +--- +- name: Create ResourceProvider test-pool-scaling-01 + kubernetes.core.k8s: + definition: + apiVersion: "{{ poolboy_domain }}/v1" + kind: ResourceProvider + metadata: + name: test-pool-scaling-01 + namespace: "{{ poolboy_namespace }}" + labels: >- + {{ { + poolboy_domain ~ "/test": "simple" + } }} + spec: + healthCheck: >- + spec.value != "failed" + override: + apiVersion: "{{ poolboy_domain }}/v1" + kind: ResourceClaimTest + metadata: + name: "test-pool-scaling-01-{% raw %}{{ guid }}{% endraw %}" + namespace: "{{ poolboy_test_namespace }}" + parameters: + - name: value + allowUpdate: true + validation: + openAPIV3Schema: + type: string + template: + definition: + spec: + value: "{% raw %}{{ value }}{% endraw %}" + updateFilters: + - pathMatch: /spec/.* + allowedOps: + - replace + +- name: Create ResourcePool test-pool-scaling-01 + kubernetes.core.k8s: + definition: + apiVersion: "{{ poolboy_domain }}/v1" + kind: ResourcePool + metadata: + name: test-pool-scaling-01 + namespace: "{{ poolboy_namespace }}" + labels: >- + {{ { + poolboy_domain ~ "/test": "simple" + } }} + spec: + deleteUnhealthyResourceHandles: true + minAvailable: 1 + provider: + name: test-pool-scaling-01 + parameterValues: + value: foo + +- name: Verify ResourceHandles for test-pool-scaling-01 + kubernetes.core.k8s_info: + api_version: "{{ poolboy_domain }}/v1" + kind: ResourceHandle + namespace: "{{ poolboy_namespace }}" + label_selectors: + - "{{ poolboy_domain }}/resource-pool-name = test-pool-scaling-01" + register: r_get_resource_handles + vars: + __unbound_handles: >- + {{ r_get_resource_handles.resources | json_query('[?spec.resourceClaim==null]') }} + failed_when: >- + __unbound_handles | length != 1 + retries: 5 + delay: 1 + +- name: Create ResourcePoolScaling test-pool-scaling-01-a + kubernetes.core.k8s: + definition: + apiVersion: "{{ poolboy_domain }}/v1" + kind: ResourcePoolScaling + metadata: + name: test-pool-scaling-01-a + namespace: "{{ poolboy_namespace }}" + labels: >- + {{ { + poolboy_domain ~ "/test": "simple" + } }} + spec: + count: 2 + resourcePool: + name: test-pool-scaling-01 + +- name: Verify ResourceHandles for test-pool-scaling-01 + kubernetes.core.k8s_info: + api_version: "{{ poolboy_domain }}/v1" + kind: ResourceHandle + namespace: "{{ poolboy_namespace }}" + label_selectors: + - "{{ poolboy_domain }}/resource-pool-name = test-pool-scaling-01" + register: r_get_resource_handles + vars: + __unbound_handles: >- + {{ r_get_resource_handles.resources | json_query('[?spec.resourceClaim==null]') }} + failed_when: >- + __unbound_handles | length != 3 + retries: 5 + delay: 1 + +- name: Create ResourcePoolScaling test-pool-scaling-01-b + kubernetes.core.k8s: + definition: + apiVersion: "{{ poolboy_domain }}/v1" + kind: ResourcePoolScaling + metadata: + name: test-pool-scaling-01-b + namespace: "{{ poolboy_namespace }}" + labels: >- + {{ { + poolboy_domain ~ "/test": "simple" + } }} + spec: + at: >- + {{ '%FT%TZ' | strftime(now(true).strftime('%s') | int + 86400) }} + count: 2 + resourcePool: + name: test-pool-scaling-01 + +- pause: + seconds: 5 + +- name: Verify ResourceHandles for test-pool-scaling-01 not increased + kubernetes.core.k8s_info: + api_version: "{{ poolboy_domain }}/v1" + kind: ResourceHandle + namespace: "{{ poolboy_namespace }}" + label_selectors: + - "{{ poolboy_domain }}/resource-pool-name = test-pool-scaling-01" + register: r_get_resource_handles + vars: + __unbound_handles: >- + {{ r_get_resource_handles.resources | json_query('[?spec.resourceClaim==null]') }} + failed_when: >- + __unbound_handles | length != 3 + +- name: Update ResourcePoolScaling test-pool-scaling-01-b + kubernetes.core.k8s: + api_version: "{{ poolboy_domain }}/v1" + kind: ResourcePoolScaling + name: test-pool-scaling-01-b + namespace: "{{ poolboy_namespace }}" + definition: + spec: + at: "{{ now(true).strftime('%FT%TZ') }}" + +- name: Verify ResourceHandles for test-pool-scaling-01-b + kubernetes.core.k8s_info: + api_version: "{{ poolboy_domain }}/v1" + kind: ResourceHandle + namespace: "{{ poolboy_namespace }}" + label_selectors: + - "{{ poolboy_domain }}/resource-pool-name = test-pool-scaling-01" + - "{{ poolboy_domain }}/resource-pool-scaling-name = test-pool-scaling-01-b" + register: r_get_resource_handles + vars: + __unbound_handles: >- + {{ r_get_resource_handles.resources | json_query('[?spec.resourceClaim==null]') }} + failed_when: >- + __unbound_handles | length != 2 + retries: 5 + delay: 1 + +- name: Set test_pool_scaling_01_b_fail_handle + ansible.builtin.set_fact: + test_pool_scaling_01_b_fail_handle: >- + {{ r_get_resource_handles.resources[0].metadata.name }} + +- name: Mark ResourceHandle for test-pool-scaling-01-b as failed + kubernetes.core.k8s: + api_version: "{{ poolboy_domain }}/v1" + kind: ResourceHandle + name: "{{ test_pool_scaling_01_b_fail_handle }}" + namespace: "{{ poolboy_namespace }}" + definition: + spec: + provider: + parameterValues: + value: failed + +- name: Verify failed ResourceHandle for test-pool-scaling-01-b deleted + kubernetes.core.k8s_info: + api_version: "{{ poolboy_domain }}/v1" + kind: ResourceHandle + name: "{{ test_pool_scaling_01_b_fail_handle }}" + namespace: "{{ poolboy_namespace }}" + register: r_get_resource_handles + failed_when: >- + r_get_resource_handles.resources | length != 0 + retries: 10 + delay: 1 + +- name: Verify failed ResourceHandle for test-pool-scaling-01-b replaced + kubernetes.core.k8s_info: + api_version: "{{ poolboy_domain }}/v1" + kind: ResourceHandle + namespace: "{{ poolboy_namespace }}" + label_selectors: + - "{{ poolboy_domain }}/resource-pool-name = test-pool-scaling-01" + - "{{ poolboy_domain }}/resource-pool-scaling-name = test-pool-scaling-01-b" + register: r_get_resource_handles + vars: + __unbound_handles: >- + {{ r_get_resource_handles.resources | json_query('[?spec.resourceClaim==null]') }} + failed_when: >- + __unbound_handles | length != 2 + retries: 5 + delay: 1 + +- name: Delete ResourcePool test-pool-scaling-01 + kubernetes.core.k8s: + api_version: "{{ poolboy_domain }}/v1" + kind: ResourcePool + name: test-pool-scaling-01 + namespace: "{{ poolboy_namespace }}" + state: absent + +- name: Verify cleanup of ResourceHandles for test-pool-scaling-01 after delete + kubernetes.core.k8s_info: + api_version: "{{ poolboy_domain }}/v1" + kind: ResourceHandle + namespace: "{{ poolboy_namespace }}" + label_selectors: + - "{{ poolboy_domain }}/resource-pool-name = test-pool-scaling-01" + register: r_get_resource_handles + vars: + __unbound_handles: >- + {{ r_get_resource_handles.resources | json_query('[?spec.resourceClaim==null]') }} + failed_when: >- + __unbound_handles | length != 0 + until: r_get_resource_handles is successful + retries: 5 + delay: 2 diff --git a/test/roles/poolboy_test_simple/tasks/test.yaml b/test/roles/poolboy_test_simple/tasks/test.yaml index c5e11be..a5f4fb0 100644 --- a/test/roles/poolboy_test_simple/tasks/test.yaml +++ b/test/roles/poolboy_test_simple/tasks/test.yaml @@ -18,6 +18,7 @@ - test-pool-02.yaml - test-pool-03.yaml - test-pool-04.yaml + - test-pool-scaling-01.yaml - test-ready-01.yaml - test-recreate-01.yaml - test-vars-01.yaml @@ -989,8 +990,8 @@ __test_lifespan_3.status.lifespan.end is undefined or 23 != (__test_lifespan_3.status.lifespan.end | to_datetime('%Y-%m-%dT%H:%M:%S%z') - __test_lifespan_3.status.lifespan.start | to_datetime('%Y-%m-%dT%H:%M:%S%z')).total_seconds() until: r_get_test_lifespan_3 is success - delay: 5 - retries: 10 + delay: 2 + retries: 5 - name: Create test-base ResourceProvider kubernetes.core.k8s: