From c1eb5960598b0ccec21265a0cba0a53747e3911c Mon Sep 17 00:00:00 2001 From: Marcos Amorim Date: Fri, 12 Sep 2025 07:03:52 -0400 Subject: [PATCH] Add Prometheus metrics instrumentation and basic auth for ServiceMonitor - Add timer decorators to K8s operations and Jinja2 templating functions - Enable basic authentication for metrics endpoint using StringSecret - Configure metrics credentials in Helm values and ServiceMonitor --- helm/templates/metrics-credentials.yaml | 19 ++++++++++++++++ helm/templates/service-monitor.yaml | 9 +++++++- helm/values.yaml | 6 +++++ operator/poolboy_k8s.py | 30 ++++++++++++++++++++----- operator/poolboy_templating.py | 13 ++++++----- 5 files changed, 66 insertions(+), 11 deletions(-) create mode 100644 helm/templates/metrics-credentials.yaml diff --git a/helm/templates/metrics-credentials.yaml b/helm/templates/metrics-credentials.yaml new file mode 100644 index 0000000..9172b6d --- /dev/null +++ b/helm/templates/metrics-credentials.yaml @@ -0,0 +1,19 @@ +{{- if .Values.enablePrometheusMetrics -}} +apiVersion: secretgenerator.mittwald.de/v1alpha1 +kind: StringSecret +metadata: + name: {{ include "poolboy.name" . }}-metrics-credentials + namespace: {{ include "poolboy.namespaceName" . }} + labels: + {{- include "poolboy.labels" . | nindent 4 }} + annotations: + secret-generator.v1.mittwald.de/type: basic-auth +spec: + forceRegenerate: false + data: + metrics_username: {{ .Values.metrics.username }} + fields: + - fieldName: metrics_password + encoding: "hex" + length: "32" +{{- end }} diff --git a/helm/templates/service-monitor.yaml b/helm/templates/service-monitor.yaml index 5494984..6273228 100644 --- a/helm/templates/service-monitor.yaml +++ b/helm/templates/service-monitor.yaml @@ -2,7 +2,7 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: - name: {{ include "poolboy.name" . }}-monitor + name: {{ include "poolboy.name" . }}-metrics namespace: {{ include "poolboy.namespaceName" . }} labels: {{- include "poolboy.labels" . | nindent 4 }} @@ -17,4 +17,11 @@ spec: - port: metrics interval: "30s" path: /metrics + basicAuth: + username: + name: {{ include "poolboy.name" . }}-metrics-credentials + key: metrics_username + password: + name: {{ include "poolboy.name" . }}-metrics-credentials + key: metrics_password {{- end }} diff --git a/helm/values.yaml b/helm/values.yaml index 22982b3..1078b9b 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -98,4 +98,10 @@ tolerations: [] affinity: {} clusterDomain: unknown + +# Enable Prometheus metrics enablePrometheusMetrics: true + +# Metrics Credentials +metrics: + username: metrics diff --git a/operator/poolboy_k8s.py b/operator/poolboy_k8s.py index 235b62d..0adbf9a 100644 --- a/operator/poolboy_k8s.py +++ b/operator/poolboy_k8s.py @@ -1,10 +1,9 @@ -import asyncio +from typing import List, Mapping + import inflection import kopf import kubernetes_asyncio - -from typing import List, Mapping - +from metrics.timer_decorator import async_timer from poolboy import Poolboy api_groups = {} @@ -12,18 +11,21 @@ class KindNotFoundException(Exception): pass +@async_timer(app="PoolboyK8s") async def create_object(definition: Mapping) -> Mapping: if '/' in definition['apiVersion']: return await create_custom_object(definition) else: return await create_core_object(definition) +@async_timer(app="PoolboyK8s") async def create_core_object(definition: Mapping) -> Mapping: if 'namespace' in definition['metadata']: return await create_namespaced_core_object(definition) else: return await create_cluster_core_object(definition) +@async_timer(app="PoolboyK8s") async def create_custom_object(definition: Mapping) -> Mapping: group, version = definition['apiVersion'].split('/') plural = await kind_to_plural(group=group, kind=definition['kind'], version=version) @@ -44,6 +46,7 @@ async def create_custom_object(definition: Mapping) -> Mapping: version = version, ) +@async_timer(app="PoolboyK8s") async def create_cluster_core_object(definition: Mapping) -> Mapping: kind = definition['kind'] method = getattr( @@ -54,6 +57,7 @@ async def create_cluster_core_object(definition: Mapping) -> Mapping: await method(body=definition) ) +@async_timer(app="PoolboyK8s") async def create_namespaced_core_object(definition: Mapping) -> Mapping: kind = definition['kind'] namespace = definition['metadata']['namespace'] @@ -65,6 +69,7 @@ async def create_namespaced_core_object(definition: Mapping) -> Mapping: await method(body=definition, namespace=namespace) ) +@async_timer(app="PoolboyK8s") async def delete_core_object( kind: str, name: str, @@ -82,6 +87,7 @@ async def delete_core_object( name = name, ) +@async_timer(app="PoolboyK8s") async def delete_cluster_core_object( kind: str, name: str, @@ -94,6 +100,7 @@ async def delete_cluster_core_object( await method(name=name) ) +@async_timer(app="PoolboyK8s") async def delete_namespaced_core_object( kind: str, name: str, @@ -107,6 +114,7 @@ async def delete_namespaced_core_object( await method(name=name, namespace=namespace) ) +@async_timer(app="PoolboyK8s") async def delete_custom_object( group: str, version: str, @@ -131,6 +139,7 @@ async def delete_custom_object( version = version, ) +@async_timer(app="PoolboyK8s") async def delete_object( api_version: str, kind: str, @@ -153,6 +162,7 @@ async def delete_object( namespace = namespace ) +@async_timer(app="PoolboyK8s") async def get_object( api_version: str, kind: str, @@ -175,6 +185,7 @@ async def get_object( namespace = namespace ) +@async_timer(app="PoolboyK8s") async def get_core_object( kind: str, name: str, @@ -192,6 +203,7 @@ async def get_core_object( name = name, ) +@async_timer(app="PoolboyK8s") async def get_cluster_core_object( kind: str, name: str, @@ -204,6 +216,7 @@ async def get_cluster_core_object( await method(name=name) ) +@async_timer(app="PoolboyK8s") async def get_namespaced_core_object( kind: str, name: str, @@ -217,6 +230,7 @@ async def get_namespaced_core_object( await method(name=name, namespace=namespace) ) +@async_timer(app="PoolboyK8s") async def get_custom_object( group: str, version: str, @@ -241,6 +255,7 @@ async def get_custom_object( version = version, ) +@async_timer(app="PoolboyK8s") async def get_requester_from_namespace(namespace: str) -> tuple[Mapping|None, List[Mapping]|None]: try: namespace_obj = await Poolboy.core_v1_api.read_namespace(namespace) @@ -290,7 +305,7 @@ async def get_requester_from_namespace(namespace: str) -> tuple[Mapping|None, Li return user, identities - +@async_timer(app="PoolboyK8s") async def kind_to_plural( group: str, kind: str, @@ -336,6 +351,7 @@ async def kind_to_plural( delay=600 ) +@async_timer(app="PoolboyK8s") async def patch_core_object( kind: str, name: str, @@ -356,6 +372,7 @@ async def patch_core_object( patch = patch, ) +@async_timer(app="PoolboyK8s") async def patch_cluster_core_object( kind: str, name: str, @@ -373,6 +390,7 @@ async def patch_cluster_core_object( ) ) +@async_timer(app="PoolboyK8s") async def patch_namespaced_core_object( kind: str, name: str, @@ -392,6 +410,7 @@ async def patch_namespaced_core_object( ) ) +@async_timer(app="PoolboyK8s") async def patch_custom_object( group: str, kind: str, @@ -421,6 +440,7 @@ async def patch_custom_object( _content_type = 'application/json-patch+json', ) +@async_timer(app="PoolboyK8s") async def patch_object( api_version: str, kind: str, diff --git a/operator/poolboy_templating.py b/operator/poolboy_templating.py index 87a79ca..a2e3be0 100644 --- a/operator/poolboy_templating.py +++ b/operator/poolboy_templating.py @@ -1,15 +1,16 @@ import copy import functools -import jinja2 -import jmespath import json -import pytimeparse import random import re - from datetime import datetime, timedelta, timezone -from strgen import StringGenerator + +import jinja2 +import jmespath +import pytimeparse +from metrics.timer_decorator import sync_timer from str2bool import str2bool +from strgen import StringGenerator MAX_RECURSION_DEPTH = 100 @@ -183,6 +184,8 @@ def j2template_get(template: str): j2template_cache[template] = j2template return j2template + +@sync_timer(app="Jinja2Process") def jinja2process(template, omit=None, variables={}, template_variables={}): variables = copy.copy(variables) variables['datetime'] = datetime