diff --git a/fastloom/monitoring.py b/fastloom/monitoring.py index 0395f8e..d221bdf 100644 --- a/fastloom/monitoring.py +++ b/fastloom/monitoring.py @@ -3,6 +3,7 @@ import logging import os import re +import shutil from collections.abc import Callable, Sequence from enum import Enum from os import getenv @@ -25,7 +26,11 @@ from fastloom.cache.settings import RedisSettings from fastloom.db.settings import MongoSettings from fastloom.launcher.utils import is_installed -from fastloom.observability.settings import ObservabilitySettings, OtelConfig +from fastloom.observability.settings import ( + ObservabilitySettings, + OtelConfig, + PrometheusConfig, +) from fastloom.settings.base import FastAPISettings from fastloom.signals.settings import RabbitmqSettings from fastloom.tenant.protocols import TenantMonitoringSchema @@ -245,11 +250,36 @@ def instrument_pydantic_ai(): logfire.instrument_pydantic_ai() +def instrument_prometheus( + settings: ObservabilitySettings, + prefix: str = "", + app: FastAPI | None = None, +): + from prometheus_fastapi_instrumentator import Instrumentator + + Instrumentator( + should_group_status_codes=settings.PROMETHEUS_GROUP_STATUS_CODES, + should_ignore_untemplated=settings.PROMETHEUS_IGNORE_UNTEMPLATED, + should_group_untemplated=settings.PROMETHEUS_GROUP_UNTEMPLATED, + should_round_latency_decimals=settings.PROMETHEUS_ROUND_LATENCY_DECIMALS, + should_instrument_requests_inprogress=settings.PROMETHEUS_INSTRUMENT_REQUESTS_INPROGRESS, + round_latency_decimals=settings.PROMETHEUS_LATENCY_DECIMALS, + inprogress_name=settings.PROMETHEUS_INPROGRESS_NAME, + inprogress_labels=settings.PROMETHEUS_INPROGRESS_LABELS, + ).instrument(app).expose( + app, + endpoint=f"{prefix}{settings.PROMETHEUS_METRICS_ENDPOINT}", + include_in_schema=settings.PROMETHEUS_INCLUDE_IN_SCHEMA, + should_gzip=settings.PROMETHEUS_SHOULD_GZIP, + ) + + class Instruments(Enum): REDIS = instrument_redis CELERY = instrument_celery RABBIT = instrument_rabbit HTTPX = instrument_httpx + PROMETHEUS = instrument_prometheus REQUESTS = instrument_requests METRICS = instrument_metrics MONGODB = instrument_mongodb @@ -306,6 +336,8 @@ def infer_instruments[T: BaseModel](settings: T) -> list[Instruments]: instruments.append(Instruments.MONGODB) if isinstance(settings, ObservabilitySettings) and settings.METRICS: instruments.append(Instruments.METRICS) + if isinstance(settings, Instruments.PROMETHEUS): + instruments.append(Instruments.PROMETHEUS) if is_installed("pydantic_ai"): instruments.append(Instruments.PYDANTIC_AI) return instruments @@ -320,6 +352,14 @@ def setup_otel_config(settings: ObservabilitySettings): os.environ[field_name] = str(value) +def setup_prometheus_multiproc(settings: PrometheusConfig) -> None: + if not (path := settings.PROMETHEUS_MULTIPROC_DIR): + return + shutil.rmtree(path, ignore_errors=True) + os.makedirs(path) + os.environ["PROMETHEUS_MULTIPROC_DIR"] = path + + class InitMonitoring: def __init__( self, @@ -331,6 +371,7 @@ def __init__( self.instruments = instruments self.otel_sampling = otel_sampling setup_otel_config(settings) + setup_prometheus_multiproc(settings) def __enter__(self): if int(self.settings.SENTRY_ENABLED): @@ -350,5 +391,13 @@ def __exit__(self, exc_type, exc_val, exc_tb): ... def instrument( self, app: FastAPI, settings: FastAPISettings | None = None ): - if app is not None and int(self.settings.OTEL_ENABLED): + if app is None: + return + if int(self.settings.OTEL_ENABLED): instrument_fastapi(app, settings) + if int(self.settings.PROMETHEUS_ENABLED): + instrument_prometheus( + self.settings, + prefix=self.settings.API_PREFIX, + app=app, + ) diff --git a/fastloom/observability/decorators.py b/fastloom/observability/decorators.py new file mode 100644 index 0000000..0f7535b --- /dev/null +++ b/fastloom/observability/decorators.py @@ -0,0 +1,47 @@ +import inspect +from collections.abc import Callable +from functools import wraps + +_CACHE: dict[str, tuple] = {} + + +def _metric_name(func: Callable) -> str: + name = getattr(func, "__name__", "unknown") + parts = (getattr(func, "__module__", None) or "").split(".") + return f"{parts[1]}_{name}" if len(parts) > 1 else name + + +def prom_track(name: str | None = None): + from prometheus_client import Counter, Histogram + + def decorator(func: Callable) -> Callable: + base = name or _metric_name(func) + counter, histogram = _CACHE.setdefault( + base, + ( + Counter(f"{base}_calls_total", f"Total calls of {base}"), + Histogram( + f"{base}_duration_seconds", + f"Duration of {base} in seconds", + ), + ), + ) + + if inspect.iscoroutinefunction(func): + + @wraps(func) + async def wrapper(*args, **kwargs): # pyright: ignore[reportRedeclaration] + counter.inc() + with histogram.time(): + return await func(*args, **kwargs) + else: + + @wraps(func) + def wrapper(*args, **kwargs): + counter.inc() + with histogram.time(): + return func(*args, **kwargs) + + return wrapper + + return decorator diff --git a/fastloom/observability/settings.py b/fastloom/observability/settings.py index 2fd1ea2..89641ef 100644 --- a/fastloom/observability/settings.py +++ b/fastloom/observability/settings.py @@ -37,8 +37,28 @@ class OtelConfig(BaseModel): ] = EnvDefault(".*") -class ObservabilitySettings(MonitoringSettings, OtelConfig): +class PrometheusConfig(BaseModel): + PROMETHEUS_METRICS_ENDPOINT: EnvBackend[str] = EnvDefault("/metrics") + PROMETHEUS_INCLUDE_IN_SCHEMA: EnvBackend[bool] = EnvDefault(True) + PROMETHEUS_SHOULD_GZIP: EnvBackend[bool] = EnvDefault(False) + PROMETHEUS_GROUP_STATUS_CODES: EnvBackend[bool] = EnvDefault(True) + PROMETHEUS_IGNORE_UNTEMPLATED: EnvBackend[bool] = EnvDefault(False) + PROMETHEUS_GROUP_UNTEMPLATED: EnvBackend[bool] = EnvDefault(True) + PROMETHEUS_ROUND_LATENCY_DECIMALS: EnvBackend[bool] = EnvDefault(False) + PROMETHEUS_LATENCY_DECIMALS: EnvBackend[int] = EnvDefault(4) + PROMETHEUS_INSTRUMENT_REQUESTS_INPROGRESS: EnvBackend[bool] = EnvDefault( + False + ) + PROMETHEUS_INPROGRESS_NAME: EnvBackend[str] = EnvDefault( + "http_requests_inprogress" + ) + PROMETHEUS_INPROGRESS_LABELS: EnvBackend[bool] = EnvDefault(False) + PROMETHEUS_MULTIPROC_DIR: EnvBackend[str] = EnvDefault("") + + +class ObservabilitySettings(MonitoringSettings, OtelConfig, PrometheusConfig): SENTRY_ENABLED: int = 0 OTEL_ENABLED: int = 0 + PROMETHEUS_ENABLED: int = 0 SENTRY_DSN: AnyHttpUrl | None = None METRICS: bool = False diff --git a/poetry.lock b/poetry.lock index 0ad25b6..c8475c5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3302,6 +3302,41 @@ nodeenv = ">=0.11.1" pyyaml = ">=5.1" virtualenv = ">=20.10.0" +[[package]] +name = "prometheus-client" +version = "0.25.0" +description = "Python client for the Prometheus monitoring system." +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"prometheus\"" +files = [ + {file = "prometheus_client-0.25.0-py3-none-any.whl", hash = "sha256:d5aec89e349a6ec230805d0df882f3807f74fd6c1a2fa86864e3c2279059fed1"}, + {file = "prometheus_client-0.25.0.tar.gz", hash = "sha256:5e373b75c31afb3c86f1a52fa1ad470c9aace18082d39ec0d2f918d11cc9ba28"}, +] + +[package.extras] +aiohttp = ["aiohttp"] +django = ["django"] +twisted = ["twisted"] + +[[package]] +name = "prometheus-fastapi-instrumentator" +version = "7.1.0" +description = "Instrument your FastAPI app with Prometheus metrics" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"prometheus\"" +files = [ + {file = "prometheus_fastapi_instrumentator-7.1.0-py3-none-any.whl", hash = "sha256:978130f3c0bb7b8ebcc90d35516a6fe13e02d2eb358c8f83887cdef7020c31e9"}, + {file = "prometheus_fastapi_instrumentator-7.1.0.tar.gz", hash = "sha256:be7cd61eeea4e5912aeccb4261c6631b3f227d8924542d79eaf5af3f439cbe5e"}, +] + +[package.dependencies] +prometheus-client = ">=0.8.0,<1.0.0" +starlette = ">=0.30.0,<1.0.0" + [[package]] name = "prompt-toolkit" version = "3.0.52" @@ -4609,14 +4644,14 @@ tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] [[package]] name = "starlette" -version = "1.0.0" +version = "0.52.1" description = "The little ASGI library that shines." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "starlette-1.0.0-py3-none-any.whl", hash = "sha256:d3ec55e0bb321692d275455ddfd3df75fff145d009685eb40dc91fc66b03d38b"}, - {file = "starlette-1.0.0.tar.gz", hash = "sha256:6a4beaf1f81bb472fd19ea9b918b50dc3a77a6f2e190a12954b25e6ed5eea149"}, + {file = "starlette-0.52.1-py3-none-any.whl", hash = "sha256:0029d43eb3d273bc4f83a08720b4912ea4b071087a3b48db01b7c839f7954d74"}, + {file = "starlette-0.52.1.tar.gz", hash = "sha256:834edd1b0a23167694292e94f597773bc3f89f362be6effee198165a35d62933"}, ] [package.dependencies] @@ -5282,6 +5317,7 @@ httpx = ["httpx"] kafka = ["faststream", "opentelemetry-instrumentation-confluent-kafka", "uvicorn"] mongo = ["beanie"] openai = ["openai"] +prometheus = ["prometheus-fastapi-instrumentator"] rabbit = ["faststream", "opentelemetry-instrumentation-aio-pika", "uvicorn"] redis = ["faststream", "redis-om"] requests = ["requests"] @@ -5290,4 +5326,4 @@ test = ["deepdiff", "freezegun", "pytest", "pytest-asyncio", "pytest-cov", "pyte [metadata] lock-version = "2.1" python-versions = ">=3.12,<3.14" -content-hash = "55a8b2480ebdd2258491e484245a4262418e60f988989f508214b488175317ed" +content-hash = "a5ffd9dbfa3e0e98d321a5c2180896c741aec12f37b8c89feba7ff08703fd432" diff --git a/pyproject.toml b/pyproject.toml index d3389e1..78ea7a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ kafka = [ ] redis = ["faststream[redis]", "redis-om>=1.0,<2.0"] fastapi = ["fastapi>=0,<1", "uvicorn", "python-multipart"] +prometheus = ["prometheus-fastapi-instrumentator>=7.0.0,<8.0.0"] mongo = ["beanie>=2.0.0,<3.0.0"] celery = ["celery>=5.5.3,<6.0.0"]