chore: add otel observability

kustomize/base/kustomization.yaml

@@ -55,6 +55,13 @@ resources:
   # Ingress
   - ingress.yaml
 
+# Replica overrides
+replicas:
+  - name: plane-api-wl
+    count: 5
+  - name: plane-worker-wl
+    count: 5
+
 # Default image tags
 images:
   - name: makeplane/backend-commercial

kustomize/components/otel-observability/configmap.yaml

@@ -0,0 +1,44 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: plane-otel-vars
+  namespace: plane
+data:
+  # OpenTelemetry instrumentation for the backend-api workload.
+  #
+  # These keys are PLACEHOLDERS — their real values are sourced from the overlay's
+  # vars.yaml (overlay-vars ConfigMap) via the OTEL replacements in the overlay
+  # kustomization.yaml. Leave a key empty here; set its value in vars.yaml.
+  #
+  # Optionality: the whole block is opt-in (enable this component in the overlay).
+  # To leave an individual variable unset, delete its line here AND the matching
+  # replacement in the overlay kustomization.yaml.
+  OTEL_ENABLED: ""
+  OTEL_EXPORTER_OTLP_ENDPOINT: ""
+  OTEL_EXPORTER_OTLP_PROTOCOL: ""
+  OTEL_SERVICE_NAME: ""
+  # Set deployment.environment.name (the CURRENT semconv key — NOT the legacy
+  # deployment.environment) so the API's environment tag matches the node/pi/
+  # browser services and cross-service "environment=" filtering works in Tempo.
+  OTEL_RESOURCE_ATTRIBUTES: ""
+  OTEL_DEBUG_CONSOLE: ""
+  # Trace sampling. Two independent gotchas:
+  #   1. Ratio: the app defaults to parentbased_traceidratio @ 0.1 (10%), so 90%
+  #      of root requests are NOT exported (their logs show trace_flags=0).
+  #   2. Parent-based: ANY parentbased_* sampler DEFERS to an incoming traceparent's
+  #      sampled flag and IGNORES the ARG ratio. With FRONTEND_OTEL_ENABLED=1 the
+  #      browser/proxy sends a traceparent; if it didn't sample (flag=0), the API
+  #      span is DROPPED regardless of ARG=1.0 — the classic "only GET traffic
+  #      (rootspans) shows up, no POST/user-action traces" symptom.
+  # => For a test/load/debug env, use always_on (captures every span the service
+  #    sees, ignores upstream decisions). For prod, use parentbased_traceidratio
+  #    with a sane ARG AND make sure browser tracing samples consistently.
+  OTEL_TRACES_SAMPLER: ""
+  OTEL_TRACES_SAMPLER_ARG: ""
+  # Browser/client tracing for web/admin/space. The API serves these to browsers
+  # via its public instance endpoint (is_otel_enabled/otlp_endpoint/otlp_headers),
+  # so they only need to be set on the API — which already consumes this ConfigMap.
+  # Read ONLY by the API; a no-op env on silo/live/space.
+  FRONTEND_OTEL_ENABLED: ""
+  FRONTEND_OTLP_ENDPOINT: ""
+  FRONTEND_OTLP_HEADERS: ""

kustomize/components/otel-observability/kustomization.yaml

@@ -0,0 +1,113 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+# ============================================================
+# OTEL OBSERVABILITY (backend-api)
+# ============================================================
+# Injects OpenTelemetry environment variables into the Plane workloads by
+# appending the plane-otel-vars ConfigMap to each container's envFrom.
+#
+# SCOPE — only env-driven OTel consumers are wired here:
+#   - api                 (Django wsgi/asgi, OTEL_* env)
+#   - worker              (Celery worker — Celery task spans + DB/Redis/HTTP)
+#   - beat-worker         (Celery beat)
+#   - automation-consumer (Django mgmt cmd; OTel bootstraps via plane.celery import)
+#   - outbox-poller       (Django mgmt cmd; same)
+#   - silo                (Node startOtel, OTEL_* env)
+#   - live                (Node startOtel, OTEL_* env)
+#   - space               (React Router SSR initServerTelemetry, OTEL_ENABLED /
+#                          OTEL_EXPORTER_OTLP_ENDPOINT / OTEL_SERVICE_NAME env)
+# All five Django processes bootstrap OTel because plane/__init__.py imports the
+# Celery app, which calls configure_otel() at import.
+# web and admin are NOT wired: their only OTel is BROWSER/client tracing, which
+# is configured from the Plane instance config (is_otel_enabled, otlp_endpoint)
+# served by the API via the FRONTEND_OTEL_* keys — not from per-workload env — so
+# OTEL_* env on them is a no-op.
+#
+# VALUES LIVE IN vars.yaml: the keys in configmap.yaml are placeholders; the
+# actual values are supplied per-overlay from overlay-vars (vars.yaml) via the
+# OTEL replacements added in the overlay's kustomization.yaml.
+#
+# PER-SERVICE service.name: the shared plane-otel-vars ConfigMap carries
+# OTEL_SERVICE_NAME for the api (from vars.yaml). Every other workload gets the
+# SAME ConfigMap but an inline env OTEL_SERVICE_NAME=<name> that overrides it
+# (inline env wins over envFrom), so each reports its own service.name. The name
+# matches the container name so it lines up with the promtail service_name label
+# for trace->logs correlation.
+#
+# OPTIONAL / OPT-IN: the env vars are added ONLY when this component is enabled
+# in the overlay. To leave OTEL off entirely, do not enable this component — no
+# OTEL env vars are then set on the api container. To drop a single variable,
+# remove its key from configmap.yaml and its replacement from the overlay.
+#
+# ORDERING: this envFrom entry is appended LAST. Enable this component AFTER
+# custom-ca so the OTEL ConfigMap lands after the CA env ConfigMap.
+#
+# Usage — enable in your overlay's kustomization.yaml and add the OTEL
+# replacements + vars (see overlays/phoenix for a worked example):
+#   components:
+#     - ../../components/otel-observability
+# ============================================================
+
+resources:
+  - configmap.yaml
+
+patches:
+  - path: patches/append-otel-envfrom.json
+    target:
+      kind: Deployment
+      name: plane-api-wl
+  - path: patches/append-otel-silo.json
+    target:
+      kind: Deployment
+      name: plane-silo-wl
+  - path: patches/append-otel-live.json
+    target:
+      kind: Deployment
+      name: plane-live-wl
+  - path: patches/append-otel-space.json
+    target:
+      kind: Deployment
+      name: plane-space-wl
+  - path: patches/append-otel-worker.json
+    target:
+      kind: Deployment
+      name: plane-worker-wl
+  - path: patches/append-otel-beat-worker.json
+    target:
+      kind: Deployment
+      name: plane-beat-worker-wl
+  - path: patches/append-otel-automation-consumer.json
+    target:
+      kind: Deployment
+      name: plane-automation-consumer-wl
+  - path: patches/append-otel-outbox-poller.json
+    target:
+      kind: Deployment
+      name: plane-outbox-poller-wl
+  - path: patches/append-otel-external-api.json
+    target:
+      kind: Deployment
+      name: plane-external-api-wl
+  - path: patches/append-otel-webhook-consumer.json
+    target:
+      kind: Deployment
+      name: plane-webhook-consumer-wl
+  - path: patches/append-otel-worker-importers.json
+    target:
+      kind: Deployment
+      name: plane-worker-importers-wl
+  # Plane Intelligence (pi) — logs-only OTel (configure_otel in pi.celery_app;
+  # reads the standard OTEL_* env, exports logs over OTLP/HTTP to /v1/logs).
+  - path: patches/append-otel-pi-api.json
+    target:
+      kind: Deployment
+      name: plane-pi-api-wl
+  - path: patches/append-otel-pi-beat.json
+    target:
+      kind: Deployment
+      name: plane-pi-beat-wl
+  - path: patches/append-otel-pi-worker.json
+    target:
+      kind: Deployment
+      name: plane-pi-worker-wl

kustomize/components/otel-observability/patches/append-otel-automation-consumer.json

@@ -0,0 +1,21 @@
+[
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/envFrom/-",
+    "value": {
+      "configMapRef": {
+        "name": "plane-otel-vars"
+      }
+    }
+  },
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/env",
+    "value": [
+      {
+        "name": "OTEL_SERVICE_NAME",
+        "value": "automation-consumer"
+      }
+    ]
+  }
+]

kustomize/components/otel-observability/patches/append-otel-beat-worker.json

@@ -0,0 +1,21 @@
+[
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/envFrom/-",
+    "value": {
+      "configMapRef": {
+        "name": "plane-otel-vars"
+      }
+    }
+  },
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/env",
+    "value": [
+      {
+        "name": "OTEL_SERVICE_NAME",
+        "value": "beat-worker"
+      }
+    ]
+  }
+]

kustomize/components/otel-observability/patches/append-otel-envfrom.json

@@ -0,0 +1,11 @@
+[
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/envFrom/-",
+    "value": {
+      "configMapRef": {
+        "name": "plane-otel-vars"
+      }
+    }
+  }
+]

kustomize/components/otel-observability/patches/append-otel-external-api.json

@@ -0,0 +1,12 @@
+[
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/envFrom/-",
+    "value": { "configMapRef": { "name": "plane-otel-vars" } }
+  },
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/env/-",
+    "value": { "name": "OTEL_SERVICE_NAME", "value": "external-api" }
+  }
+]

kustomize/components/otel-observability/patches/append-otel-live.json

@@ -0,0 +1,21 @@
+[
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/envFrom/-",
+    "value": {
+      "configMapRef": {
+        "name": "plane-otel-vars"
+      }
+    }
+  },
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/env",
+    "value": [
+      {
+        "name": "OTEL_SERVICE_NAME",
+        "value": "live"
+      }
+    ]
+  }
+]

kustomize/components/otel-observability/patches/append-otel-outbox-poller.json

@@ -0,0 +1,21 @@
+[
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/envFrom/-",
+    "value": {
+      "configMapRef": {
+        "name": "plane-otel-vars"
+      }
+    }
+  },
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/env",
+    "value": [
+      {
+        "name": "OTEL_SERVICE_NAME",
+        "value": "outbox-poller"
+      }
+    ]
+  }
+]

kustomize/components/otel-observability/patches/append-otel-pi-api.json

@@ -0,0 +1,12 @@
+[
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/envFrom/-",
+    "value": { "configMapRef": { "name": "plane-otel-vars" } }
+  },
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/env",
+    "value": [ { "name": "OTEL_SERVICE_NAME", "value": "pi-api" } ]
+  }
+]

kustomize/components/otel-observability/patches/append-otel-pi-beat.json

@@ -0,0 +1,12 @@
+[
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/envFrom/-",
+    "value": { "configMapRef": { "name": "plane-otel-vars" } }
+  },
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/env",
+    "value": [ { "name": "OTEL_SERVICE_NAME", "value": "pi-beat" } ]
+  }
+]

kustomize/components/otel-observability/patches/append-otel-pi-worker.json

@@ -0,0 +1,12 @@
+[
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/envFrom/-",
+    "value": { "configMapRef": { "name": "plane-otel-vars" } }
+  },
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/env",
+    "value": [ { "name": "OTEL_SERVICE_NAME", "value": "pi-worker" } ]
+  }
+]

kustomize/components/otel-observability/patches/append-otel-silo.json

@@ -0,0 +1,21 @@
+[
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/envFrom/-",
+    "value": {
+      "configMapRef": {
+        "name": "plane-otel-vars"
+      }
+    }
+  },
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/env",
+    "value": [
+      {
+        "name": "OTEL_SERVICE_NAME",
+        "value": "silo"
+      }
+    ]
+  }
+]

kustomize/components/otel-observability/patches/append-otel-space.json

@@ -0,0 +1,21 @@
+[
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/envFrom/-",
+    "value": {
+      "configMapRef": {
+        "name": "plane-otel-vars"
+      }
+    }
+  },
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/env",
+    "value": [
+      {
+        "name": "OTEL_SERVICE_NAME",
+        "value": "space"
+      }
+    ]
+  }
+]

kustomize/components/otel-observability/patches/append-otel-webhook-consumer.json

@@ -0,0 +1,12 @@
+[
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/envFrom/-",
+    "value": { "configMapRef": { "name": "plane-otel-vars" } }
+  },
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/env",
+    "value": [ { "name": "OTEL_SERVICE_NAME", "value": "webhook-consumer" } ]
+  }
+]

kustomize/components/otel-observability/patches/append-otel-worker-importers.json

@@ -0,0 +1,12 @@
+[
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/envFrom/-",
+    "value": { "configMapRef": { "name": "plane-otel-vars" } }
+  },
+  {
+    "op": "add",
+    "path": "/spec/template/spec/containers/0/env/-",
+    "value": { "name": "OTEL_SERVICE_NAME", "value": "worker-importers" }
+  }
+]