From fc1fc8b75a862998c4d8782e29472bd9f470313f Mon Sep 17 00:00:00 2001 From: Pratapa Lakshmi Date: Tue, 7 Apr 2026 13:45:56 +0530 Subject: [PATCH 01/15] feat: add optional email service NLB with deployment instructions --- README.md | 57 +++++++ .../components/email-service/service.yaml | 6 +- terraform/main.tf | 13 ++ terraform/modules/email-nlb/main.tf | 160 ++++++++++++++++++ terraform/modules/email-nlb/outputs.tf | 23 +++ terraform/modules/email-nlb/variables.tf | 43 +++++ terraform/outputs.tf | 5 + terraform/variables.tf | 6 + 8 files changed, 311 insertions(+), 2 deletions(-) create mode 100644 terraform/modules/email-nlb/main.tf create mode 100644 terraform/modules/email-nlb/outputs.tf create mode 100644 terraform/modules/email-nlb/variables.tf diff --git a/README.md b/README.md index 2a6cc17..19c8d91 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,63 @@ module "plane_infra" { } ``` +### Email Service NLB (optional) + +Plane includes a built-in email service for receiving inbound email. To expose it externally via a dedicated AWS Network Load Balancer, set `enable_email_nlb = true`. + +> **Important:** Deploy in two stages. The NLB targets EKS worker nodes via fixed NodePorts, so the EKS cluster and the email service must exist before the NLB is created. + +**Stage 1 — Deploy infrastructure and the Plane application:** + +```hcl +module "plane_infra" { + source = "git::https://github.com/makeplane/commercial-deployments.git//terraform?ref=main" + + cluster_name = "plane-eks-cluster" + region = "us-west-2" + enable_aws_lb_controller = true + # enable_email_nlb = false # default — omit until Stage 2 +} +``` + +```bash +terraform apply +aws eks update-kubeconfig --region us-west-2 --name plane-eks-cluster +kubectl apply -k kustomize/overlays/example # deploys email-service with fixed NodePorts +``` + +**Stage 2 — Create the NLB once the email service is running:** + +```hcl +module "plane_infra" { + source = "git::https://github.com/makeplane/commercial-deployments.git//terraform?ref=main" + + cluster_name = "plane-eks-cluster" + region = "us-west-2" + enable_aws_lb_controller = true + enable_email_nlb = true # creates NLB + registers EKS nodes in target groups +} +``` + +```bash +terraform apply +terraform output email_nlb_dns_name # use this value for your MX record +``` + +**Route53 MX record:** + +After apply, create the following records in Route53 for your domain: + +| Type | Name | TTL | Value | +|------|------|-----|-------| +| MX | `yourdomain.com` | 300 | `10 .` | +| TXT | `yourdomain.com` | 300 | `"v=spf1 include: ~all"` | + +The NLB listens on: +- Port **25** — SMTP (inbound email from other mail servers) +- Port **465** — SMTPS +- Port **587** — Submission + Override defaults by passing `eks`, `cache`, `mq`, `opensearch`, `object_store`, or `db` objects. See [terraform/README.md](terraform/README.md) for all options. ### Outputs diff --git a/kustomize/components/email-service/service.yaml b/kustomize/components/email-service/service.yaml index 97c68ff..d872827 100644 --- a/kustomize/components/email-service/service.yaml +++ b/kustomize/components/email-service/service.yaml @@ -7,8 +7,7 @@ metadata: app.kubernetes.io/name: plane-enterprise app.kubernetes.io/component: email-service spec: - type: LoadBalancer - externalTrafficPolicy: Local # Important for email servers + type: NodePort selector: app.kubernetes.io/name: plane-enterprise app.kubernetes.io/component: email-service @@ -16,12 +15,15 @@ spec: - name: smtp port: 25 targetPort: 10025 + nodePort: 30025 protocol: TCP - name: smtps port: 465 targetPort: 10465 + nodePort: 30465 protocol: TCP - name: submission port: 587 targetPort: 10587 + nodePort: 30587 protocol: TCP \ No newline at end of file diff --git a/terraform/main.tf b/terraform/main.tf index f3fb4f7..31f659a 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -61,6 +61,19 @@ module "aws_lb_controller" { depends_on = [module.eks] } +module "email_nlb" { + count = var.enable_email_nlb ? 1 : 0 + source = "./modules/email-nlb" + + cluster_name = var.cluster_name + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.public_subnet_ids + node_security_group_id = module.eks.node_security_group_id + tags = var.tags + + depends_on = [module.vpc, module.eks] +} + resource "random_password" "opensearch" { length = 32 special = true diff --git a/terraform/modules/email-nlb/main.tf b/terraform/modules/email-nlb/main.tf new file mode 100644 index 0000000..96c1581 --- /dev/null +++ b/terraform/modules/email-nlb/main.tf @@ -0,0 +1,160 @@ +# Discover the EKS node group ASG so we can attach target groups to it. +# EKS managed node groups tag their ASGs with eks:cluster-name automatically. +data "aws_autoscaling_groups" "eks_nodes" { + filter { + name = "tag:eks:cluster-name" + values = [var.cluster_name] + } +} + +# Allow inbound traffic from NLB to EKS node NodePorts +resource "aws_security_group_rule" "smtp_nodeport" { + type = "ingress" + from_port = var.smtp_node_port + to_port = var.smtp_node_port + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + security_group_id = var.node_security_group_id + description = "Email NLB -> SMTP NodePort" +} + +resource "aws_security_group_rule" "smtps_nodeport" { + type = "ingress" + from_port = var.smtps_node_port + to_port = var.smtps_node_port + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + security_group_id = var.node_security_group_id + description = "Email NLB -> SMTPS NodePort" +} + +resource "aws_security_group_rule" "submission_nodeport" { + type = "ingress" + from_port = var.submission_node_port + to_port = var.submission_node_port + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + security_group_id = var.node_security_group_id + description = "Email NLB -> Submission NodePort" +} + +# Target groups (instance target type — routes to NodePorts on worker nodes) +resource "aws_lb_target_group" "smtp" { + name = "${var.cluster_name}-email-smtp" + port = var.smtp_node_port + protocol = "TCP" + vpc_id = var.vpc_id + target_type = "instance" + + health_check { + protocol = "TCP" + port = tostring(var.smtp_node_port) + healthy_threshold = 2 + unhealthy_threshold = 2 + interval = 10 + } + + tags = merge(var.tags, { Name = "${var.cluster_name}-email-smtp" }) +} + +resource "aws_lb_target_group" "smtps" { + name = "${var.cluster_name}-email-smtps" + port = var.smtps_node_port + protocol = "TCP" + vpc_id = var.vpc_id + target_type = "instance" + + health_check { + protocol = "TCP" + port = tostring(var.smtps_node_port) + healthy_threshold = 2 + unhealthy_threshold = 2 + interval = 10 + } + + tags = merge(var.tags, { Name = "${var.cluster_name}-email-smtps" }) +} + +resource "aws_lb_target_group" "submission" { + name = "${var.cluster_name}-email-sub" + port = var.submission_node_port + protocol = "TCP" + vpc_id = var.vpc_id + target_type = "instance" + + health_check { + protocol = "TCP" + port = tostring(var.submission_node_port) + healthy_threshold = 2 + unhealthy_threshold = 2 + interval = 10 + } + + tags = merge(var.tags, { Name = "${var.cluster_name}-email-sub" }) +} + +# Internet-facing Network Load Balancer +resource "aws_lb" "email" { + name = "${var.cluster_name}-email" + internal = false + load_balancer_type = "network" + subnets = var.subnet_ids + + enable_deletion_protection = false + + tags = merge(var.tags, { Name = "${var.cluster_name}-email-nlb" }) +} + +# Listeners +resource "aws_lb_listener" "smtp" { + load_balancer_arn = aws_lb.email.arn + port = 25 + protocol = "TCP" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.smtp.arn + } +} + +resource "aws_lb_listener" "smtps" { + load_balancer_arn = aws_lb.email.arn + port = 465 + protocol = "TCP" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.smtps.arn + } +} + +resource "aws_lb_listener" "submission" { + load_balancer_arn = aws_lb.email.arn + port = 587 + protocol = "TCP" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.submission.arn + } +} + +# Attach each target group to every EKS node ASG so nodes are registered +# automatically as the node group scales up/down. +resource "aws_autoscaling_attachment" "smtp" { + for_each = toset(data.aws_autoscaling_groups.eks_nodes.names) + autoscaling_group_name = each.value + lb_target_group_arn = aws_lb_target_group.smtp.arn +} + +resource "aws_autoscaling_attachment" "smtps" { + for_each = toset(data.aws_autoscaling_groups.eks_nodes.names) + autoscaling_group_name = each.value + lb_target_group_arn = aws_lb_target_group.smtps.arn +} + +resource "aws_autoscaling_attachment" "submission" { + for_each = toset(data.aws_autoscaling_groups.eks_nodes.names) + autoscaling_group_name = each.value + lb_target_group_arn = aws_lb_target_group.submission.arn +} diff --git a/terraform/modules/email-nlb/outputs.tf b/terraform/modules/email-nlb/outputs.tf new file mode 100644 index 0000000..dc717c8 --- /dev/null +++ b/terraform/modules/email-nlb/outputs.tf @@ -0,0 +1,23 @@ +output "nlb_arn" { + description = "ARN of the email NLB" + value = aws_lb.email.arn +} + +output "nlb_dns_name" { + description = "DNS name of the email NLB — use as MX record target" + value = aws_lb.email.dns_name +} + +output "nlb_zone_id" { + description = "Route53 hosted zone ID of the NLB — use for alias records" + value = aws_lb.email.zone_id +} + +output "target_group_arns" { + description = "Target group ARNs keyed by port name (smtp, smtps, submission)" + value = { + smtp = aws_lb_target_group.smtp.arn + smtps = aws_lb_target_group.smtps.arn + submission = aws_lb_target_group.submission.arn + } +} diff --git a/terraform/modules/email-nlb/variables.tf b/terraform/modules/email-nlb/variables.tf new file mode 100644 index 0000000..6c3f9b5 --- /dev/null +++ b/terraform/modules/email-nlb/variables.tf @@ -0,0 +1,43 @@ +variable "cluster_name" { + description = "EKS cluster name (used for naming and tagging)" + type = string +} + +variable "vpc_id" { + description = "VPC ID where the NLB will be created" + type = string +} + +variable "subnet_ids" { + description = "Public subnet IDs for the internet-facing NLB (one per AZ)" + type = list(string) +} + +variable "node_security_group_id" { + description = "Security group ID of EKS worker nodes — NodePort ingress rules are added here" + type = string +} + +variable "smtp_node_port" { + description = "Fixed NodePort for SMTP (port 25)" + type = number + default = 30025 +} + +variable "smtps_node_port" { + description = "Fixed NodePort for SMTPS (port 465)" + type = number + default = 30465 +} + +variable "submission_node_port" { + description = "Fixed NodePort for Submission (port 587)" + type = number + default = 30587 +} + +variable "tags" { + description = "Additional tags to apply to all resources" + type = map(string) + default = {} +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf index db78bfe..bd42654 100644 --- a/terraform/outputs.tf +++ b/terraform/outputs.tf @@ -119,4 +119,9 @@ output "rds_password_secret_arn" { description = "ARN of the RDS master user password secret in Secrets Manager" value = module.rds.master_user_secret[0].secret_arn sensitive = true +} + +output "email_nlb_dns_name" { + description = "DNS name of the email NLB — add as MX record in Route53 (e.g. 10 .). Null when enable_email_nlb = false." + value = var.enable_email_nlb ? module.email_nlb[0].nlb_dns_name : null } \ No newline at end of file diff --git a/terraform/variables.tf b/terraform/variables.tf index 0e2de7b..c1be129 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -72,6 +72,12 @@ variable "enable_aws_lb_controller" { default = false } +variable "enable_email_nlb" { + description = "Create an internet-facing NLB for the email service (ports 25/465/587) and add NodePort ingress rules to EKS nodes" + type = bool + default = false +} + variable "cache" { description = "ElastiCache Redis configuration" type = object({ From c174d278c1e16c0184cea932645bae34e09fca39 Mon Sep 17 00:00:00 2001 From: Pratapa Lakshmi Date: Fri, 12 Jun 2026 17:15:37 +0530 Subject: [PATCH 02/15] feat: add draw.io kustomize component Add a cluster-internal self-hosted draw.io as an optional Kustomize component (Deployment + ClusterIP Service, image jgraph/drawio:30.0.4) and enable it in the default, managed, and phoenix overlays. - Reject plane-draw-io-wl from each overlay's APP_VERSION image-tag replacement so the pinned third-party tag is not clobbered. - In phoenix, order draw-io after nonroot-security-context and custom-ca (it has no envFrom, like iframely) and reject it from the initContainers image replacement since it has no init container. Co-Authored-By: Claude Opus 4.8 (1M context) --- kustomize/components/draw-io/deployment.yaml | 41 +++++++++++++++++++ .../components/draw-io/kustomization.yaml | 19 +++++++++ kustomize/components/draw-io/service.yaml | 18 ++++++++ kustomize/overlays/default/kustomization.yaml | 4 ++ kustomize/overlays/managed/kustomization.yaml | 4 ++ kustomize/overlays/phoenix/kustomization.yaml | 6 +++ 6 files changed, 92 insertions(+) create mode 100644 kustomize/components/draw-io/deployment.yaml create mode 100644 kustomize/components/draw-io/kustomization.yaml create mode 100644 kustomize/components/draw-io/service.yaml diff --git a/kustomize/components/draw-io/deployment.yaml b/kustomize/components/draw-io/deployment.yaml new file mode 100644 index 0000000..28c4f8a --- /dev/null +++ b/kustomize/components/draw-io/deployment.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: plane-draw-io-wl + namespace: plane + labels: + app.kubernetes.io/name: plane-enterprise + app.kubernetes.io/component: draw-io +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: plane-enterprise + app.kubernetes.io/component: draw-io + template: + metadata: + labels: + app.kubernetes.io/name: plane-enterprise + app.kubernetes.io/component: draw-io + spec: + serviceAccountName: plane-srv-account + containers: + - name: draw-io + image: jgraph/drawio:30.0.4 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + readinessProbe: + tcpSocket: + port: 8080 + failureThreshold: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + memory: 256Mi + cpu: 100m + limits: + memory: 512Mi + cpu: 500m diff --git a/kustomize/components/draw-io/kustomization.yaml b/kustomize/components/draw-io/kustomization.yaml new file mode 100644 index 0000000..77b4440 --- /dev/null +++ b/kustomize/components/draw-io/kustomization.yaml @@ -0,0 +1,19 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +# ============================================================ +# DRAW.IO (diagram editor) +# ============================================================ +# Cluster-internal self-hosted draw.io. Ships a Deployment + ClusterIP +# Service only; reachable in-cluster at plane-draw-io:8080. No public +# ingress is added here — wire it up at the overlay/infra layer if a +# public route is needed. +# +# IMAGE PINNING: the Deployment uses jgraph/drawio:30.0.4 (third-party, +# NOT driven by APP_VERSION). The overlay's APP_VERSION → image-tag +# replacement must reject plane-draw-io-wl, otherwise the tag is clobbered. +# ============================================================ + +resources: + - deployment.yaml + - service.yaml diff --git a/kustomize/components/draw-io/service.yaml b/kustomize/components/draw-io/service.yaml new file mode 100644 index 0000000..b3a7b4f --- /dev/null +++ b/kustomize/components/draw-io/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: plane-draw-io + namespace: plane + labels: + app.kubernetes.io/name: plane-enterprise + app.kubernetes.io/component: draw-io +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: plane-enterprise + app.kubernetes.io/component: draw-io + ports: + - name: draw-io-8080 + port: 8080 + protocol: TCP + targetPort: 8080 diff --git a/kustomize/overlays/default/kustomization.yaml b/kustomize/overlays/default/kustomization.yaml index c9c605a..fe5aa9c 100644 --- a/kustomize/overlays/default/kustomization.yaml +++ b/kustomize/overlays/default/kustomization.yaml @@ -42,6 +42,9 @@ components: # Webhook consumer (RabbitMQ consumer for webhook delivery) - ../../components/webhook-consumer + # draw.io diagram editor (cluster-internal) + - ../../components/draw-io + # Optional: harden all workloads with non-root security context # - ../../components/nonroot-security-context @@ -184,6 +187,7 @@ replacements: kind: Deployment reject: - name: plane-iframely-wl + - name: plane-draw-io-wl # pinned jgraph/drawio tag — not APP_VERSION-driven fieldPaths: - spec.template.spec.containers.*.image options: diff --git a/kustomize/overlays/managed/kustomization.yaml b/kustomize/overlays/managed/kustomization.yaml index 1211565..0cae2e0 100644 --- a/kustomize/overlays/managed/kustomization.yaml +++ b/kustomize/overlays/managed/kustomization.yaml @@ -42,6 +42,9 @@ components: # Webhook consumer (RabbitMQ consumer for webhook delivery) - ../../components/webhook-consumer + # draw.io diagram editor (cluster-internal) + - ../../components/draw-io + # Optional: harden all workloads with non-root security context # - ../../components/nonroot-security-context @@ -139,6 +142,7 @@ replacements: kind: Deployment reject: - name: plane-iframely-wl + - name: plane-draw-io-wl # pinned jgraph/drawio tag — not APP_VERSION-driven fieldPaths: - spec.template.spec.containers.*.image options: diff --git a/kustomize/overlays/phoenix/kustomization.yaml b/kustomize/overlays/phoenix/kustomization.yaml index 2dfbe6e..7f4a7ca 100644 --- a/kustomize/overlays/phoenix/kustomization.yaml +++ b/kustomize/overlays/phoenix/kustomization.yaml @@ -49,6 +49,10 @@ components: # iframely ElastiCache/redis cache. MUST come after custom-ca: it appends to the # envFrom array that custom-ca creates on the base iframely deployment. - ../../components/iframely + # draw.io diagram editor (cluster-internal). MUST come after nonroot-security-context + # and custom-ca: it has no envFrom (like iframely) and its third-party image should + # not be patched by those components, which target all plane-enterprise Deployments. + - ../../components/draw-io # Uncomment these if you want to deploy the infrastructure components locally # - ../../components/postgres # - ../../components/redis @@ -181,6 +185,7 @@ replacements: reject: - name: plane-iframely-wl - name: plane-mcp-server + - name: plane-draw-io-wl # pinned jgraph/drawio tag — not APP_VERSION-driven fieldPaths: - spec.template.spec.containers.*.image options: @@ -190,6 +195,7 @@ replacements: kind: Deployment reject: - name: plane-iframely-wl + - name: plane-draw-io-wl # no init container (added after custom-ca) fieldPaths: - spec.template.spec.initContainers.*.image options: From bac70ac2a339467d435181d7b28ef4bf9249f853 Mon Sep 17 00:00:00 2001 From: Pratapa Lakshmi Date: Fri, 12 Jun 2026 17:19:34 +0530 Subject: [PATCH 03/15] feat: drive draw.io image tag from vars.yaml DRAWIO_VERSION Source the draw-io image tag from overlay-vars.DRAWIO_VERSION via a replacement in the component (mirrors how iframely reads overlay-vars), so version bumps are a one-line change in each overlay's vars.yaml instead of editing the component deployment.yaml. The in-file tag is now just a placeholder default. Document the new key in each overlay's vars.yaml.example (the live vars.yaml is gitignored / user-provided). Co-Authored-By: Claude Opus 4.8 (1M context) --- kustomize/components/draw-io/deployment.yaml | 2 +- .../components/draw-io/kustomization.yaml | 24 ++++++++++++++++--- kustomize/overlays/default/vars.yaml.example | 3 +++ kustomize/overlays/managed/vars.yaml.example | 3 +++ kustomize/overlays/phoenix/vars.yaml.example | 2 ++ 5 files changed, 30 insertions(+), 4 deletions(-) diff --git a/kustomize/components/draw-io/deployment.yaml b/kustomize/components/draw-io/deployment.yaml index 28c4f8a..564cc3d 100644 --- a/kustomize/components/draw-io/deployment.yaml +++ b/kustomize/components/draw-io/deployment.yaml @@ -21,7 +21,7 @@ spec: serviceAccountName: plane-srv-account containers: - name: draw-io - image: jgraph/drawio:30.0.4 + image: jgraph/drawio:30.0.4 # tag overridden by overlay-vars.DRAWIO_VERSION (vars.yaml) imagePullPolicy: IfNotPresent ports: - containerPort: 8080 diff --git a/kustomize/components/draw-io/kustomization.yaml b/kustomize/components/draw-io/kustomization.yaml index 77b4440..dff95aa 100644 --- a/kustomize/components/draw-io/kustomization.yaml +++ b/kustomize/components/draw-io/kustomization.yaml @@ -9,11 +9,29 @@ kind: Component # ingress is added here — wire it up at the overlay/infra layer if a # public route is needed. # -# IMAGE PINNING: the Deployment uses jgraph/drawio:30.0.4 (third-party, -# NOT driven by APP_VERSION). The overlay's APP_VERSION → image-tag -# replacement must reject plane-draw-io-wl, otherwise the tag is clobbered. +# IMAGE VERSIONING: third-party image, independently versioned (NOT driven +# by APP_VERSION). The tag is sourced from overlay-vars.DRAWIO_VERSION via the +# replacement below — bump the version in each overlay's vars.yaml, not in +# deployment.yaml (the tag there is only a placeholder default). The overlay's +# APP_VERSION → image-tag replacement must still reject plane-draw-io-wl. # ============================================================ resources: - deployment.yaml - service.yaml + +replacements: + # Drive the draw.io image tag from overlay-vars.DRAWIO_VERSION (vars.yaml). + - source: + kind: ConfigMap + name: overlay-vars + fieldPath: data.DRAWIO_VERSION + targets: + - select: + kind: Deployment + name: plane-draw-io-wl + fieldPaths: + - spec.template.spec.containers.*.image + options: + delimiter: ":" + index: 1 diff --git a/kustomize/overlays/default/vars.yaml.example b/kustomize/overlays/default/vars.yaml.example index 334f7f5..0ddff0c 100644 --- a/kustomize/overlays/default/vars.yaml.example +++ b/kustomize/overlays/default/vars.yaml.example @@ -10,6 +10,9 @@ data: # Application version used across all images and config APP_VERSION: "latest" + # draw.io image tag (third-party, independently versioned) — used by the draw-io component + DRAWIO_VERSION: "30.0.4" + # Domain configuration — UPDATE THESE before applying APP_DOMAIN: "kustomize.plane.town" diff --git a/kustomize/overlays/managed/vars.yaml.example b/kustomize/overlays/managed/vars.yaml.example index 167d28d..31c9e73 100644 --- a/kustomize/overlays/managed/vars.yaml.example +++ b/kustomize/overlays/managed/vars.yaml.example @@ -16,6 +16,9 @@ data: # Application version used across all images and config APP_VERSION: "latest" + # draw.io image tag (third-party, independently versioned) — used by the draw-io component + DRAWIO_VERSION: "30.0.4" + # Domain configuration — set to your actual domain (without protocol) APP_DOMAIN: "plane.example.com" diff --git a/kustomize/overlays/phoenix/vars.yaml.example b/kustomize/overlays/phoenix/vars.yaml.example index 4d34508..429a4d6 100644 --- a/kustomize/overlays/phoenix/vars.yaml.example +++ b/kustomize/overlays/phoenix/vars.yaml.example @@ -15,6 +15,8 @@ metadata: data: # Application version used across all images and config APP_VERSION: "v2.3.4" + # draw.io image tag (third-party, independently versioned) — used by the draw-io component + DRAWIO_VERSION: "30.0.4" # Domain configuration (without protocol) APP_DOMAIN: "plane.mycompany.com" From a3341468789aa309bfc9295644132ba3b9ab731a Mon Sep 17 00:00:00 2001 From: Pratapa Lakshmi Date: Mon, 15 Jun 2026 17:42:22 +0530 Subject: [PATCH 04/15] chore: add otel observability --- .../otel-observability/configmap.yaml | 21 ++++++ .../otel-observability/kustomization.yaml | 36 ++++++++++ .../patches/append-otel-envfrom.json | 11 +++ kustomize/overlays/default/kustomization.yaml | 5 ++ kustomize/overlays/managed/kustomization.yaml | 5 ++ kustomize/overlays/phoenix/kustomization.yaml | 72 +++++++++++++++++++ 6 files changed, 150 insertions(+) create mode 100644 kustomize/components/otel-observability/configmap.yaml create mode 100644 kustomize/components/otel-observability/kustomization.yaml create mode 100644 kustomize/components/otel-observability/patches/append-otel-envfrom.json diff --git a/kustomize/components/otel-observability/configmap.yaml b/kustomize/components/otel-observability/configmap.yaml new file mode 100644 index 0000000..c5e96ac --- /dev/null +++ b/kustomize/components/otel-observability/configmap.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: plane-otel-vars + namespace: plane +data: + # OpenTelemetry instrumentation for the backend-api workload. + # + # These keys are PLACEHOLDERS — their real values are sourced from the overlay's + # vars.yaml (overlay-vars ConfigMap) via the OTEL replacements in the overlay + # kustomization.yaml. Leave a key empty here; set its value in vars.yaml. + # + # Optionality: the whole block is opt-in (enable this component in the overlay). + # To leave an individual variable unset, delete its line here AND the matching + # replacement in the overlay kustomization.yaml. + OTEL_ENABLED: "" + OTEL_EXPORTER_OTLP_ENDPOINT: "" + OTEL_EXPORTER_OTLP_PROTOCOL: "" + OTEL_SERVICE_NAME: "" + OTEL_RESOURCE_ATTRIBUTES: "" + OTEL_DEBUG_CONSOLE: "" diff --git a/kustomize/components/otel-observability/kustomization.yaml b/kustomize/components/otel-observability/kustomization.yaml new file mode 100644 index 0000000..ea8201a --- /dev/null +++ b/kustomize/components/otel-observability/kustomization.yaml @@ -0,0 +1,36 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +# ============================================================ +# OTEL OBSERVABILITY (backend-api) +# ============================================================ +# Injects OpenTelemetry environment variables into the backend-api workload +# (plane-api-wl) by appending the plane-otel-vars ConfigMap to the api +# container's envFrom. +# +# VALUES LIVE IN vars.yaml: the keys in configmap.yaml are placeholders; the +# actual values are supplied per-overlay from overlay-vars (vars.yaml) via the +# OTEL replacements added in the overlay's kustomization.yaml. +# +# OPTIONAL / OPT-IN: the env vars are added ONLY when this component is enabled +# in the overlay. To leave OTEL off entirely, do not enable this component — no +# OTEL env vars are then set on the api container. To drop a single variable, +# remove its key from configmap.yaml and its replacement from the overlay. +# +# ORDERING: this envFrom entry is appended LAST. Enable this component AFTER +# custom-ca so the OTEL ConfigMap lands after the CA env ConfigMap. +# +# Usage — enable in your overlay's kustomization.yaml and add the OTEL +# replacements + vars (see overlays/phoenix for a worked example): +# components: +# - ../../components/otel-observability +# ============================================================ + +resources: + - configmap.yaml + +patches: + - path: patches/append-otel-envfrom.json + target: + kind: Deployment + name: plane-api-wl diff --git a/kustomize/components/otel-observability/patches/append-otel-envfrom.json b/kustomize/components/otel-observability/patches/append-otel-envfrom.json new file mode 100644 index 0000000..d502b62 --- /dev/null +++ b/kustomize/components/otel-observability/patches/append-otel-envfrom.json @@ -0,0 +1,11 @@ +[ + { + "op": "add", + "path": "/spec/template/spec/containers/0/envFrom/-", + "value": { + "configMapRef": { + "name": "plane-otel-vars" + } + } + } +] diff --git a/kustomize/overlays/default/kustomization.yaml b/kustomize/overlays/default/kustomization.yaml index fe5aa9c..55db33e 100644 --- a/kustomize/overlays/default/kustomization.yaml +++ b/kustomize/overlays/default/kustomization.yaml @@ -45,6 +45,11 @@ components: # draw.io diagram editor (cluster-internal) - ../../components/draw-io + # Optional: OpenTelemetry env vars for the backend-api. Enable AFTER custom-ca, + # add the OTEL_* vars to vars.yaml and the OTEL replacements to this file + # (see overlays/phoenix for a worked example). + # - ../../components/otel-observability + # Optional: harden all workloads with non-root security context # - ../../components/nonroot-security-context diff --git a/kustomize/overlays/managed/kustomization.yaml b/kustomize/overlays/managed/kustomization.yaml index 0cae2e0..6af9797 100644 --- a/kustomize/overlays/managed/kustomization.yaml +++ b/kustomize/overlays/managed/kustomization.yaml @@ -52,6 +52,11 @@ components: # Paste your CA cert (PEM) into ../../components/custom-ca/customCA.crt, then uncomment: # - ../../components/custom-ca + # Optional: OpenTelemetry env vars for the backend-api. Enable AFTER custom-ca, + # add the OTEL_* vars to vars.yaml and the OTEL replacements to this file + # (see overlays/phoenix for a worked example). + # - ../../components/otel-observability + # Optional: external OpenSearch with username/password auth # - ../../components/opensearch-external-auth diff --git a/kustomize/overlays/phoenix/kustomization.yaml b/kustomize/overlays/phoenix/kustomization.yaml index 7f4a7ca..d8a5069 100644 --- a/kustomize/overlays/phoenix/kustomization.yaml +++ b/kustomize/overlays/phoenix/kustomization.yaml @@ -53,6 +53,10 @@ components: # and custom-ca: it has no envFrom (like iframely) and its third-party image should # not be patched by those components, which target all plane-enterprise Deployments. - ../../components/draw-io + # OpenTelemetry env vars for the backend-api. Comes after custom-ca so its + # ConfigMap is appended last to the api envFrom. Values are sourced from + # vars.yaml via the OTEL replacements below. + - ../../components/otel-observability # Uncomment these if you want to deploy the infrastructure components locally # - ../../components/postgres # - ../../components/redis @@ -980,3 +984,71 @@ replacements: name: plane-opensearch-secrets fieldPaths: - stringData.OPENSEARCH_INDEX_PREFIX + + # ====== OpenTelemetry (backend-api) ====== + # Drive plane-otel-vars (otel-observability component) from vars.yaml. + - source: + kind: ConfigMap + name: overlay-vars + fieldPath: data.OTEL_ENABLED + targets: + - select: + kind: ConfigMap + name: plane-otel-vars + fieldPaths: + - data.OTEL_ENABLED + + - source: + kind: ConfigMap + name: overlay-vars + fieldPath: data.OTEL_EXPORTER_OTLP_ENDPOINT + targets: + - select: + kind: ConfigMap + name: plane-otel-vars + fieldPaths: + - data.OTEL_EXPORTER_OTLP_ENDPOINT + + - source: + kind: ConfigMap + name: overlay-vars + fieldPath: data.OTEL_EXPORTER_OTLP_PROTOCOL + targets: + - select: + kind: ConfigMap + name: plane-otel-vars + fieldPaths: + - data.OTEL_EXPORTER_OTLP_PROTOCOL + + - source: + kind: ConfigMap + name: overlay-vars + fieldPath: data.OTEL_SERVICE_NAME + targets: + - select: + kind: ConfigMap + name: plane-otel-vars + fieldPaths: + - data.OTEL_SERVICE_NAME + + - source: + kind: ConfigMap + name: overlay-vars + fieldPath: data.OTEL_RESOURCE_ATTRIBUTES + targets: + - select: + kind: ConfigMap + name: plane-otel-vars + fieldPaths: + - data.OTEL_RESOURCE_ATTRIBUTES + + - source: + kind: ConfigMap + name: overlay-vars + fieldPath: data.OTEL_DEBUG_CONSOLE + targets: + - select: + kind: ConfigMap + name: plane-otel-vars + fieldPaths: + - data.OTEL_DEBUG_CONSOLE From eefef5f8c254df50a020ca363f442d2ad86760b2 Mon Sep 17 00:00:00 2001 From: Pratapa Lakshmi Date: Tue, 16 Jun 2026 13:18:16 +0530 Subject: [PATCH 05/15] add otel related env variables --- kustomize/overlays/phoenix/vars.yaml.example | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/kustomize/overlays/phoenix/vars.yaml.example b/kustomize/overlays/phoenix/vars.yaml.example index 429a4d6..15930e0 100644 --- a/kustomize/overlays/phoenix/vars.yaml.example +++ b/kustomize/overlays/phoenix/vars.yaml.example @@ -104,3 +104,21 @@ data: # MCP server: internal K8s service URL used for server-to-server Plane API calls # Follows the pattern: http://..svc.cluster.local: MCP_PLANE_INTERNAL_BASE_URL: "http://plane-api.plane-ns.svc.cluster.local:8000" + + # ============================================================ + # OpenTelemetry (backend-api) — used by the otel-observability component. + # Optional: only applied when ../../components/otel-observability is enabled. + # Delete a line here AND its replacement to leave that env var unset. + # ============================================================ + OTEL_ENABLED: "1" + OTEL_EXPORTER_OTLP_ENDPOINT: "https://telemetry.plane.town" + OTEL_EXPORTER_OTLP_PROTOCOL: "grpc" + OTEL_SERVICE_NAME: "api" # per-service: api, silo, live, etc. + # developer=.. is a placeholder — set it before applying. + # k8s.namespace.name lets Grafana's trace->logs map the span to this namespace's + # logs (must match the overlay's deploy namespace). Add k8s.pod.name via the + # downward API if you want pod-exact correlation. + OTEL_RESOURCE_ATTRIBUTES: "deployment.environment=loadtest,developer=,k8s.namespace.name=plane-ns" + # Debug: print spans to stdout to confirm the app is emitting/exporting them. + # Set to "" (or remove this line + its replacement) once tracing is confirmed. + OTEL_DEBUG_CONSOLE: "1" From e47f86a91608f14c7b11c44fdbf052465afbaec6 Mon Sep 17 00:00:00 2001 From: Pratapa Lakshmi Date: Tue, 16 Jun 2026 13:38:19 +0530 Subject: [PATCH 06/15] feat(otel-observability): wire OTel into silo with service.name=silo Extend the otel-observability component to also inject the shared plane-otel-vars ConfigMap into plane-silo-wl, plus an inline OTEL_SERVICE_NAME=silo that overrides the ConfigMap's api value (inline env wins over envFrom). So silo emits service.name=silo (api stays api) sharing the same endpoint + k8s.namespace.name resource attrs. Also documents the OTEL block in vars.yaml.example. Co-Authored-By: Claude Opus 4.8 --- .../otel-observability/kustomization.yaml | 15 ++++++++++--- .../patches/append-otel-silo.json | 21 +++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 kustomize/components/otel-observability/patches/append-otel-silo.json diff --git a/kustomize/components/otel-observability/kustomization.yaml b/kustomize/components/otel-observability/kustomization.yaml index ea8201a..4fda782 100644 --- a/kustomize/components/otel-observability/kustomization.yaml +++ b/kustomize/components/otel-observability/kustomization.yaml @@ -4,14 +4,19 @@ kind: Component # ============================================================ # OTEL OBSERVABILITY (backend-api) # ============================================================ -# Injects OpenTelemetry environment variables into the backend-api workload -# (plane-api-wl) by appending the plane-otel-vars ConfigMap to the api -# container's envFrom. +# Injects OpenTelemetry environment variables into the backend workloads +# (plane-api-wl, plane-silo-wl) by appending the plane-otel-vars ConfigMap to +# each container's envFrom. # # VALUES LIVE IN vars.yaml: the keys in configmap.yaml are placeholders; the # actual values are supplied per-overlay from overlay-vars (vars.yaml) via the # OTEL replacements added in the overlay's kustomization.yaml. # +# PER-SERVICE service.name: the shared plane-otel-vars ConfigMap carries +# OTEL_SERVICE_NAME for the api (from vars.yaml). silo gets the SAME ConfigMap +# but an inline env OTEL_SERVICE_NAME=silo that overrides it (inline env wins +# over envFrom), so each workload reports its own service.name in traces. +# # OPTIONAL / OPT-IN: the env vars are added ONLY when this component is enabled # in the overlay. To leave OTEL off entirely, do not enable this component — no # OTEL env vars are then set on the api container. To drop a single variable, @@ -34,3 +39,7 @@ patches: target: kind: Deployment name: plane-api-wl + - path: patches/append-otel-silo.json + target: + kind: Deployment + name: plane-silo-wl diff --git a/kustomize/components/otel-observability/patches/append-otel-silo.json b/kustomize/components/otel-observability/patches/append-otel-silo.json new file mode 100644 index 0000000..07b3b83 --- /dev/null +++ b/kustomize/components/otel-observability/patches/append-otel-silo.json @@ -0,0 +1,21 @@ +[ + { + "op": "add", + "path": "/spec/template/spec/containers/0/envFrom/-", + "value": { + "configMapRef": { + "name": "plane-otel-vars" + } + } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env", + "value": [ + { + "name": "OTEL_SERVICE_NAME", + "value": "silo" + } + ] + } +] From a359e497856fba6b0697f4311d21f14eee02de9b Mon Sep 17 00:00:00 2001 From: Pratapa Lakshmi Date: Thu, 18 Jun 2026 19:56:33 +0530 Subject: [PATCH 07/15] feat(otel-observability): wire OTel into live + space (service.name=live/space) Extend the otel-observability component to also inject the shared plane-otel-vars ConfigMap into the env-driven OTel consumers: - live (Node startOtel) -> inline OTEL_SERVICE_NAME=live - space (React Router SSR initServerTelemetry) -> inline OTEL_SERVICE_NAME=space Same pattern as silo: shared endpoint/resource-attrs from the ConfigMap, inline service name override per workload. web and admin are intentionally NOT wired: their only OTel is browser/client tracing, configured from the Plane instance config (is_otel_enabled, otlp_endpoint), not deployment env. OTEL_* env on them would be a no-op. Co-Authored-By: Claude Opus 4.8 --- .../otel-observability/kustomization.yaml | 29 +++++++++++++++---- .../patches/append-otel-live.json | 21 ++++++++++++++ .../patches/append-otel-space.json | 21 ++++++++++++++ 3 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 kustomize/components/otel-observability/patches/append-otel-live.json create mode 100644 kustomize/components/otel-observability/patches/append-otel-space.json diff --git a/kustomize/components/otel-observability/kustomization.yaml b/kustomize/components/otel-observability/kustomization.yaml index 4fda782..e34f370 100644 --- a/kustomize/components/otel-observability/kustomization.yaml +++ b/kustomize/components/otel-observability/kustomization.yaml @@ -5,17 +5,28 @@ kind: Component # OTEL OBSERVABILITY (backend-api) # ============================================================ # Injects OpenTelemetry environment variables into the backend workloads -# (plane-api-wl, plane-silo-wl) by appending the plane-otel-vars ConfigMap to -# each container's envFrom. +# (plane-api-wl, plane-silo-wl, plane-live-wl, plane-space-wl) by appending the +# plane-otel-vars ConfigMap to each container's envFrom. +# +# SCOPE — only env-driven OTel consumers are wired here: +# - api (Django, OTEL_* env) +# - silo (Node startOtel, OTEL_* env) +# - live (Node startOtel, OTEL_* env) +# - space (React Router SSR initServerTelemetry, OTEL_ENABLED / +# OTEL_EXPORTER_OTLP_ENDPOINT / OTEL_SERVICE_NAME env) +# web and admin are NOT wired: their only OTel is BROWSER/client tracing, which +# is configured from the Plane instance config (is_otel_enabled, otlp_endpoint) +# served by the API — not from deployment env vars — so OTEL_* env on them is a +# no-op. Enable those via instance settings, not this component. # # VALUES LIVE IN vars.yaml: the keys in configmap.yaml are placeholders; the # actual values are supplied per-overlay from overlay-vars (vars.yaml) via the # OTEL replacements added in the overlay's kustomization.yaml. # # PER-SERVICE service.name: the shared plane-otel-vars ConfigMap carries -# OTEL_SERVICE_NAME for the api (from vars.yaml). silo gets the SAME ConfigMap -# but an inline env OTEL_SERVICE_NAME=silo that overrides it (inline env wins -# over envFrom), so each workload reports its own service.name in traces. +# OTEL_SERVICE_NAME for the api (from vars.yaml). silo/live/space get the SAME +# ConfigMap but an inline env OTEL_SERVICE_NAME= that overrides it (inline +# env wins over envFrom), so each workload reports its own service.name. # # OPTIONAL / OPT-IN: the env vars are added ONLY when this component is enabled # in the overlay. To leave OTEL off entirely, do not enable this component — no @@ -43,3 +54,11 @@ patches: target: kind: Deployment name: plane-silo-wl + - path: patches/append-otel-live.json + target: + kind: Deployment + name: plane-live-wl + - path: patches/append-otel-space.json + target: + kind: Deployment + name: plane-space-wl diff --git a/kustomize/components/otel-observability/patches/append-otel-live.json b/kustomize/components/otel-observability/patches/append-otel-live.json new file mode 100644 index 0000000..157ffb0 --- /dev/null +++ b/kustomize/components/otel-observability/patches/append-otel-live.json @@ -0,0 +1,21 @@ +[ + { + "op": "add", + "path": "/spec/template/spec/containers/0/envFrom/-", + "value": { + "configMapRef": { + "name": "plane-otel-vars" + } + } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env", + "value": [ + { + "name": "OTEL_SERVICE_NAME", + "value": "live" + } + ] + } +] diff --git a/kustomize/components/otel-observability/patches/append-otel-space.json b/kustomize/components/otel-observability/patches/append-otel-space.json new file mode 100644 index 0000000..8d39142 --- /dev/null +++ b/kustomize/components/otel-observability/patches/append-otel-space.json @@ -0,0 +1,21 @@ +[ + { + "op": "add", + "path": "/spec/template/spec/containers/0/envFrom/-", + "value": { + "configMapRef": { + "name": "plane-otel-vars" + } + } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env", + "value": [ + { + "name": "OTEL_SERVICE_NAME", + "value": "space" + } + ] + } +] From f097367d44fa22a74fd7dee1d774061fd452a942 Mon Sep 17 00:00:00 2001 From: Pratapa Lakshmi Date: Thu, 18 Jun 2026 20:04:12 +0530 Subject: [PATCH 08/15] feat(otel-observability): wire browser tracing via FRONTEND_OTEL_* on the API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Browser/client tracing for web/admin/space is driven by the API: it serves is_otel_enabled/otlp_endpoint/otlp_headers to browsers from FRONTEND_OTEL_ENABLED / FRONTEND_OTLP_ENDPOINT / FRONTEND_OTLP_HEADERS (DB config or API env). So these only need to be set on the API, which already consumes plane-otel-vars. Adds the three keys to the component ConfigMap, vars.yaml(.example) values, and phoenix replacements. Note: FRONTEND_OTLP_ENDPOINT is sent to browsers (public) and must be an internet-reachable, CORS-enabled OTLP/HTTP collector route — not the backend gRPC path. Co-Authored-By: Claude Opus 4.8 --- .../otel-observability/configmap.yaml | 7 ++++ kustomize/overlays/phoenix/kustomization.yaml | 34 +++++++++++++++++++ kustomize/overlays/phoenix/vars.yaml.example | 11 ++++++ 3 files changed, 52 insertions(+) diff --git a/kustomize/components/otel-observability/configmap.yaml b/kustomize/components/otel-observability/configmap.yaml index c5e96ac..5d8f103 100644 --- a/kustomize/components/otel-observability/configmap.yaml +++ b/kustomize/components/otel-observability/configmap.yaml @@ -19,3 +19,10 @@ data: OTEL_SERVICE_NAME: "" OTEL_RESOURCE_ATTRIBUTES: "" OTEL_DEBUG_CONSOLE: "" + # Browser/client tracing for web/admin/space. The API serves these to browsers + # via its public instance endpoint (is_otel_enabled/otlp_endpoint/otlp_headers), + # so they only need to be set on the API — which already consumes this ConfigMap. + # Read ONLY by the API; a no-op env on silo/live/space. + FRONTEND_OTEL_ENABLED: "" + FRONTEND_OTLP_ENDPOINT: "" + FRONTEND_OTLP_HEADERS: "" diff --git a/kustomize/overlays/phoenix/kustomization.yaml b/kustomize/overlays/phoenix/kustomization.yaml index d8a5069..6e4abcf 100644 --- a/kustomize/overlays/phoenix/kustomization.yaml +++ b/kustomize/overlays/phoenix/kustomization.yaml @@ -1052,3 +1052,37 @@ replacements: name: plane-otel-vars fieldPaths: - data.OTEL_DEBUG_CONSOLE + + # Browser/client tracing (served by the API to web/admin/space browsers). + - source: + kind: ConfigMap + name: overlay-vars + fieldPath: data.FRONTEND_OTEL_ENABLED + targets: + - select: + kind: ConfigMap + name: plane-otel-vars + fieldPaths: + - data.FRONTEND_OTEL_ENABLED + + - source: + kind: ConfigMap + name: overlay-vars + fieldPath: data.FRONTEND_OTLP_ENDPOINT + targets: + - select: + kind: ConfigMap + name: plane-otel-vars + fieldPaths: + - data.FRONTEND_OTLP_ENDPOINT + + - source: + kind: ConfigMap + name: overlay-vars + fieldPath: data.FRONTEND_OTLP_HEADERS + targets: + - select: + kind: ConfigMap + name: plane-otel-vars + fieldPaths: + - data.FRONTEND_OTLP_HEADERS diff --git a/kustomize/overlays/phoenix/vars.yaml.example b/kustomize/overlays/phoenix/vars.yaml.example index 15930e0..23b66b8 100644 --- a/kustomize/overlays/phoenix/vars.yaml.example +++ b/kustomize/overlays/phoenix/vars.yaml.example @@ -122,3 +122,14 @@ data: # Debug: print spans to stdout to confirm the app is emitting/exporting them. # Set to "" (or remove this line + its replacement) once tracing is confirmed. OTEL_DEBUG_CONSOLE: "1" + # ---- Browser/client tracing (web/admin/space) ---- + # Set on the API ONLY; the API serves these to browsers via its public instance + # endpoint. Browser tracing turns on when FRONTEND_OTEL_ENABLED=1 AND + # FRONTEND_OTLP_ENDPOINT is non-empty. + FRONTEND_OTEL_ENABLED: "0" + # PUBLIC endpoint the BROWSER posts to (OTLP/HTTP — the client appends + # /v1/traces). MUST be internet-reachable AND CORS-enabled for the Plane web + # origin. NOT the backend gRPC path; needs an HTTP+CORS collector route. + FRONTEND_OTLP_ENDPOINT: "" + # Optional extra headers for the browser exporter, e.g. "x-api-key=...". + FRONTEND_OTLP_HEADERS: "" From 5ea0e7be77035d3d41e5eea041e4d085d4106e90 Mon Sep 17 00:00:00 2001 From: Pratapa Lakshmi Date: Thu, 18 Jun 2026 20:39:03 +0530 Subject: [PATCH 09/15] feat(otel-observability): wire OTel into all backend workers + pi (full coverage) Extends the component to every OTel-capable Plane workload, each with an inline OTEL_SERVICE_NAME matching its container (for promtail service_name correlation): Django (backend-commercial, OTel via plane.celery import): worker, beat-worker, automation-consumer, outbox-poller, external-api, webhook-consumer, worker-importers pi (plane-pi-commercial, logs-only OTel via pi.celery_app.configure_otel): pi-api, pi-beat, pi-worker external-api + worker-importers already carry an inline env (GUNICORN_WORKERS), so their patches append to /env/- instead of creating /env (no clobber). web/admin stay excluded (browser tracing via the API's FRONTEND_OTEL_* keys). flux/email/node-runner/mcp images have no OTel bootstrap -> not wired. Co-Authored-By: Claude Opus 4.8 --- .../otel-observability/kustomization.yaml | 75 +++++++++++++++---- .../append-otel-automation-consumer.json | 21 ++++++ .../patches/append-otel-beat-worker.json | 21 ++++++ .../patches/append-otel-external-api.json | 12 +++ .../patches/append-otel-outbox-poller.json | 21 ++++++ .../patches/append-otel-pi-api.json | 12 +++ .../patches/append-otel-pi-beat.json | 12 +++ .../patches/append-otel-pi-worker.json | 12 +++ .../patches/append-otel-webhook-consumer.json | 12 +++ .../patches/append-otel-worker-importers.json | 12 +++ .../patches/append-otel-worker.json | 21 ++++++ 11 files changed, 218 insertions(+), 13 deletions(-) create mode 100644 kustomize/components/otel-observability/patches/append-otel-automation-consumer.json create mode 100644 kustomize/components/otel-observability/patches/append-otel-beat-worker.json create mode 100644 kustomize/components/otel-observability/patches/append-otel-external-api.json create mode 100644 kustomize/components/otel-observability/patches/append-otel-outbox-poller.json create mode 100644 kustomize/components/otel-observability/patches/append-otel-pi-api.json create mode 100644 kustomize/components/otel-observability/patches/append-otel-pi-beat.json create mode 100644 kustomize/components/otel-observability/patches/append-otel-pi-worker.json create mode 100644 kustomize/components/otel-observability/patches/append-otel-webhook-consumer.json create mode 100644 kustomize/components/otel-observability/patches/append-otel-worker-importers.json create mode 100644 kustomize/components/otel-observability/patches/append-otel-worker.json diff --git a/kustomize/components/otel-observability/kustomization.yaml b/kustomize/components/otel-observability/kustomization.yaml index e34f370..6cafaba 100644 --- a/kustomize/components/otel-observability/kustomization.yaml +++ b/kustomize/components/otel-observability/kustomization.yaml @@ -4,29 +4,36 @@ kind: Component # ============================================================ # OTEL OBSERVABILITY (backend-api) # ============================================================ -# Injects OpenTelemetry environment variables into the backend workloads -# (plane-api-wl, plane-silo-wl, plane-live-wl, plane-space-wl) by appending the -# plane-otel-vars ConfigMap to each container's envFrom. +# Injects OpenTelemetry environment variables into the Plane workloads by +# appending the plane-otel-vars ConfigMap to each container's envFrom. # # SCOPE — only env-driven OTel consumers are wired here: -# - api (Django, OTEL_* env) -# - silo (Node startOtel, OTEL_* env) -# - live (Node startOtel, OTEL_* env) -# - space (React Router SSR initServerTelemetry, OTEL_ENABLED / -# OTEL_EXPORTER_OTLP_ENDPOINT / OTEL_SERVICE_NAME env) +# - api (Django wsgi/asgi, OTEL_* env) +# - worker (Celery worker — Celery task spans + DB/Redis/HTTP) +# - beat-worker (Celery beat) +# - automation-consumer (Django mgmt cmd; OTel bootstraps via plane.celery import) +# - outbox-poller (Django mgmt cmd; same) +# - silo (Node startOtel, OTEL_* env) +# - live (Node startOtel, OTEL_* env) +# - space (React Router SSR initServerTelemetry, OTEL_ENABLED / +# OTEL_EXPORTER_OTLP_ENDPOINT / OTEL_SERVICE_NAME env) +# All five Django processes bootstrap OTel because plane/__init__.py imports the +# Celery app, which calls configure_otel() at import. # web and admin are NOT wired: their only OTel is BROWSER/client tracing, which # is configured from the Plane instance config (is_otel_enabled, otlp_endpoint) -# served by the API — not from deployment env vars — so OTEL_* env on them is a -# no-op. Enable those via instance settings, not this component. +# served by the API via the FRONTEND_OTEL_* keys — not from per-workload env — so +# OTEL_* env on them is a no-op. # # VALUES LIVE IN vars.yaml: the keys in configmap.yaml are placeholders; the # actual values are supplied per-overlay from overlay-vars (vars.yaml) via the # OTEL replacements added in the overlay's kustomization.yaml. # # PER-SERVICE service.name: the shared plane-otel-vars ConfigMap carries -# OTEL_SERVICE_NAME for the api (from vars.yaml). silo/live/space get the SAME -# ConfigMap but an inline env OTEL_SERVICE_NAME= that overrides it (inline -# env wins over envFrom), so each workload reports its own service.name. +# OTEL_SERVICE_NAME for the api (from vars.yaml). Every other workload gets the +# SAME ConfigMap but an inline env OTEL_SERVICE_NAME= that overrides it +# (inline env wins over envFrom), so each reports its own service.name. The name +# matches the container name so it lines up with the promtail service_name label +# for trace->logs correlation. # # OPTIONAL / OPT-IN: the env vars are added ONLY when this component is enabled # in the overlay. To leave OTEL off entirely, do not enable this component — no @@ -62,3 +69,45 @@ patches: target: kind: Deployment name: plane-space-wl + - path: patches/append-otel-worker.json + target: + kind: Deployment + name: plane-worker-wl + - path: patches/append-otel-beat-worker.json + target: + kind: Deployment + name: plane-beat-worker-wl + - path: patches/append-otel-automation-consumer.json + target: + kind: Deployment + name: plane-automation-consumer-wl + - path: patches/append-otel-outbox-poller.json + target: + kind: Deployment + name: plane-outbox-poller-wl + - path: patches/append-otel-external-api.json + target: + kind: Deployment + name: plane-external-api-wl + - path: patches/append-otel-webhook-consumer.json + target: + kind: Deployment + name: plane-webhook-consumer-wl + - path: patches/append-otel-worker-importers.json + target: + kind: Deployment + name: plane-worker-importers-wl + # Plane Intelligence (pi) — logs-only OTel (configure_otel in pi.celery_app; + # reads the standard OTEL_* env, exports logs over OTLP/HTTP to /v1/logs). + - path: patches/append-otel-pi-api.json + target: + kind: Deployment + name: plane-pi-api-wl + - path: patches/append-otel-pi-beat.json + target: + kind: Deployment + name: plane-pi-beat-wl + - path: patches/append-otel-pi-worker.json + target: + kind: Deployment + name: plane-pi-worker-wl diff --git a/kustomize/components/otel-observability/patches/append-otel-automation-consumer.json b/kustomize/components/otel-observability/patches/append-otel-automation-consumer.json new file mode 100644 index 0000000..21058e9 --- /dev/null +++ b/kustomize/components/otel-observability/patches/append-otel-automation-consumer.json @@ -0,0 +1,21 @@ +[ + { + "op": "add", + "path": "/spec/template/spec/containers/0/envFrom/-", + "value": { + "configMapRef": { + "name": "plane-otel-vars" + } + } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env", + "value": [ + { + "name": "OTEL_SERVICE_NAME", + "value": "automation-consumer" + } + ] + } +] diff --git a/kustomize/components/otel-observability/patches/append-otel-beat-worker.json b/kustomize/components/otel-observability/patches/append-otel-beat-worker.json new file mode 100644 index 0000000..ea19ba8 --- /dev/null +++ b/kustomize/components/otel-observability/patches/append-otel-beat-worker.json @@ -0,0 +1,21 @@ +[ + { + "op": "add", + "path": "/spec/template/spec/containers/0/envFrom/-", + "value": { + "configMapRef": { + "name": "plane-otel-vars" + } + } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env", + "value": [ + { + "name": "OTEL_SERVICE_NAME", + "value": "beat-worker" + } + ] + } +] diff --git a/kustomize/components/otel-observability/patches/append-otel-external-api.json b/kustomize/components/otel-observability/patches/append-otel-external-api.json new file mode 100644 index 0000000..cbd35ff --- /dev/null +++ b/kustomize/components/otel-observability/patches/append-otel-external-api.json @@ -0,0 +1,12 @@ +[ + { + "op": "add", + "path": "/spec/template/spec/containers/0/envFrom/-", + "value": { "configMapRef": { "name": "plane-otel-vars" } } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env/-", + "value": { "name": "OTEL_SERVICE_NAME", "value": "external-api" } + } +] diff --git a/kustomize/components/otel-observability/patches/append-otel-outbox-poller.json b/kustomize/components/otel-observability/patches/append-otel-outbox-poller.json new file mode 100644 index 0000000..12b4e5c --- /dev/null +++ b/kustomize/components/otel-observability/patches/append-otel-outbox-poller.json @@ -0,0 +1,21 @@ +[ + { + "op": "add", + "path": "/spec/template/spec/containers/0/envFrom/-", + "value": { + "configMapRef": { + "name": "plane-otel-vars" + } + } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env", + "value": [ + { + "name": "OTEL_SERVICE_NAME", + "value": "outbox-poller" + } + ] + } +] diff --git a/kustomize/components/otel-observability/patches/append-otel-pi-api.json b/kustomize/components/otel-observability/patches/append-otel-pi-api.json new file mode 100644 index 0000000..4538fb3 --- /dev/null +++ b/kustomize/components/otel-observability/patches/append-otel-pi-api.json @@ -0,0 +1,12 @@ +[ + { + "op": "add", + "path": "/spec/template/spec/containers/0/envFrom/-", + "value": { "configMapRef": { "name": "plane-otel-vars" } } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env", + "value": [ { "name": "OTEL_SERVICE_NAME", "value": "pi-api" } ] + } +] diff --git a/kustomize/components/otel-observability/patches/append-otel-pi-beat.json b/kustomize/components/otel-observability/patches/append-otel-pi-beat.json new file mode 100644 index 0000000..20b3a02 --- /dev/null +++ b/kustomize/components/otel-observability/patches/append-otel-pi-beat.json @@ -0,0 +1,12 @@ +[ + { + "op": "add", + "path": "/spec/template/spec/containers/0/envFrom/-", + "value": { "configMapRef": { "name": "plane-otel-vars" } } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env", + "value": [ { "name": "OTEL_SERVICE_NAME", "value": "pi-beat" } ] + } +] diff --git a/kustomize/components/otel-observability/patches/append-otel-pi-worker.json b/kustomize/components/otel-observability/patches/append-otel-pi-worker.json new file mode 100644 index 0000000..726c92f --- /dev/null +++ b/kustomize/components/otel-observability/patches/append-otel-pi-worker.json @@ -0,0 +1,12 @@ +[ + { + "op": "add", + "path": "/spec/template/spec/containers/0/envFrom/-", + "value": { "configMapRef": { "name": "plane-otel-vars" } } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env", + "value": [ { "name": "OTEL_SERVICE_NAME", "value": "pi-worker" } ] + } +] diff --git a/kustomize/components/otel-observability/patches/append-otel-webhook-consumer.json b/kustomize/components/otel-observability/patches/append-otel-webhook-consumer.json new file mode 100644 index 0000000..1ea70e8 --- /dev/null +++ b/kustomize/components/otel-observability/patches/append-otel-webhook-consumer.json @@ -0,0 +1,12 @@ +[ + { + "op": "add", + "path": "/spec/template/spec/containers/0/envFrom/-", + "value": { "configMapRef": { "name": "plane-otel-vars" } } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env", + "value": [ { "name": "OTEL_SERVICE_NAME", "value": "webhook-consumer" } ] + } +] diff --git a/kustomize/components/otel-observability/patches/append-otel-worker-importers.json b/kustomize/components/otel-observability/patches/append-otel-worker-importers.json new file mode 100644 index 0000000..6a38c6c --- /dev/null +++ b/kustomize/components/otel-observability/patches/append-otel-worker-importers.json @@ -0,0 +1,12 @@ +[ + { + "op": "add", + "path": "/spec/template/spec/containers/0/envFrom/-", + "value": { "configMapRef": { "name": "plane-otel-vars" } } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env/-", + "value": { "name": "OTEL_SERVICE_NAME", "value": "worker-importers" } + } +] diff --git a/kustomize/components/otel-observability/patches/append-otel-worker.json b/kustomize/components/otel-observability/patches/append-otel-worker.json new file mode 100644 index 0000000..5f516a7 --- /dev/null +++ b/kustomize/components/otel-observability/patches/append-otel-worker.json @@ -0,0 +1,21 @@ +[ + { + "op": "add", + "path": "/spec/template/spec/containers/0/envFrom/-", + "value": { + "configMapRef": { + "name": "plane-otel-vars" + } + } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env", + "value": [ + { + "name": "OTEL_SERVICE_NAME", + "value": "worker" + } + ] + } +] From 6b4e9ba9ff1191669db34dd6475ac22e0b97da2a Mon Sep 17 00:00:00 2001 From: Pratapa Lakshmi Date: Thu, 18 Jun 2026 21:16:31 +0530 Subject: [PATCH 10/15] fix(otel-observability): set FRONTEND_OTLP_HEADERS so browser tracing works cross-origin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The OTLP web exporter uses navigator.sendBeacon when no headers are set, and sendBeacon always sends credentials (mode 'include') — which the CORS spec rejects against the collector's wildcard Access-Control-Allow-Origin: * (observed: "blocked by CORS ... must not be the wildcard '*' when credentials mode is include"). Setting a non-empty FRONTEND_OTLP_HEADERS forces the exporter onto XHR, which sends no credentials, so the wildcard is accepted. Value is arbitrary; the collector allows all headers. Co-Authored-By: Claude Opus 4.8 --- kustomize/overlays/phoenix/vars.yaml.example | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kustomize/overlays/phoenix/vars.yaml.example b/kustomize/overlays/phoenix/vars.yaml.example index 23b66b8..b6dbd41 100644 --- a/kustomize/overlays/phoenix/vars.yaml.example +++ b/kustomize/overlays/phoenix/vars.yaml.example @@ -131,5 +131,9 @@ data: # /v1/traces). MUST be internet-reachable AND CORS-enabled for the Plane web # origin. NOT the backend gRPC path; needs an HTTP+CORS collector route. FRONTEND_OTLP_ENDPOINT: "" - # Optional extra headers for the browser exporter, e.g. "x-api-key=...". - FRONTEND_OTLP_HEADERS: "" + # A non-empty header is REQUIRED for cross-origin browser tracing: it forces + # the OTLP web exporter to use XHR instead of navigator.sendBeacon. sendBeacon + # always sends credentials (mode 'include'), which CORS rejects against the + # collector's wildcard 'Access-Control-Allow-Origin: *'; XHR sends none, so the + # wildcard is accepted. Value is arbitrary; format "key=value,key2=value2". + FRONTEND_OTLP_HEADERS: "x-otlp-browser=1" From 58dfd77621e49c19377701ff1732803492a875e5 Mon Sep 17 00:00:00 2001 From: Pratapa Lakshmi Date: Sat, 20 Jun 2026 14:19:09 +0530 Subject: [PATCH 11/15] feat(otel-observability): make trace sampling configurable (default 1.0 in phoenix) The app's configure_otel defaults to parentbased_traceidratio @ 0.1 (10%), so 90% of requests are never exported to Tempo (their logs show trace_flags=0 and the trace_id is unfindable). Expose OTEL_TRACES_SAMPLER / OTEL_TRACES_SAMPLER_ARG via the component ConfigMap + vars replacements; phoenix sets ARG=1.0 so every request is traceable in this load/debug env. Lower for high-traffic prod. Co-Authored-By: Claude Opus 4.8 --- .../otel-observability/configmap.yaml | 6 +++++ kustomize/overlays/phoenix/kustomization.yaml | 22 +++++++++++++++++++ kustomize/overlays/phoenix/vars.yaml.example | 4 ++++ 3 files changed, 32 insertions(+) diff --git a/kustomize/components/otel-observability/configmap.yaml b/kustomize/components/otel-observability/configmap.yaml index 5d8f103..3e90a36 100644 --- a/kustomize/components/otel-observability/configmap.yaml +++ b/kustomize/components/otel-observability/configmap.yaml @@ -19,6 +19,12 @@ data: OTEL_SERVICE_NAME: "" OTEL_RESOURCE_ATTRIBUTES: "" OTEL_DEBUG_CONSOLE: "" + # Trace sampling. The app defaults to parentbased_traceidratio @ 0.1 (10%), so + # 90% of requests are NOT exported to Tempo (their logs show trace_flags=0 and + # the trace_id is not findable). Set ARG=1.0 to export every trace; lower it for + # high-traffic prod. Overrides the app default via the SDK's standard env vars. + OTEL_TRACES_SAMPLER: "" + OTEL_TRACES_SAMPLER_ARG: "" # Browser/client tracing for web/admin/space. The API serves these to browsers # via its public instance endpoint (is_otel_enabled/otlp_endpoint/otlp_headers), # so they only need to be set on the API — which already consumes this ConfigMap. diff --git a/kustomize/overlays/phoenix/kustomization.yaml b/kustomize/overlays/phoenix/kustomization.yaml index 6e4abcf..d6d689a 100644 --- a/kustomize/overlays/phoenix/kustomization.yaml +++ b/kustomize/overlays/phoenix/kustomization.yaml @@ -1053,6 +1053,28 @@ replacements: fieldPaths: - data.OTEL_DEBUG_CONSOLE + - source: + kind: ConfigMap + name: overlay-vars + fieldPath: data.OTEL_TRACES_SAMPLER + targets: + - select: + kind: ConfigMap + name: plane-otel-vars + fieldPaths: + - data.OTEL_TRACES_SAMPLER + + - source: + kind: ConfigMap + name: overlay-vars + fieldPath: data.OTEL_TRACES_SAMPLER_ARG + targets: + - select: + kind: ConfigMap + name: plane-otel-vars + fieldPaths: + - data.OTEL_TRACES_SAMPLER_ARG + # Browser/client tracing (served by the API to web/admin/space browsers). - source: kind: ConfigMap diff --git a/kustomize/overlays/phoenix/vars.yaml.example b/kustomize/overlays/phoenix/vars.yaml.example index b6dbd41..073f210 100644 --- a/kustomize/overlays/phoenix/vars.yaml.example +++ b/kustomize/overlays/phoenix/vars.yaml.example @@ -122,6 +122,10 @@ data: # Debug: print spans to stdout to confirm the app is emitting/exporting them. # Set to "" (or remove this line + its replacement) once tracing is confirmed. OTEL_DEBUG_CONSOLE: "1" + # Trace sampling — app default is 10% (0.1), so most requests aren't exported to + # Tempo (logs show trace_flags=0). 1.0 = every trace; lower for high-traffic prod. + OTEL_TRACES_SAMPLER: "parentbased_traceidratio" + OTEL_TRACES_SAMPLER_ARG: "1.0" # ---- Browser/client tracing (web/admin/space) ---- # Set on the API ONLY; the API serves these to browsers via its public instance # endpoint. Browser tracing turns on when FRONTEND_OTEL_ENABLED=1 AND From 7390561cb601a26662c30f5d519208ae6d30add6 Mon Sep 17 00:00:00 2001 From: Pratapa Lakshmi Date: Sat, 20 Jun 2026 15:52:51 +0530 Subject: [PATCH 12/15] fix(otel-observability): use deployment.environment.name + always_on sampler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two trace-visibility fixes found while validating tracing on aurtest: - Sampler: parentbased_traceidratio defers to an incoming traceparent's sampled flag and IGNORES OTEL_TRACES_SAMPLER_ARG. With FRONTEND_OTEL_ENABLED=1 the browser/proxy sends a traceparent; if it didn't sample (flag=0), the API span is dropped even at ARG=1.0 — so only root GET traffic showed up, no browser POST/user-action traces. Switch the phoenix overlay to always_on (captures every span, ignores upstream decisions). Documented both the ratio and parent-based gotchas in the component configmap; prod should use parentbased_traceidratio + a ratio with consistent browser sampling. - Resource attribute: use deployment.environment.name (current semconv key, matches the node/pi/browser services) instead of the legacy deployment.environment, which lands under a separate attribute and breaks cross-service environment filtering in Tempo. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../otel-observability/configmap.yaml | 18 ++++++++++++++---- kustomize/overlays/phoenix/vars.yaml.example | 14 ++++++++++---- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/kustomize/components/otel-observability/configmap.yaml b/kustomize/components/otel-observability/configmap.yaml index 3e90a36..d3e87fb 100644 --- a/kustomize/components/otel-observability/configmap.yaml +++ b/kustomize/components/otel-observability/configmap.yaml @@ -17,12 +17,22 @@ data: OTEL_EXPORTER_OTLP_ENDPOINT: "" OTEL_EXPORTER_OTLP_PROTOCOL: "" OTEL_SERVICE_NAME: "" + # Set deployment.environment.name (the CURRENT semconv key — NOT the legacy + # deployment.environment) so the API's environment tag matches the node/pi/ + # browser services and cross-service "environment=" filtering works in Tempo. OTEL_RESOURCE_ATTRIBUTES: "" OTEL_DEBUG_CONSOLE: "" - # Trace sampling. The app defaults to parentbased_traceidratio @ 0.1 (10%), so - # 90% of requests are NOT exported to Tempo (their logs show trace_flags=0 and - # the trace_id is not findable). Set ARG=1.0 to export every trace; lower it for - # high-traffic prod. Overrides the app default via the SDK's standard env vars. + # Trace sampling. Two independent gotchas: + # 1. Ratio: the app defaults to parentbased_traceidratio @ 0.1 (10%), so 90% + # of root requests are NOT exported (their logs show trace_flags=0). + # 2. Parent-based: ANY parentbased_* sampler DEFERS to an incoming traceparent's + # sampled flag and IGNORES the ARG ratio. With FRONTEND_OTEL_ENABLED=1 the + # browser/proxy sends a traceparent; if it didn't sample (flag=0), the API + # span is DROPPED regardless of ARG=1.0 — the classic "only GET traffic + # (rootspans) shows up, no POST/user-action traces" symptom. + # => For a test/load/debug env, use always_on (captures every span the service + # sees, ignores upstream decisions). For prod, use parentbased_traceidratio + # with a sane ARG AND make sure browser tracing samples consistently. OTEL_TRACES_SAMPLER: "" OTEL_TRACES_SAMPLER_ARG: "" # Browser/client tracing for web/admin/space. The API serves these to browsers diff --git a/kustomize/overlays/phoenix/vars.yaml.example b/kustomize/overlays/phoenix/vars.yaml.example index 073f210..9034fac 100644 --- a/kustomize/overlays/phoenix/vars.yaml.example +++ b/kustomize/overlays/phoenix/vars.yaml.example @@ -118,13 +118,19 @@ data: # k8s.namespace.name lets Grafana's trace->logs map the span to this namespace's # logs (must match the overlay's deploy namespace). Add k8s.pod.name via the # downward API if you want pod-exact correlation. - OTEL_RESOURCE_ATTRIBUTES: "deployment.environment=loadtest,developer=,k8s.namespace.name=plane-ns" + # Use deployment.environment.name (current semconv key, matches node/pi/browser + # services) — NOT the legacy deployment.environment, which breaks cross-service + # environment filtering in Tempo. + OTEL_RESOURCE_ATTRIBUTES: "deployment.environment.name=loadtest,developer=,k8s.namespace.name=plane-ns" # Debug: print spans to stdout to confirm the app is emitting/exporting them. # Set to "" (or remove this line + its replacement) once tracing is confirmed. OTEL_DEBUG_CONSOLE: "1" - # Trace sampling — app default is 10% (0.1), so most requests aren't exported to - # Tempo (logs show trace_flags=0). 1.0 = every trace; lower for high-traffic prod. - OTEL_TRACES_SAMPLER: "parentbased_traceidratio" + # Trace sampling — always_on for a test/load/debug env: captures every span and + # does NOT defer to an upstream traceparent's sampled flag (parentbased_* does, + # which silently drops browser-initiated POST traces — the "only GET shows up" + # symptom). For prod use parentbased_traceidratio + an ARG (e.g. 0.1). ARG is + # ignored by always_on; kept for an easy prod switch-back. + OTEL_TRACES_SAMPLER: "always_on" OTEL_TRACES_SAMPLER_ARG: "1.0" # ---- Browser/client tracing (web/admin/space) ---- # Set on the API ONLY; the API serves these to browsers via its public instance From cec1d8e41621b3124e0d4fa86b01d9d594749751 Mon Sep 17 00:00:00 2001 From: Pratapa Lakshmi Date: Mon, 22 Jun 2026 11:08:55 +0530 Subject: [PATCH 13/15] feat(otel-observability): add k8s.cluster.name to phoenix OTEL_RESOURCE_ATTRIBUTES MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spans from plane-ns carried namespace but no k8s.cluster.name, so traces weren't filterable by cluster in Tempo (e.g. {resource.k8s.cluster.name="plane-eks-dev"} returned nothing). Add k8s.cluster.name to the example so it's stamped on every span, consistent with the metrics/logs cluster label. (Live vars.yaml — gitignored — set to plane-eks-dev for this overlay.) Co-Authored-By: Claude Opus 4.8 (1M context) --- kustomize/overlays/phoenix/vars.yaml.example | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kustomize/overlays/phoenix/vars.yaml.example b/kustomize/overlays/phoenix/vars.yaml.example index 9034fac..7463a86 100644 --- a/kustomize/overlays/phoenix/vars.yaml.example +++ b/kustomize/overlays/phoenix/vars.yaml.example @@ -121,7 +121,11 @@ data: # Use deployment.environment.name (current semconv key, matches node/pi/browser # services) — NOT the legacy deployment.environment, which breaks cross-service # environment filtering in Tempo. - OTEL_RESOURCE_ATTRIBUTES: "deployment.environment.name=loadtest,developer=,k8s.namespace.name=plane-ns" + # k8s.cluster.name tags every span with the source cluster so traces are + # filterable per cluster in Tempo (e.g. {resource.k8s.cluster.name="plane-eks-dev"}), + # consistent with the metrics/logs cluster label. Set it to the cluster this + # overlay deploys to. + OTEL_RESOURCE_ATTRIBUTES: "k8s.cluster.name=,deployment.environment.name=loadtest,developer=,k8s.namespace.name=plane-ns" # Debug: print spans to stdout to confirm the app is emitting/exporting them. # Set to "" (or remove this line + its replacement) once tracing is confirmed. OTEL_DEBUG_CONSOLE: "1" From 717e7b2dc5f1b623c128d83aa4798c0cf15b368a Mon Sep 17 00:00:00 2001 From: Pratapa Lakshmi Date: Mon, 22 Jun 2026 11:14:37 +0530 Subject: [PATCH 14/15] chore(otel-observability): set phoenix deployment.environment.name=kustomize Was "loadtest" (stale); this overlay is the kustomize-deployed instance, so label its environment accordingly. Applies to all services (they share the api's OTEL_RESOURCE_ATTRIBUTES via the plane-otel-vars ConfigMap). Co-Authored-By: Claude Opus 4.8 (1M context) --- kustomize/overlays/phoenix/vars.yaml.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kustomize/overlays/phoenix/vars.yaml.example b/kustomize/overlays/phoenix/vars.yaml.example index af878b1..30d7ca2 100644 --- a/kustomize/overlays/phoenix/vars.yaml.example +++ b/kustomize/overlays/phoenix/vars.yaml.example @@ -125,7 +125,7 @@ data: # filterable per cluster in Tempo (e.g. {resource.k8s.cluster.name="plane-eks-dev"}), # consistent with the metrics/logs cluster label. Set it to the cluster this # overlay deploys to. - OTEL_RESOURCE_ATTRIBUTES: "k8s.cluster.name=,deployment.environment.name=loadtest,developer=,k8s.namespace.name=plane-ns" + OTEL_RESOURCE_ATTRIBUTES: "k8s.cluster.name=,deployment.environment.name=kustomize,developer=,k8s.namespace.name=plane-ns" # Debug: print spans to stdout to confirm the app is emitting/exporting them. # Set to "" (or remove this line + its replacement) once tracing is confirmed. OTEL_DEBUG_CONSOLE: "1" From 25afa5f927eeabbd1236e465a127c398874d2244 Mon Sep 17 00:00:00 2001 From: Pratapa Lakshmi Date: Mon, 22 Jun 2026 11:40:49 +0530 Subject: [PATCH 15/15] chore(kustomize): scale api and worker to 5 replicas Add replica overrides in base kustomization for plane-api-wl and plane-worker-wl instead of editing base manifests directly. Co-Authored-By: Claude Opus 4.8 (1M context) --- kustomize/base/kustomization.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml index 8a70c60..1fab72a 100644 --- a/kustomize/base/kustomization.yaml +++ b/kustomize/base/kustomization.yaml @@ -55,6 +55,13 @@ resources: # Ingress - ingress.yaml +# Replica overrides +replicas: + - name: plane-api-wl + count: 5 + - name: plane-worker-wl + count: 5 + # Default image tags images: - name: makeplane/backend-commercial