From 2bcd873a235da968c8879c68ecd9d364d5ea0498 Mon Sep 17 00:00:00 2001 From: mikkeldamsgaard Date: Wed, 6 May 2026 22:34:43 +0200 Subject: [PATCH] feat(netbird): support external relay config and verify relay in e2e (#83) Add `server.config.relays.{addresses, credentialsTTL}` and a dedicated `server.secrets.relaySecret` reference so users can point peers at an external relay deployment when the public relay hostname differs from the management hostname. When `relays.addresses` is empty (default), the embedded relay continues to run and reuse `authSecret` exactly as before. The chart's e2e script now runs `netbird status --detail` inside a registered peer pod and asserts the relay shows `is Available`, so the exact symptom in #83 ("relay client not connected") becomes a CI failure rather than a silent runtime regression. Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 14 +++++ charts/netbird/README.md | 29 ++++++--- charts/netbird/templates/_helpers.tpl | 10 ++++ .../netbird/templates/server-deployment.yaml | 17 ++++++ .../templates/server-generated-secret.yaml | 7 ++- .../netbird/tests/server-configmap_test.yaml | 59 +++++++++++++++++++ .../netbird/tests/server-deployment_test.yaml | 40 +++++++++++++ .../tests/server-generated-secret_test.yaml | 47 +++++++++++++++ charts/netbird/values.yaml | 31 ++++++++++ ci/scripts/netbird/e2e.sh | 36 ++++++++++- 10 files changed, 280 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11db984..b8e841c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/). - **netbird**: Add `server.stunService.nodePort` value to allow specifying a fixed NodePort number when `server.stunService.type` is `NodePort`. +- **netbird**: External relay configuration via `server.config.relays` + (`addresses`, `credentialsTTL`) and a dedicated + `server.secrets.relaySecret` reference. When `relays.addresses` is + non-empty the embedded relay is disabled and peers are directed at the + listed external relay URLs. The relay credential secret is taken from + `relaySecret` (with `autoGenerate` and `secretName` semantics + matching `authSecret`). When `relays.addresses` is empty (default) the + embedded relay continues to run and re-uses `authSecret` exactly as + before. Helps users whose relay hostname differs from the management + hostname or who run a separate relay deployment. Refs #83. +- **netbird**: E2E test now verifies relay reachability — runs + `netbird status --detail` inside a registered peer pod and asserts the + relay shows `is Available`, catching regressions where peers can + register but the relay path is broken. Refs #83. ## [0.4.2] — 2026-04-21 diff --git a/charts/netbird/README.md b/charts/netbird/README.md index 67da228..fa304f5 100644 --- a/charts/netbird/README.md +++ b/charts/netbird/README.md @@ -716,17 +716,30 @@ ADFS) can be tested manually: | `server.config.auth.signKeyRefreshEnabled` | bool | `true` | Auto-refresh IdP signing keys | | `server.config.auth.dashboardRedirectURIs` | list | `[]` | Dashboard OAuth2 redirect URIs | | `server.config.auth.cliRedirectURIs` | list | `["http://localhost:53000/"]` | CLI redirect URIs | +| `server.config.relays.addresses` | list | `[]` | External relay URLs (e.g. `rels://relay.example.com:443`); empty = run embedded relay | +| `server.config.relays.credentialsTTL` | string | `"12h"` | TTL for HMAC relay credentials handed to peers | + +By default the combined NetBird server runs an embedded relay on +`server.config.listenAddress` and advertises +`rels://:` to peers, sharing +`server.secrets.authSecret` as the relay credential secret. Set +`server.config.relays.addresses` to point peers at an external relay +instead — this disables the embedded relay and uses +`server.secrets.relaySecret` for the credential secret. #### Server Secrets -| Key | Type | Default | Description | -| ------------------------------------------------ | ------ | ----------------- | -------------------------------------------- | -| `server.secrets.authSecret.secretName` | string | `""` | Existing Secret name (empty = auto-generate) | -| `server.secrets.authSecret.secretKey` | string | `"authSecret"` | Key in the Secret | -| `server.secrets.authSecret.autoGenerate` | bool | `true` | Auto-generate on first install | -| `server.secrets.storeEncryptionKey.secretName` | string | `""` | Existing Secret name (empty = auto-generate) | -| `server.secrets.storeEncryptionKey.secretKey` | string | `"encryptionKey"` | Key in the Secret | -| `server.secrets.storeEncryptionKey.autoGenerate` | bool | `true` | Auto-generate on first install | +| Key | Type | Default | Description | +| ------------------------------------------------ | ------ | ----------------- | ---------------------------------------------------------- | +| `server.secrets.authSecret.secretName` | string | `""` | Existing Secret name (empty = auto-generate) | +| `server.secrets.authSecret.secretKey` | string | `"authSecret"` | Key in the Secret | +| `server.secrets.authSecret.autoGenerate` | bool | `true` | Auto-generate on first install | +| `server.secrets.storeEncryptionKey.secretName` | string | `""` | Existing Secret name (empty = auto-generate) | +| `server.secrets.storeEncryptionKey.secretKey` | string | `"encryptionKey"` | Key in the Secret | +| `server.secrets.storeEncryptionKey.autoGenerate` | bool | `true` | Auto-generate on first install | +| `server.secrets.relaySecret.secretName` | string | `""` | Existing Secret name (only consumed for external relay) | +| `server.secrets.relaySecret.secretKey` | string | `"relaySecret"` | Key in the Secret | +| `server.secrets.relaySecret.autoGenerate` | bool | `false` | Auto-generate on first install (only when relays are set) | #### Server Storage diff --git a/charts/netbird/templates/_helpers.tpl b/charts/netbird/templates/_helpers.tpl index 804405f..c8edce6 100644 --- a/charts/netbird/templates/_helpers.tpl +++ b/charts/netbird/templates/_helpers.tpl @@ -289,6 +289,16 @@ server: engine: {{ include "netbird.database.engine" . | quote }} dsn: {{ if eq (include "netbird.database.isExternal" .) "true" }}"{{ include "netbird.database.dsn" . }}"{{ else }}""{{ end }} encryptionKey: "${ENCRYPTION_KEY}" + {{- if .Values.server.config.relays.addresses }} + + relays: + addresses: + {{- range .Values.server.config.relays.addresses }} + - {{ include "netbird.escapeEnvsubst" . | quote }} + {{- end }} + credentialsTTL: {{ include "netbird.escapeEnvsubst" .Values.server.config.relays.credentialsTTL | quote }} + secret: "${RELAY_SECRET}" + {{- end }} {{- if .Values.oidc.enabled }} http: diff --git a/charts/netbird/templates/server-deployment.yaml b/charts/netbird/templates/server-deployment.yaml index ce90851..ef1745d 100644 --- a/charts/netbird/templates/server-deployment.yaml +++ b/charts/netbird/templates/server-deployment.yaml @@ -38,6 +38,11 @@ spec: {{- $generated := include "netbird.server.generatedSecretName" . }} {{- $authSecretName := include "netbird.server.resolveSecretName" (dict "ref" .Values.server.secrets.authSecret "generated" $generated) }} {{- $encKeySecretName := include "netbird.server.resolveSecretName" (dict "ref" .Values.server.secrets.storeEncryptionKey "generated" $generated) }} +{{- $relayConfigured := gt (len .Values.server.config.relays.addresses) 0 }} +{{- $relaySecretName := "" }} +{{- if $relayConfigured }} +{{- $relaySecretName = include "netbird.server.resolveSecretName" (dict "ref" .Values.server.secrets.relaySecret "generated" $generated) }} +{{- end }} {{- $isExternal := eq (include "netbird.database.isExternal" .) "true" }} {{- $patSidecar := and .Values.pat.enabled (not $isExternal) }} initContainers: @@ -117,6 +122,18 @@ spec: - name: ENCRYPTION_KEY value: "" {{- end }} +{{- if $relayConfigured }} +{{- if $relaySecretName }} + - name: RELAY_SECRET + valueFrom: + secretKeyRef: + name: {{ $relaySecretName }} + key: {{ .Values.server.secrets.relaySecret.secretKey }} +{{- else }} + - name: RELAY_SECRET + value: "" +{{- end }} +{{- end }} {{- if $isExternal }} - name: DB_PASSWORD valueFrom: diff --git a/charts/netbird/templates/server-generated-secret.yaml b/charts/netbird/templates/server-generated-secret.yaml index 112860e..3f14d77 100644 --- a/charts/netbird/templates/server-generated-secret.yaml +++ b/charts/netbird/templates/server-generated-secret.yaml @@ -7,7 +7,9 @@ using Helm's lookup function — subsequent upgrades reuse the existing Secret. {{- $genName := include "netbird.server.generatedSecretName" . -}} {{- $needAuth := and .Values.server.secrets.authSecret.autoGenerate (not .Values.server.secrets.authSecret.secretName) -}} {{- $needEnc := and .Values.server.secrets.storeEncryptionKey.autoGenerate (not .Values.server.secrets.storeEncryptionKey.secretName) -}} -{{- if or $needAuth $needEnc }} +{{- $relayConfigured := gt (len .Values.server.config.relays.addresses) 0 -}} +{{- $needRelay := and $relayConfigured (and .Values.server.secrets.relaySecret.autoGenerate (not .Values.server.secrets.relaySecret.secretName)) -}} +{{- if or $needAuth $needEnc $needRelay }} {{- $existingSecret := lookup "v1" "Secret" .Release.Namespace $genName }} apiVersion: v1 kind: Secret @@ -26,5 +28,8 @@ data: {{- if $needEnc }} {{ .Values.server.secrets.storeEncryptionKey.secretKey }}: {{ if and $existingSecret $existingSecret.data (index $existingSecret.data .Values.server.secrets.storeEncryptionKey.secretKey) }}{{ index $existingSecret.data .Values.server.secrets.storeEncryptionKey.secretKey }}{{ else }}{{ randBytes 32 | b64enc }}{{ end }} {{- end }} + {{- if $needRelay }} + {{ .Values.server.secrets.relaySecret.secretKey }}: {{ if and $existingSecret $existingSecret.data (index $existingSecret.data .Values.server.secrets.relaySecret.secretKey) }}{{ index $existingSecret.data .Values.server.secrets.relaySecret.secretKey }}{{ else }}{{ randBytes 32 | b64enc }}{{ end }} + {{- end }} {{- end }} diff --git a/charts/netbird/tests/server-configmap_test.yaml b/charts/netbird/tests/server-configmap_test.yaml index aaf1d1b..f574576 100644 --- a/charts/netbird/tests/server-configmap_test.yaml +++ b/charts/netbird/tests/server-configmap_test.yaml @@ -455,3 +455,62 @@ tests: asserts: - hasDocuments: count: 1 + + # ── Relay configuration (issue #83) ─────────────────────────────────── + + - it: should not render relays block when no relay addresses configured + asserts: + - notMatchRegex: + path: data["config.yaml.tpl"] + pattern: "(?m)^\\s+relays:" + + - it: should render relays block with addresses and secret placeholder + set: + server.config.relays.addresses: + - "rels://relay.example.com:443" + server.secrets.relaySecret.secretName: my-relay-secret + asserts: + - matchRegex: + path: data["config.yaml.tpl"] + pattern: "(?s)relays:\\s*\\n\\s+addresses:\\s*\\n\\s+- \"rels://relay\\.example\\.com:443\"" + - matchRegex: + path: data["config.yaml.tpl"] + pattern: 'credentialsTTL: "12h"' + - matchRegex: + path: data["config.yaml.tpl"] + pattern: 'secret: "\$\{RELAY_SECRET\}"' + + - it: should render multiple relay addresses + set: + server.config.relays.addresses: + - "rels://relay-1.example.com:443" + - "rels://relay-2.example.com:443" + server.secrets.relaySecret.secretName: my-relay-secret + asserts: + - matchRegex: + path: data["config.yaml.tpl"] + pattern: '"rels://relay-1\.example\.com:443"' + - matchRegex: + path: data["config.yaml.tpl"] + pattern: '"rels://relay-2\.example\.com:443"' + + - it: should render configurable credentialsTTL + set: + server.config.relays.addresses: + - "rels://relay.example.com:443" + server.config.relays.credentialsTTL: "1h" + server.secrets.relaySecret.secretName: my-relay-secret + asserts: + - matchRegex: + path: data["config.yaml.tpl"] + pattern: 'credentialsTTL: "1h"' + + - it: should escape dollar signs in relay addresses + set: + server.config.relays.addresses: + - "rels://relay$weird.example.com:443" + server.secrets.relaySecret.secretName: my-relay-secret + asserts: + - matchRegex: + path: data["config.yaml.tpl"] + pattern: '"rels://relay\$\{DOLLAR\}weird\.example\.com:443"' diff --git a/charts/netbird/tests/server-deployment_test.yaml b/charts/netbird/tests/server-deployment_test.yaml index 55cdb8e..726b3d4 100644 --- a/charts/netbird/tests/server-deployment_test.yaml +++ b/charts/netbird/tests/server-deployment_test.yaml @@ -206,6 +206,46 @@ tests: name: DB_PASSWORD any: true + # ── Relay env wiring (issue #83) ────────────────────────────────────── + + - it: should not inject RELAY_SECRET env var when relays are not configured + asserts: + - notContains: + path: spec.template.spec.initContainers[0].env + content: + name: RELAY_SECRET + any: true + + - it: should inject RELAY_SECRET env var from external secret when configured + set: + server.config.relays.addresses: + - "rels://relay.example.com:443" + server.secrets.relaySecret.secretName: my-relay-secret + asserts: + - contains: + path: spec.template.spec.initContainers[0].env + content: + name: RELAY_SECRET + valueFrom: + secretKeyRef: + name: my-relay-secret + key: relaySecret + + - it: should inject RELAY_SECRET env var from auto-generated secret when configured + set: + server.config.relays.addresses: + - "rels://relay.example.com:443" + server.secrets.relaySecret.autoGenerate: true + asserts: + - contains: + path: spec.template.spec.initContainers[0].env + content: + name: RELAY_SECRET + valueFrom: + secretKeyRef: + name: RELEASE-NAME-netbird-server-generated + key: relaySecret + # ── Init containers: db-wait + db-seed for external DB ─────────────── - it: should add db-wait and db-seed init containers for postgresql diff --git a/charts/netbird/tests/server-generated-secret_test.yaml b/charts/netbird/tests/server-generated-secret_test.yaml index dc183f3..47ea40f 100644 --- a/charts/netbird/tests/server-generated-secret_test.yaml +++ b/charts/netbird/tests/server-generated-secret_test.yaml @@ -70,3 +70,50 @@ tests: content: app.kubernetes.io/name: netbird app.kubernetes.io/component: server + + # ── Relay secret (issue #83) ────────────────────────────────────────── + + - it: should not include relaySecret key when relays are not configured + set: + server.secrets.relaySecret.autoGenerate: true + asserts: + - notExists: + path: data.relaySecret + + - it: should include relaySecret key when autoGenerate and relays addresses set + set: + server.config.relays.addresses: + - "rels://relay.example.com:443" + server.secrets.relaySecret.autoGenerate: true + asserts: + - exists: + path: data.relaySecret + + - it: should not include relaySecret key when external secretName provided + set: + server.config.relays.addresses: + - "rels://relay.example.com:443" + server.secrets.relaySecret.secretName: external-relay + server.secrets.relaySecret.autoGenerate: false + asserts: + - notExists: + path: data.relaySecret + + - it: should create Secret with only relaySecret when other autoGenerate flags off + set: + server.secrets.authSecret.secretName: external-auth + server.secrets.authSecret.autoGenerate: false + server.secrets.storeEncryptionKey.secretName: external-enc + server.secrets.storeEncryptionKey.autoGenerate: false + server.config.relays.addresses: + - "rels://relay.example.com:443" + server.secrets.relaySecret.autoGenerate: true + asserts: + - hasDocuments: + count: 1 + - notExists: + path: data.authSecret + - notExists: + path: data.encryptionKey + - exists: + path: data.relaySecret diff --git a/charts/netbird/values.yaml b/charts/netbird/values.yaml index d9099fc..7baa0ec 100644 --- a/charts/netbird/values.yaml +++ b/charts/netbird/values.yaml @@ -357,6 +357,22 @@ server: cliRedirectURIs: - "http://localhost:53000/" + # -- External relay override. + # By default (addresses empty) the combined server runs an embedded relay + # on listenAddress and advertises rels://: to + # peers, sharing server.secrets.authSecret as the relay credential secret. + # When addresses is non-empty the embedded relay is NOT started; peers + # receive these addresses for relay candidate gathering and the credential + # secret is taken from server.secrets.relaySecret. Use this when the + # public relay hostname differs from the management hostname, or when the + # relay is hosted on a separate deployment. + relays: + # -- List of public relay URLs (e.g. "rels://relay.example.com:443"). + # When non-empty the embedded relay is disabled. + addresses: [] + # -- Time-to-live for HMAC relay credentials handed to peers. + credentialsTTL: "12h" + # --------------------------------------------------------------------------- # Secret references — sensitive config.yaml values # @@ -391,6 +407,21 @@ server: # -- Auto-generate on first install. autoGenerate: true + # -- HMAC shared secret used to validate credentials against an external + # relay. Only consumed when server.config.relays.addresses is non-empty; + # when the embedded relay is in use it shares the authSecret instead. + # Provide an existing Secret matching the secret your external relay + # was deployed with, or set autoGenerate to have the chart create one + # (you'll then need to configure your external relay with the same + # value). + relaySecret: + # -- Name of an existing Kubernetes Secret. Leave empty to auto-generate. + secretName: "" + # -- Key within the Secret. + secretKey: "relaySecret" + # -- Auto-generate on first install. + autoGenerate: false + # -- Persistent volume for server data. persistentVolume: # -- Whether to create a PersistentVolumeClaim. diff --git a/ci/scripts/netbird/e2e.sh b/ci/scripts/netbird/e2e.sh index 02a9fc9..56e173d 100755 --- a/ci/scripts/netbird/e2e.sh +++ b/ci/scripts/netbird/e2e.sh @@ -543,4 +543,38 @@ kubectl -n "$NAMESPACE" wait --for=jsonpath='{.status.phase}'=Succeeded pod/peer log "peer-verify pod logs:" kubectl -n "$NAMESPACE" logs peer-verify || true -log "E2E test with $BACKEND backend PASSED (including PAT seeding, peer registration, and network map sync)!" +# ── Verify relay reachability (issue #83) ──────────────────────────── +# Run `netbird status --detail` inside one of the peer pods and parse +# the Relays block. The line for the relay URL must say "is Available"; +# anything else (e.g. "is Unavailable, reason: relay client not connected") +# means the embedded relay isn't actually reachable, which is exactly +# the symptom reported in issue #83. +# +# We poll for up to 60s because relay connection is established +# asynchronously after `netbird up` returns. +log "Verifying relay availability via 'netbird status --detail'..." +RELAY_OK=0 +RELAY_STATUS_OUTPUT="" +for i in $(seq 1 20); do + RELAY_STATUS_OUTPUT=$(kubectl -n "$NAMESPACE" exec netbird-peer-1 -- netbird status --detail 2>&1 || true) + # Extract the Relays section (between "Relays:" and the next blank line + # or the next top-level header) and look for an "is Available" entry + # whose URL starts with "rel" (covers both rel:// and rels://). + RELAYS_BLOCK=$(echo "$RELAY_STATUS_OUTPUT" | awk '/^Relays:/{flag=1;next} /^[A-Za-z].*:$/{flag=0} flag') + if echo "$RELAYS_BLOCK" | grep -E '^\s*\[rels?://[^]]+\] is Available' >/dev/null; then + RELAY_OK=1 + break + fi + sleep 3 +done + +if [ "$RELAY_OK" -ne 1 ]; then + log "Final 'netbird status --detail' output:" + echo "$RELAY_STATUS_OUTPUT" + log "netbird-peer-1 logs (last 80 lines):" + kubectl -n "$NAMESPACE" logs netbird-peer-1 2>/dev/null | tail -80 || true + fail "Relay never reached Available state — issue #83 regression" +fi +log "Relay is Available — relay reachability verified" + +log "E2E test with $BACKEND backend PASSED (including PAT seeding, peer registration, network map sync, and relay reachability)!"