From 8d2b56e01e977e82b15b6e859337ea62a77587d8 Mon Sep 17 00:00:00 2001 From: Rob Esker Date: Thu, 9 Jul 2026 13:52:51 -0500 Subject: [PATCH 1/2] feat(chart): add opt-in NetworkPolicy for the Topograph release Ship a bundled NetworkPolicy, off by default (networkPolicy.enabled), covering all three components (API server, node-observer, node-data-broker) selected by the release instance label. When enabled, ingress is denied except intra-release traffic (so the node-observer can reach the API server to trigger regeneration) and, when serviceMonitor.enabled, the Prometheus scrape namespace. Egress stays unconstrained until networkPolicy.extraEgress is set, so enabling the policy does not break egress on clusters without a default-deny. When extraEgress is set, an Egress policyType is added with DNS and intra-release allows plus the supplied rules. Because the kube-apiserver address and the provider's topology-source endpoint are cluster-specific, they are operator-supplied via extraEgress and are required under default-deny egress. networkPolicy.extraIngress appends custom ingress rules. Adds a values.schema.json entry, a helm-unittest suite, and documents the policy (including the NMC/Calico namespace-list alternative) in the Kubernetes engine docs. Signed-off-by: Rob Esker --- CHANGELOG.md | 1 + charts/topograph/templates/networkpolicy.yaml | 73 +++++++++++++++++ .../topograph/tests/networkpolicy_test.yaml | 80 +++++++++++++++++++ charts/topograph/values.schema.json | 15 ++++ charts/topograph/values.yaml | 41 ++++++++++ docs/engines/k8s.md | 64 +++++++++------ 6 files changed, 251 insertions(+), 23 deletions(-) create mode 100644 charts/topograph/templates/networkpolicy.yaml create mode 100644 charts/topograph/tests/networkpolicy_test.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index a3eef8b9..f34693eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - Helm chart metadata: `home`, `icon`, `maintainers`, `keywords`, and Artifact Hub annotations ([#377](https://github.com/NVIDIA/topograph/pull/377)). - Helm `env`, `initContainers`, and `lifecycle` overrides across the API server, node-observer, and node-data-broker containers. - Lambda provider Kubernetes node-data-broker support: Topograph instance and region annotations are derived from Lambda node `.spec.providerID` and `topology.kubernetes.io/region`, enabling automatic node discovery with the Kubernetes engine ([#375](https://github.com/NVIDIA/topograph/pull/375)). +- Opt-in Helm `NetworkPolicy` covering all three components — API server, node-observer, node-data-broker (`networkPolicy.enabled`, default `false`): denies ingress except intra-release traffic (so the node-observer can reach the API server) and, when `serviceMonitor.enabled`, the Prometheus scrape namespace. Egress stays unconstrained until `networkPolicy.extraEgress` is set, at which point an `Egress` policyType is added with DNS (scoped to the kube-dns pods) and intra-release allows plus the supplied rules (which must include the kube-apiserver and provider endpoint under default-deny egress); `networkPolicy.extraIngress` appends custom ingress rules. ### Changed diff --git a/charts/topograph/templates/networkpolicy.yaml b/charts/topograph/templates/networkpolicy.yaml new file mode 100644 index 00000000..536819e7 --- /dev/null +++ b/charts/topograph/templates/networkpolicy.yaml @@ -0,0 +1,73 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "topograph.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "topograph.labels" . | nindent 4 }} +spec: + # Select every Topograph component in this release (API server, node-observer, + # node-data-broker) via the shared instance label, so the policy protects the + # whole release rather than just the API server. + podSelector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + policyTypes: + - Ingress + {{- if .Values.networkPolicy.extraEgress }} + - Egress + {{- end }} + ingress: + # Intra-release traffic: the node-observer calls the API server to trigger + # regeneration. Allowing release pods to reach each other covers it without + # naming ports. + - from: + - podSelector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.serviceMonitor.enabled }} + # Prometheus scrapes the API server /metrics on the service port. + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Values.serviceMonitor.namespace }} + ports: + - protocol: TCP + port: {{ .Values.service.port }} + {{- end }} + {{- with .Values.networkPolicy.extraIngress }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.networkPolicy.extraEgress }} + egress: + # DNS resolution to CoreDNS, required before any component can reach a named + # endpoint. Scoped to the kube-dns pods in kube-system (the standard + # `k8s-app: kube-dns` label). Clusters whose CoreDNS carries a different + # label (e.g. `app.kubernetes.io/name: coredns`) can widen this via + # `networkPolicy.extraEgress`. + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # Intra-release traffic: the node-observer reaching the API server Service. + - to: + - podSelector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + # Operator-supplied egress. Under a default-deny-egress cluster this MUST + # include the Kubernetes API server (all three components call it) and the + # provider's topology source (e.g. the BCM/NetQ endpoint) -- those addresses + # are cluster-specific and cannot be templated portably. See the + # networkPolicy.extraEgress comments in values.yaml for examples. + {{- toYaml .Values.networkPolicy.extraEgress | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/topograph/tests/networkpolicy_test.yaml b/charts/topograph/tests/networkpolicy_test.yaml new file mode 100644 index 00000000..fda0f41f --- /dev/null +++ b/charts/topograph/tests/networkpolicy_test.yaml @@ -0,0 +1,80 @@ +suite: topograph networkpolicy +templates: + - templates/networkpolicy.yaml +release: + name: chart-ci + namespace: topograph +tests: + - it: renders nothing by default + asserts: + - hasDocuments: + count: 0 + + - it: renders a NetworkPolicy covering the whole release when enabled + set: + networkPolicy: + enabled: true + asserts: + - isKind: + of: NetworkPolicy + - equal: + path: spec.podSelector.matchLabels["app.kubernetes.io/instance"] + value: chart-ci + - contains: + path: spec.policyTypes + content: Ingress + - notContains: + path: spec.policyTypes + content: Egress + # Intra-release ingress so the node-observer can reach the API server. + - equal: + path: spec.ingress[0].from[0].podSelector.matchLabels["app.kubernetes.io/instance"] + value: chart-ci + + - it: adds a Prometheus ingress rule when serviceMonitor is enabled + set: + networkPolicy: + enabled: true + serviceMonitor: + enabled: true + namespace: monitoring + asserts: + - lengthEqual: + path: spec.ingress + count: 2 + - equal: + path: spec.ingress[1].from[0].namespaceSelector.matchLabels["kubernetes.io/metadata.name"] + value: monitoring + + - it: adds Egress with DNS and intra-release allows when extraEgress is set + set: + networkPolicy: + enabled: true + extraEgress: + - to: + - ipBlock: + cidr: 10.1.2.0/24 + ports: + - protocol: TCP + port: 8081 + asserts: + - contains: + path: spec.policyTypes + content: Egress + # egress[0] = DNS (scoped to kube-dns pods in kube-system), + # egress[1] = intra-release, egress[2] = operator-supplied + - equal: + path: spec.egress[0].to[0].namespaceSelector.matchLabels["kubernetes.io/metadata.name"] + value: kube-system + - equal: + path: spec.egress[0].to[0].podSelector.matchLabels["k8s-app"] + value: kube-dns + - equal: + path: spec.egress[0].ports[0].port + value: 53 + - equal: + path: spec.egress[1].to[0].podSelector.matchLabels["app.kubernetes.io/instance"] + value: chart-ci + - equal: + path: spec.egress[2].to[0].ipBlock.cidr + value: 10.1.2.0/24 diff --git a/charts/topograph/values.schema.json b/charts/topograph/values.schema.json index 499eba82..e8580dca 100644 --- a/charts/topograph/values.schema.json +++ b/charts/topograph/values.schema.json @@ -184,6 +184,21 @@ } } }, + "networkPolicy": { + "type": "object", + "description": "Opt-in NetworkPolicy for the whole release. extraEgress, when non-empty, adds an Egress policyType plus a DNS allow rule. extraIngress/extraEgress items must be non-empty objects — an empty {} rule matches all traffic.", + "properties": { + "enabled": { "type": "boolean" }, + "extraIngress": { + "type": "array", + "items": { "type": "object", "minProperties": 1 } + }, + "extraEgress": { + "type": "array", + "items": { "type": "object", "minProperties": 1 } + } + } + }, "tests": { "type": "object", "properties": { diff --git a/charts/topograph/values.yaml b/charts/topograph/values.yaml index 36c41eb3..f4aa49e0 100644 --- a/charts/topograph/values.yaml +++ b/charts/topograph/values.yaml @@ -212,6 +212,47 @@ serviceMonitor: interval: 15s scheme: http +# Opt-in NetworkPolicy covering all Topograph components (API server, +# node-observer, node-data-broker), selected by the release instance label. +# When enabled, ingress is denied except: intra-release traffic (so the +# node-observer can reach the API server) and, when serviceMonitor.enabled, the +# Prometheus scrape namespace (serviceMonitor.namespace). +# +# Egress is only constrained when `extraEgress` is set -- this keeps enabling +# the policy from breaking egress on clusters that do not run a default-deny. +# When `extraEgress` is set, an Egress policyType is added with DNS and +# intra-release allows, then your rules. On a default-deny-egress cluster you +# MUST include the Kubernetes API server (all three components call it) and the +# provider's topology source (e.g. the BCM/NetQ endpoint); those addresses are +# cluster-specific and cannot be templated portably. `extraIngress` appends +# custom ingress rules. Note: an empty rule object (`{}`) in extraIngress / +# extraEgress matches ALL traffic in that direction -- always give explicit +# `to`/`from`/`ports`. +# +# NetworkPolicy egress cannot target hostnames, so provider endpoints must be +# given as ipBlock CIDRs. See docs/engines/k8s.md "Per-provider egress" for a +# per-provider table (netq, CSP/IMDS caveat, dra/infiniband need none). +networkPolicy: + enabled: false + extraIngress: [] + extraEgress: [] + # # Kubernetes API server (all providers under default-deny; adjust to your + # # cluster -- `kubectl get endpoints kubernetes -n default`): + # - to: + # - ipBlock: + # cidr: 10.96.0.1/32 + # ports: + # - protocol: TCP + # port: 6443 + # # Provider topology source, e.g. netq apiUrl host or the BCM endpoint + # # (resolve the hostname to an ipBlock): + # - to: + # - ipBlock: + # cidr: 10.0.0.0/24 + # ports: + # - protocol: TCP + # port: 8081 + tests: # `helm test` hook pods (templates/tests/) probe the Topograph API's # /healthz and /metrics endpoints via the in-cluster Service. diff --git a/docs/engines/k8s.md b/docs/engines/k8s.md index d895cf69..ef0ea525 100644 --- a/docs/engines/k8s.md +++ b/docs/engines/k8s.md @@ -266,37 +266,55 @@ When you additionally **enforce** the standard on the release namespace (`pod-se ### NetworkPolicy -The chart does not ship a `NetworkPolicy` template at this time. For clusters that enforce NetworkPolicy, a recommended starting point allows ingress to port `49021` only from the Topograph namespace and from the Prometheus scraper namespace (when `serviceMonitor.enabled: true`), and denies all other ingress. Replace `` with the namespace you deployed the chart into and `` with the namespace running your Prometheus instance: +The chart ships an opt-in `NetworkPolicy` covering **all three components** (API server, node-observer, node-data-broker), off by default: ```yaml -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: topograph - namespace: -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: topograph - policyTypes: [Ingress] - ingress: - - from: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: +networkPolicy: + enabled: true +``` + +The policy selects every pod in the release by the `app.kubernetes.io/instance` label. When enabled, ingress is denied except: + +- **intra-release traffic** — so the node-observer can reach the API server to trigger regeneration, and +- the **Prometheus scrape namespace** (`serviceMonitor.namespace`) when `serviceMonitor.enabled: true`. + +**Egress stays unconstrained until you set `extraEgress`.** Adding an egress rule to a pod turns on default-deny egress for it, so enabling the policy does *not* add an `Egress` policyType by itself — that would break egress on clusters without a default-deny. When you set `extraEgress` (i.e. you *do* run default-deny egress), the chart adds an `Egress` policyType with a **DNS** allow, an **intra-release** allow (observer→API), and then your rules. + +Because a portable chart cannot know your cluster's API-server address or your provider's topology-source endpoint, those egress targets are operator-supplied — and on a default-deny-egress cluster they are **required** (all three components call the Kubernetes API server; the API server calls the provider): + +```yaml +networkPolicy: + enabled: true + extraEgress: + # Kubernetes API server (adjust to your cluster's apiserver CIDR/port): + - to: + - ipBlock: + cidr: 10.96.0.1/32 ports: - protocol: TCP - port: 49021 - - from: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: + port: 443 + # Provider topology source (e.g. the BCM management endpoint): + - to: + - ipBlock: + cidr: 10.0.0.0/24 ports: - protocol: TCP - port: 49021 + port: 8081 ``` -Apply alongside the chart. A bundled template is under consideration. +`extraIngress` similarly appends custom ingress rules. **Give every `extraIngress`/`extraEgress` entry explicit `to`/`from`/`ports`** — an empty rule object (`{}`) matches *all* traffic in that direction, which defeats the default-deny. Note that a NetworkPolicy has no effect unless your CNI enforces it (Calico, Cilium, etc.). + +**Per-provider egress.** Kubernetes NetworkPolicy egress cannot target hostnames — only `ipBlock`, selectors, and ports — so a provider's endpoint must be supplied as an `ipBlock` (resolve the hostname to its IP/CIDR). Typical `extraEgress` targets by provider: + +| Provider(s) | `extraEgress` target | +|---|---| +| all (under default-deny) | the kube-apiserver endpoint — `kubectl get endpoints kubernetes -n default` — typically `/32` on `6443` | +| `netq` | the NetQ server IP (resolve the `apiUrl` host) on its port (`443` by default) | +| BCM (planned) | the BCM head-node CIDR on `8081` | +| `aws`, `gcp`, `oci`, `nebius`, `nscale`, `lambdai`, `cw` | the cloud API over `443`; if the provider reads instance metadata, `169.254.169.254/32` — **IMDS access is a credential-exposure vector**, so scope it tightly and prefer workload identity where the provider supports it | +| `dra`, `infiniband-k8s` | none beyond the kube-apiserver + DNS (they operate in-cluster) | + +> **On NMC / Calico clusters** the tenant baseline is a cluster-wide Calico `GlobalNetworkPolicy` default-deny that already allows kube-apiserver and DNS egress and governs intra-tenant traffic by namespace-list membership. There, prefer adding the Topograph namespace to the NMC network-policy `user.namespaceList` (and the BCM CIDR to `bcmHeadNodeCidrs`) rather than this chart policy; this `networkPolicy` is aimed at clusters that expect each workload to ship its own default-deny allow-rules. ### Node Observer RBAC From 427c0ae43a739324b0aed6b108f7ff412ce4889a Mon Sep 17 00:00:00 2001 From: Rob Esker Date: Wed, 15 Jul 2026 17:52:40 -0500 Subject: [PATCH 2/2] feat(chart): make NetworkPolicy DNS pod selector configurable Scope the DNS egress allow rule via networkPolicy.dnsPodSelector instead of hardcoding k8s-app: kube-dns, so clusters whose CoreDNS carries a different label (e.g. app.kubernetes.io/name: coredns) can allow DNS without falling back to a broad extraEgress rule. Default is empty in values.yaml so a Helm value override replaces the selector cleanly; the template falls back to { k8s-app: kube-dns } when unset. A non-empty default would deep-merge with an override and AND both labels, matching no DNS pod. Signed-off-by: Rob Esker --- CHANGELOG.md | 2 +- charts/topograph/templates/networkpolicy.yaml | 10 +++++----- charts/topograph/tests/networkpolicy_test.yaml | 17 +++++++++++++++++ charts/topograph/values.schema.json | 4 ++++ charts/topograph/values.yaml | 8 ++++++++ docs/engines/k8s.md | 9 +++++++++ 6 files changed, 44 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f34693eb..bf3d4de5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - Helm chart metadata: `home`, `icon`, `maintainers`, `keywords`, and Artifact Hub annotations ([#377](https://github.com/NVIDIA/topograph/pull/377)). - Helm `env`, `initContainers`, and `lifecycle` overrides across the API server, node-observer, and node-data-broker containers. - Lambda provider Kubernetes node-data-broker support: Topograph instance and region annotations are derived from Lambda node `.spec.providerID` and `topology.kubernetes.io/region`, enabling automatic node discovery with the Kubernetes engine ([#375](https://github.com/NVIDIA/topograph/pull/375)). -- Opt-in Helm `NetworkPolicy` covering all three components — API server, node-observer, node-data-broker (`networkPolicy.enabled`, default `false`): denies ingress except intra-release traffic (so the node-observer can reach the API server) and, when `serviceMonitor.enabled`, the Prometheus scrape namespace. Egress stays unconstrained until `networkPolicy.extraEgress` is set, at which point an `Egress` policyType is added with DNS (scoped to the kube-dns pods) and intra-release allows plus the supplied rules (which must include the kube-apiserver and provider endpoint under default-deny egress); `networkPolicy.extraIngress` appends custom ingress rules. +- Opt-in Helm `NetworkPolicy` covering all three components — API server, node-observer, node-data-broker (`networkPolicy.enabled`, default `false`): denies ingress except intra-release traffic (so the node-observer can reach the API server) and, when `serviceMonitor.enabled`, the Prometheus scrape namespace. Egress stays unconstrained until `networkPolicy.extraEgress` is set, at which point an `Egress` policyType is added with DNS (scoped to the kube-dns pods, overridable via `networkPolicy.dnsPodSelector` for CoreDNS deployments with a non-standard label) and intra-release allows plus the supplied rules (which must include the kube-apiserver and provider endpoint under default-deny egress); `networkPolicy.extraIngress` appends custom ingress rules. ### Changed diff --git a/charts/topograph/templates/networkpolicy.yaml b/charts/topograph/templates/networkpolicy.yaml index 536819e7..4ffe590b 100644 --- a/charts/topograph/templates/networkpolicy.yaml +++ b/charts/topograph/templates/networkpolicy.yaml @@ -42,17 +42,17 @@ spec: {{- if .Values.networkPolicy.extraEgress }} egress: # DNS resolution to CoreDNS, required before any component can reach a named - # endpoint. Scoped to the kube-dns pods in kube-system (the standard - # `k8s-app: kube-dns` label). Clusters whose CoreDNS carries a different - # label (e.g. `app.kubernetes.io/name: coredns`) can widen this via - # `networkPolicy.extraEgress`. + # endpoint. Scoped to the kube-dns pods in kube-system. The pod label + # defaults to the standard `k8s-app: kube-dns`; clusters whose CoreDNS + # carries a different label (e.g. `app.kubernetes.io/name: coredns`) can + # override it via `networkPolicy.dnsPodSelector`. - to: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: kube-system podSelector: matchLabels: - k8s-app: kube-dns + {{- toYaml (.Values.networkPolicy.dnsPodSelector | default (dict "k8s-app" "kube-dns")) | nindent 14 }} ports: - protocol: UDP port: 53 diff --git a/charts/topograph/tests/networkpolicy_test.yaml b/charts/topograph/tests/networkpolicy_test.yaml index fda0f41f..8158bdbe 100644 --- a/charts/topograph/tests/networkpolicy_test.yaml +++ b/charts/topograph/tests/networkpolicy_test.yaml @@ -78,3 +78,20 @@ tests: - equal: path: spec.egress[2].to[0].ipBlock.cidr value: 10.1.2.0/24 + + - it: scopes the DNS egress rule to a custom dnsPodSelector + set: + networkPolicy: + enabled: true + dnsPodSelector: + app.kubernetes.io/name: coredns + extraEgress: + - to: + - ipBlock: + cidr: 10.1.2.0/24 + asserts: + - equal: + path: spec.egress[0].to[0].podSelector.matchLabels["app.kubernetes.io/name"] + value: coredns + - notExists: + path: spec.egress[0].to[0].podSelector.matchLabels["k8s-app"] diff --git a/charts/topograph/values.schema.json b/charts/topograph/values.schema.json index e8580dca..90427cd5 100644 --- a/charts/topograph/values.schema.json +++ b/charts/topograph/values.schema.json @@ -189,6 +189,10 @@ "description": "Opt-in NetworkPolicy for the whole release. extraEgress, when non-empty, adds an Egress policyType plus a DNS allow rule. extraIngress/extraEgress items must be non-empty objects — an empty {} rule matches all traffic.", "properties": { "enabled": { "type": "boolean" }, + "dnsPodSelector": { + "type": "object", + "description": "Pod label selector matching the cluster's DNS server, used to scope the DNS egress allow rule. Defaults to { k8s-app: kube-dns }; override for CoreDNS deployments carrying a different label." + }, "extraIngress": { "type": "array", "items": { "type": "object", "minProperties": 1 } diff --git a/charts/topograph/values.yaml b/charts/topograph/values.yaml index f4aa49e0..6f24c65e 100644 --- a/charts/topograph/values.yaml +++ b/charts/topograph/values.yaml @@ -234,6 +234,14 @@ serviceMonitor: # per-provider table (netq, CSP/IMDS caveat, dra/infiniband need none). networkPolicy: enabled: false + # Pod label selector for the cluster's DNS server, used to scope the DNS + # egress allow rule (only rendered when extraEgress is set). Leave empty to use + # the standard kube-dns/CoreDNS label `{ k8s-app: kube-dns }`. Set this to + # replace that default for clusters whose CoreDNS carries a different label, + # e.g. `{ app.kubernetes.io/name: coredns }`. (Kept empty by default so a Helm + # value override replaces it cleanly rather than deep-merging both labels, + # which would AND them and match no DNS pod.) + dnsPodSelector: {} extraIngress: [] extraEgress: [] # # Kubernetes API server (all providers under default-deny; adjust to your diff --git a/docs/engines/k8s.md b/docs/engines/k8s.md index ef0ea525..f2d5b31f 100644 --- a/docs/engines/k8s.md +++ b/docs/engines/k8s.md @@ -304,6 +304,15 @@ networkPolicy: `extraIngress` similarly appends custom ingress rules. **Give every `extraIngress`/`extraEgress` entry explicit `to`/`from`/`ports`** — an empty rule object (`{}`) matches *all* traffic in that direction, which defeats the default-deny. Note that a NetworkPolicy has no effect unless your CNI enforces it (Calico, Cilium, etc.). +The DNS allow rule is scoped to the standard `k8s-app: kube-dns` pod label. If your cluster's CoreDNS carries a different label, override `networkPolicy.dnsPodSelector` so DNS resolution isn't blocked: + +```yaml +networkPolicy: + enabled: true + dnsPodSelector: + app.kubernetes.io/name: coredns +``` + **Per-provider egress.** Kubernetes NetworkPolicy egress cannot target hostnames — only `ipBlock`, selectors, and ports — so a provider's endpoint must be supplied as an `ipBlock` (resolve the hostname to its IP/CIDR). Typical `extraEgress` targets by provider: | Provider(s) | `extraEgress` target |