From 458d8346a2601d555869d69f17e4248131305e20 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Mon, 11 Aug 2025 17:51:39 +0300 Subject: [PATCH 01/10] refactor!: switch from opa / envoy to opala --- helm/Chart.lock | 6 + helm/Chart.yaml | 4 + helm/config/default.conf | 21 ++- helm/config/envoy.yaml | 105 -------------- helm/config/jwt.js | 37 ----- helm/config/nginx.conf | 16 ++- helm/config/policy.rego | 58 -------- helm/templates/envoy-configmap.yaml | 9 -- helm/templates/ingress-tls-secret.yaml | 10 -- helm/templates/ingress.yaml | 46 ------- helm/templates/nginx-configmap.yaml | 3 +- helm/templates/nginx-deployment.yaml | 181 ------------------------- helm/templates/nginx-service.yaml | 42 ------ helm/templates/opa-secret.yaml | 11 -- helm/templates/route.yaml | 36 ----- helm/values.yaml | 106 +++++++-------- 16 files changed, 90 insertions(+), 601 deletions(-) create mode 100644 helm/Chart.lock delete mode 100644 helm/config/envoy.yaml delete mode 100644 helm/config/jwt.js delete mode 100644 helm/config/policy.rego delete mode 100644 helm/templates/envoy-configmap.yaml delete mode 100644 helm/templates/ingress-tls-secret.yaml delete mode 100644 helm/templates/ingress.yaml delete mode 100644 helm/templates/nginx-deployment.yaml delete mode 100644 helm/templates/nginx-service.yaml delete mode 100644 helm/templates/opa-secret.yaml delete mode 100644 helm/templates/route.yaml diff --git a/helm/Chart.lock b/helm/Chart.lock new file mode 100644 index 0000000..561eb50 --- /dev/null +++ b/helm/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: nginx + repository: oci://acrarolibotnonprod.azurecr.io/helm + version: 2.0.0 +digest: sha256:a121ab1fbbc1465cf315c21d22afb866bd942f2cf20e59b175374ae51d74dfda +generated: "2025-08-03T16:49:25.655465433+03:00" diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 1bc831b..dbf5b31 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -4,3 +4,7 @@ description: Helm chart for heights service type: application version: 2.1.7 appVersion: 2.1.7 +dependencies: + - name: nginx + version: 2.0.0 + repository: oci://acrarolibotnonprod.azurecr.io/helm \ No newline at end of file diff --git a/helm/config/default.conf b/helm/config/default.conf index 32b61b8..182ce6a 100644 --- a/helm/config/default.conf +++ b/helm/config/default.conf @@ -1,5 +1,6 @@ {{- $serviceName := include "service.fullname" . -}} +{{- $opalaEnabled := .Values.nginx.authorization.enabled }} upstream heights { server {{ $serviceName }}:{{ .Values.env.port }}; @@ -63,7 +64,25 @@ server { proxy_hide_header Set-Cookie; # ensures the header will not be passed back to the client proxy_ignore_headers Set-Cookie; # ensures that the header will not automatically disable caching within nginx proxy_set_header Cookie ""; # ensures that a client cannot pass any prior cookies to the webapp and spoil your cache - proxy_set_header X-Client-Id $jwt_payload_sub; + + {{- if $opalaEnabled }} + set $domain {{ .Values.nginx.authorization.domain }}; + auth_request /_validate_jwt; + {{ end }} + proxy_pass http://heights; } + + {{- if $opalaEnabled }} + location = /_validate_jwt { + internal; + js_content auth.opaAuth; + } + + location = /opa { + internal; + proxy_set_header Content-Type application/json; + proxy_pass {{ .Values.nginx.authorization.url }}; + } + {{ end }} } diff --git a/helm/config/envoy.yaml b/helm/config/envoy.yaml deleted file mode 100644 index 25f3539..0000000 --- a/helm/config/envoy.yaml +++ /dev/null @@ -1,105 +0,0 @@ -static_resources: - listeners: - - address: - socket_address: - address: 0.0.0.0 - port_value: {{ .Values.envoy.port }} - filter_chains: - - filters: - - name: envoy.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - codec_type: auto - stat_prefix: ingress_http - access_log: - - name: envoy.access_loggers.File - typed_config: - "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog - path: "/dev/null" - route_config: - name: local_route - virtual_hosts: - - name: backend - domains: - - "*" - routes: - - match: - prefix: "/" - route: - cluster: service - http_filters: - - name: envoy.filters.http.jwt_authn - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.jwt_authn.v3.JwtAuthentication - providers: - map-colonies-jwt: - local_jwks: - filename: /keys/mapColonies.jwks - forward: false #remove the jwt (if header is used) - from_headers: - - name: {{ .Values.authentication.opa.customHeaderName }} - from_params: - - token - jwt_cache_config: - jwt_cache_size: 100 - payload_in_metadata: map_colonies_token_payload - rules: - - match: - prefix: "/" - requires: - provider_name: "map-colonies-jwt" - bypass_cors_preflight: true # all cors preflight requests without auth - - name: envoy.filters.http.set_metadata - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.set_metadata.v3.Config - metadata_namespace: map_colonies - value: - domain: {{ .Values.envoy.requestedDomain }} - - name: envoy.ext_authz - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz - transport_api_version: V3 - with_request_body: - max_request_bytes: {{ .Values.envoy.maxRequestBytes }} - allow_partial_message: true - failure_mode_allow: false - grpc_service: - google_grpc: - target_uri: localhost:{{ .Values.opa.port }} - stat_prefix: ext_authz - timeout: 0.5s - metadata_context_namespaces: - - map_colonies - - envoy.filters.http.jwt_authn - - name: envoy.filters.http.router - clusters: - - name: service - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: 127.0.0.1 - port_value: {{ .Values.env.port }} -admin: - access_log_path: "/dev/null" - address: - socket_address: - address: 0.0.0.0 - port_value: 8001 -layered_runtime: - layers: - - name: static_layer_0 - static_layer: - envoy: - resource_limits: - listeners: - example_listener_name: - connection_limit: 10000 - overload: - global_downstream_max_connections: 50000 diff --git a/helm/config/jwt.js b/helm/config/jwt.js deleted file mode 100644 index eec836c..0000000 --- a/helm/config/jwt.js +++ /dev/null @@ -1,37 +0,0 @@ -function jwt(data) { - if (data) { - var parts = data - .split('.') - .slice(0, 2) - .map(v => Buffer.from(v, 'base64url').toString()) - .map(JSON.parse); - return { headers: parts[0], payload: parts[1] }; - } else { - return; - } -} - -function jwt_payload_sub(r) { - try { - let token; - if (r.args['token']) { - token = jwt(r.args['token']); - if (!token) { - return ''; - } - return token.payload.sub; - } - if (r.headersIn['x-api-key']) { - token = jwt(r.headersIn['x-api-key']); - if (!token) { - return ''; - } - return token.payload.sub; - } - return ''; - } catch (error) { - return ''; - } -} - -export default { jwt_payload_sub }; diff --git a/helm/config/nginx.conf b/helm/config/nginx.conf index 8a6d857..7489370 100644 --- a/helm/config/nginx.conf +++ b/helm/config/nginx.conf @@ -1,23 +1,27 @@ #user nginx; +load_module modules/ngx_http_js_module.so; +load_module /otel/otel_ngx_module.so; worker_processes 4; error_log /var/log/nginx/error.log notice; -pid /var/run/nginx.pid; -load_module modules/ngx_http_js_module.so; +pid /tmp/nginx.pid; events { worker_connections 1024; } http { + opentelemetry_config /otel/otel-nginx.toml; include /etc/nginx/mime.types; default_type application/octet-stream; - js_import main from /etc/nginx/jwt.js; - js_set $jwt_payload_sub main.jwt_payload_sub; - include /etc/nginx/log_format.conf; + {{ if .Values.nginx.authorization.enabled }} + js_import auth from /etc/nginx/auth.js; + js_set $jwt_payload_sub auth.jwt_payload_sub; + {{ end }} - access_log /var/log/nginx/access.log main; + include /etc/nginx/log_format.conf; + access_log /var/log/nginx/access.log json; sendfile on; #tcp_nopush on; diff --git a/helm/config/policy.rego b/helm/config/policy.rego deleted file mode 100644 index f5df5a8..0000000 --- a/helm/config/policy.rego +++ /dev/null @@ -1,58 +0,0 @@ -package envoy.authz - -import input.attributes.request.http as http_request -import input.attributes.metadataContext.filterMetadata["envoy.filters.http.jwt_authn"].map_colonies_token_payload as payload -import input.attributes.metadataContext.filterMetadata.map_colonies as map_colonies - -# Decline until I allow it. -default allow = false - -### Resources Access ### -user_has_resource_access[payload] { - lower(payload.d[_]) = lower(map_colonies.domain) -} -### Resources Access ### - -### ORIGIN and REFERRER ### -# Checks if origin is in allowed origin -valid_origin[payload] { - payload.ao[_] = http_request.headers.origin -} - -# Checks if origin is allowed origin (if ao is not an arr) -valid_origin[payload] { - payload.ao == http_request.headers.origin -} - -# Checks if referrer is in allowed origin -valid_referrer[payload] { - payload.ao[_] = http_request.headers.referrer -} - -# Checks if referrer is allowed origin (if ao is not an arr) -valid_referrer[payload] { - payload.ao == http_request.headers.referrer -} - -# Checks if there is allowed origin -valid_origin[payload] { - not payload.ao -} -### ORIGIN and REFERRER ### - -allow { - valid_origin[payload] - user_has_resource_access[payload] -} - -allow { - valid_referrer[payload] - user_has_resource_access[payload] -} - -# Allow cors preflight WITHOUT AUTHENTICATION -allow { - http_request.method == "OPTIONS" - _ = http_request.headers["access-control-request-method"] - _ = http_request.headers["access-control-request-headers"] -} diff --git a/helm/templates/envoy-configmap.yaml b/helm/templates/envoy-configmap.yaml deleted file mode 100644 index 95b6dae..0000000 --- a/helm/templates/envoy-configmap.yaml +++ /dev/null @@ -1,9 +0,0 @@ -{{- if .Values.authentication.opa.enabled }} -kind: ConfigMap -apiVersion: v1 -metadata: - name: {{ .Release.Name }}-envoy -data: - envoy.yaml: |- -{{ tpl (.Files.Get "config/envoy.yaml") . | indent 4 }} -{{- end }} diff --git a/helm/templates/ingress-tls-secret.yaml b/helm/templates/ingress-tls-secret.yaml deleted file mode 100644 index b2fcf1a..0000000 --- a/helm/templates/ingress-tls-secret.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{- if and .Values.ingress.tls.enabled .Values.ingress.enabled (not .Values.ingress.tls.useExternal) -}} -apiVersion: v1 -kind: Secret -metadata: - name: ingress-tls-secret -type: kubernetes.io/tls -data: - tls.crt: {{ .Files.Get "config/cert.pem" | b64enc }} - tls.key: {{ .Files.Get "config/key.pem" | b64enc }} -{{- end }} diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml deleted file mode 100644 index 8b23c52..0000000 --- a/helm/templates/ingress.yaml +++ /dev/null @@ -1,46 +0,0 @@ -{{- if and (.Values.enabled) (.Values.ingress.enabled) -}} -{{- $releaseName := .Release.Name -}} -{{- $chartName := include "heights.name" . -}} -{{- $serviceName := include "service.fullname" . -}} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ $releaseName }}-{{ $chartName }}-nginx - annotations: - kubernetes.io/ingress.class: "nginx" - {{- if eq .Values.ingress.type "nginx-org" }} - nginx.org/mergeable-ingress-type: "minion" - nginx.org/rewrites: 'serviceName={{ $serviceName }}-nginx rewrite=/' - {{- end }} - {{- if eq .Values.ingress.type "nginx-kubernetes" }} - nginx.ingress.kubernetes.io/rewrite-target: /$1 - {{- end }} - nginx.org/location-snippets: | - if ($request_method = OPTIONS) { - return 204; - } - {{- if .Values.ingress.cors.enabled }} - add_header 'Access-Control-Allow-Origin' '{{- .Values.ingress.cors.origin -}}'; - add_header 'Access-Control-Max-Age' 3600; - add_header 'Access-Control-Expose-Headers' 'Content-Length'; - add_header 'Access-Control-Allow-Headers' '*'; - {{- end }} -spec: - {{- if .Values.ingress.tls.enabled }} - tls: - - hosts: - - {{ .Values.ingress.host | quote }} - secretName: ingress-tls-secret - {{- end }} - rules: - - http: - paths: - - path: {{ .Values.ingress.path }}{{- if eq .Values.ingress.type "nginx-kubernetes" }}(/|$)(.*){{- end }} - pathType: Prefix - backend: - service: - name: {{ $serviceName }}-nginx - port: - number: {{ .Values.nginx.port }} - host: {{ .Values.ingress.host | quote }} -{{- end -}} diff --git a/helm/templates/nginx-configmap.yaml b/helm/templates/nginx-configmap.yaml index f063dca..315bc2d 100644 --- a/helm/templates/nginx-configmap.yaml +++ b/helm/templates/nginx-configmap.yaml @@ -10,8 +10,7 @@ metadata: environment: {{ include "heights.environment" . }} release: {{ .Release.Name }} data: - jwt.js: {{ .Files.Get "config/jwt.js" | quote }} log_format.conf: {{ tpl (.Files.Get "config/log_format.conf") . | quote }} default.conf: {{ tpl (.Files.Get "config/default.conf") . | quote }} - nginx.conf: {{ .Files.Get "config/nginx.conf" | quote }} + nginx.conf: {{ tpl (.Files.Get "config/nginx.conf") . | quote }} {{- end }} diff --git a/helm/templates/nginx-deployment.yaml b/helm/templates/nginx-deployment.yaml deleted file mode 100644 index 5f4bb48..0000000 --- a/helm/templates/nginx-deployment.yaml +++ /dev/null @@ -1,181 +0,0 @@ -{{- if .Values.nginx.enabled -}} -{{- $releaseName := .Release.Name -}} -{{- $chartName := include "heights.name" . -}} -{{- $cloudProviderDockerRegistryUrl := include "heights.cloudProviderDockerRegistryUrl" . -}} -{{- $cloudProviderImagePullSecretName := include "heights.cloudProviderImagePullSecretName" . -}} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ $releaseName }}-{{ $chartName }}-nginx -{{- with .Values.annotations }} - annotations: -{{ toYaml . | indent 4 }} -{{- end }} - labels: - app: {{ $releaseName }}-{{ $chartName }} - component: nginx-deployment - environment: {{ include "heights.environment" . }} - release: {{ $releaseName }} -spec: - replicas: {{ .Values.nginx.replicaCount }} - selector: - matchLabels: - app: {{ $releaseName }}-{{ $chartName }} - release: {{ $releaseName }} - run: {{ $releaseName }}-nginx - template: - metadata: - labels: - app: {{ $releaseName }}-{{ $chartName }} - release: {{ $releaseName }} - run: {{ $releaseName }}-nginx - # Autodiscover annotations should be here - # https://github.com/elastic/beats/issues/7914 - annotations: - co.elastic.logs/enabled: 'true' - co.elastic.logs/module: 'nginx' - co.elastic.logs/fileset.stdout: 'access' - co.elastic.logs/fileset.stderr: 'error' - {{- if .Values.resetOnConfigChange }} - checksum/nginx-configmap: {{ include (print $.Template.BasePath "/nginx-configmap.yaml") . | sha256sum }} - checksum/envoy-configmap: {{ include (print $.Template.BasePath "/envoy-configmap.yaml") . | sha256sum }} - {{- end }} - spec: - {{- if $cloudProviderImagePullSecretName }} - imagePullSecrets: - - name: {{ $cloudProviderImagePullSecretName | quote }} - {{- end }} - volumes: - {{- if .Values.authentication.opa.enabled }} - - name: opa-policy - secret: - secretName: {{ $releaseName }}-{{ $chartName }}-opa - - name: proxy-config - configMap: - name: {{ $releaseName }}-envoy - - name: jwt-public-keys - secret: - secretName: {{ .Values.authentication.jwtPublicKeySecretName }} - {{- end }} - - name: nginx-jwt-js - configMap: - name: {{ $releaseName }}-nginx-configmap - items: - - key: jwt.js - path: jwt.js - - name: nginx-log-format-conf - configMap: - name: {{ $releaseName }}-nginx-configmap - items: - - key: log_format.conf - path: log_format.conf - - name: nginx-custom-conf - configMap: - name: {{ $releaseName }}-nginx-configmap - items: - - key: default.conf - path: default.conf - - name: nginx-root-conf - configMap: - name: {{ $releaseName }}-nginx-configmap - items: - - key: nginx.conf - path: nginx.conf - containers: - ## Nginx Deployment - - name: {{ $chartName }}-nginx - {{- with .Values.nginx.image }} - image: {{ $cloudProviderDockerRegistryUrl }}{{ .repository }}:{{ .tag }} - {{- end }} - imagePullPolicy: {{ .Values.imagePullPolicy }} - # command: - # - sleep - # - infinity - volumeMounts: - - name: nginx-jwt-js - mountPath: /etc/nginx/jwt.js - subPath: jwt.js - - name: nginx-log-format-conf - mountPath: /etc/nginx/log_format.conf - subPath: log_format.conf - - name: nginx-custom-conf - mountPath: /etc/nginx/conf.d - - name: nginx-root-conf - mountPath: /etc/nginx/nginx.conf - subPath: nginx.conf - ports: - - name: http - containerPort: {{ .Values.env.port }} - protocol: TCP - livenessProbe: - initialDelaySeconds: {{ .Values.initialDelaySeconds }} - httpGet: - path: /liveness - port: {{ .Values.nginx.port }} - readinessProbe: - initialDelaySeconds: {{ .Values.initialDelaySeconds }} - httpGet: - path: /liveness - port: {{ .Values.env.port }} - {{- if .Values.nginx.resources.enabled }} - resources: - {{- toYaml .Values.nginx.resources.value | nindent 12 }} - {{- end }} - {{- if .Values.authentication.opa.enabled }} - - name: envoy - image: "{{ .Values.envoy.image }}:{{ .Values.envoy.imageTag }}" - volumeMounts: - - name: proxy-config - mountPath: /config - readOnly: true - - name: jwt-public-keys - mountPath: /keys/mapColonies.jwks - subPath: mapColonies.jwks - readOnly: true - args: - - "envoy" - - "--config-path" - - "/config/envoy.yaml" - ports: - - name: http - containerPort: 8000 - protocol: TCP - {{- if .Values.envoy.resources.enabled }} - resources: - {{- toYaml .Values.envoy.resources.value | nindent 12 }} - {{- end }} - - name: opa - image: "{{ .Values.opa.image }}:{{ .Values.opa.imageTag }}" - volumeMounts: - - name: opa-policy - mountPath: /policy - readOnly: true - args: - - "run" - - "--server" - - "--addr=0.0.0.0:8181" - - "--diagnostic-addr=0.0.0.0:8282" - - "--set=plugins.envoy_ext_authz_grpc.add=:{{ .Values.opa.imageTag }}" - - "--set=plugins.envoy_ext_authz_grpc.path=envoy/authz/allow" - {{- if .Values.opa.enableLogging }} - - "--set=decision_logs.console=true" - {{- end }} - - "/policy/policy.rego" - {{- if .Values.opa.resources.enabled }} - resources: - {{- toYaml .Values.opa.resources.value | nindent 12 }} - {{- end }} - livenessProbe: - httpGet: - path: /health?plugins - port: 8282 - initialDelaySeconds: 5 - periodSeconds: 30 - readinessProbe: - httpGet: - path: /health?plugins - port: 8282 - initialDelaySeconds: 5 - periodSeconds: 30 - {{- end }} -{{- end }} diff --git a/helm/templates/nginx-service.yaml b/helm/templates/nginx-service.yaml deleted file mode 100644 index b1d1444..0000000 --- a/helm/templates/nginx-service.yaml +++ /dev/null @@ -1,42 +0,0 @@ -{{- if .Values.nginx.enabled -}} -{{- $releaseName := .Release.Name -}} -{{- $chartName := include "heights.name" . -}} -{{- $serviceName := include "service.fullname" . -}} -{{- $cloudProviderFlavor := include "heights.cloudProviderFlavor" . -}} -apiVersion: v1 -kind: Service -metadata: - name: {{ $serviceName }}-nginx - labels: - app: {{ $releaseName }}-{{ $chartName }} - component: {{ $chartName }}-nginx-service - environment: {{ include "heights.environment" . }} - release: {{ $releaseName }} -spec: - # In case cloudProavider=azure - {{- if eq $cloudProviderFlavor "azure" }} - type: ClusterIP - {{- end }} - # In case cloudProvider=minikube - {{- if eq $cloudProviderFlavor "minikube" }} - type: NodePort - {{- end }} - ports: - - name: internal - port: {{ .Values.nginx.internalPort }} - targetPort: {{ .Values.env.port }} - protocol: {{ .Values.env.protocol }} - {{- if .Values.authentication.opa.enabled }} - - name: external - port: {{ .Values.nginx.port }} - targetPort: {{ .Values.envoy.port }} - protocol: {{ .Values.env.protocol }} - {{- end }} - {{- if eq $cloudProviderFlavor "minikube" }} - nodePort: {{ .Values.nginx.nodePort }} - {{- end }} - selector: - app: {{ $releaseName }}-{{ $chartName }} - release: {{ $releaseName }} - run: {{ $releaseName }}-nginx -{{- end }} diff --git a/helm/templates/opa-secret.yaml b/helm/templates/opa-secret.yaml deleted file mode 100644 index f42f64a..0000000 --- a/helm/templates/opa-secret.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{- if .Values.authentication.opa.enabled }} -{{- $releaseName := .Release.Name -}} -{{- $chartName := include "heights.name" . -}} -kind: Secret -apiVersion: v1 -metadata: - name: {{ $releaseName }}-{{ $chartName }}-opa -data: - policy.rego: |- -{{ tpl (.Files.Get "config/policy.rego") . | b64enc | indent 4 }} -{{- end }} diff --git a/helm/templates/route.yaml b/helm/templates/route.yaml deleted file mode 100644 index 2d0186f..0000000 --- a/helm/templates/route.yaml +++ /dev/null @@ -1,36 +0,0 @@ -{{- $cloudProviderFlavor := include "heights.cloudProviderFlavor" . -}} -{{- if and (and (.Values.enabled) (eq $cloudProviderFlavor "openshift")) (.Values.route.enabled) -}} -{{- $releaseName := .Release.Name -}} -{{- $chartName := include "heights.name" . -}} -{{- $serviceName := include "service.fullname" . -}} -{{- $targetPort := ternary "external" "internal" .Values.authentication.opa.enabled -}} -apiVersion: route.openshift.io/v1 -kind: Route -metadata: - name: {{ $releaseName }}-{{ $chartName }}-nginx - annotations: - haproxy.router.openshift.io/rewrite-target: / - {{- if .Values.route.timeout.enabled }} - haproxy.router.openshift.io/timeout: {{ .Values.route.timeout.duration }} - {{- end }} -spec: - {{- if .Values.route.host }} - host: {{ .Values.route.host | quote }} - {{- end }} - path: {{ .Values.route.path | default "/" }} - port: - targetPort: {{ $targetPort }} - to: - kind: Service - name: {{ $serviceName }}-nginx - {{- if .Values.route.tls.enabled }} - tls: - termination: edge - {{- if .Values.route.tls.useCert }} - certificate: {{ .Values.route.tls.certificate | quote }} - key: {{ .Values.route.tls.key | quote }} - caCertificate: {{ .Values.route.tls.caCertificate | quote }} - insecureEdgeTerminationPolicy: Redirect - {{- end }} - {{- end }} -{{- end -}} diff --git a/helm/values.yaml b/helm/values.yaml index 5000900..7676073 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -26,12 +26,6 @@ image: repository: dem-heights tag: v1.0.2 -authentication: - jwtPublicKeySecretName: map-colonies-jwks - opa: - enabled: false - customHeaderName: x-api-key - accessToken: attributeName: token injectionType: queryParam @@ -80,44 +74,39 @@ resources: nginx: enabled: true + fullnameOverride: "dem-serving-heights-nginx" replicaCount: 1 image: repository: nginx - tag: merge + tag: latest port: 8080 - internalPort: 80 + targetPort: 80 nodePort: 30001 - resources: - enabled: true - value: - limits: - cpu: 500m - memory: 900Mi - requests: - cpu: 125m - memory: 900Mi - -opa: - image: openpolicyagent/opa - imageTag: 0.29.4-envoy-3 - port: 9191 - enableLogging: false - resources: + maxAge: '3600' + authorization: enabled: true - value: - limits: - cpu: 400m - memory: 200Mi - requests: - cpu: 100m - memory: 200Mi - -envoy: - image: envoyproxy/envoy - imageTag: v1.20.7 - port: 6565 - maxRequestBytes: 8192 - requestedDomain: dem + domain: "dem" + url: http://opa-int-opa-service.infra-services/v1/data/http/authz/decision + # url: http://opa.service/v1/data/http/authz/decision + additionalPodAnnotations: + co.elastic.logs/enabled: 'true' + co.elastic.logs/module: 'nginx' + co.elastic.logs/fileset.stdout: 'access' + co.elastic.logs/fileset.stderr: 'error' + extraVolumes: + - name: nginx-config + configMap: + name: "{{ .Release.Name }}-nginx-configmap" + extraVolumeMounts: + - name: nginx-config + mountPath: "/etc/nginx/conf.d/default.conf" + subPath: default.conf + - name: nginx-config + mountPath: "/etc/nginx/nginx.conf" + subPath: nginx.conf + - name: nginx-config + mountPath: "/etc/nginx/log_format.conf" + subPath: log_format.conf resources: enabled: true value: @@ -128,25 +117,28 @@ envoy: cpu: 125m memory: 900Mi -route: - enabled: true - host: - path: /api/heights/v1 - timeout: - enabled: false # defaults to 30s by openshift - duration: 60s # supported units (us, ms, s, m, h, d) - tls: + route: enabled: true - useCert: true + routesMapping: + - host: + path: /api/heights/v1 + timeout: + enabled: false # defaults to 30s by openshift + duration: 60s # supported units (us, ms, s, m, h, d) + rewriteTarget: "" + tls: + enabled: false + useCert: false + certificate: "" + key: "" + caCertificate: "" -ingress: - enabled: false - type: nginx-org # supported values: 'nginx-org' | 'nginx-kubernetes' - tls: + ingress: enabled: false - useExternal: '' - host: - path: /dem-heights - cors: - enabled: true - origin: '*' + type: nginx-org # supported values: 'nginx-org' | 'nginx-kubernetes' + ingressMapping: + - host: + path: /dem-heights + tls: + enabled: false + secretName: ingress-tls-secret From 64fdb9a2bc6eef19db8d116e2255daf9c637cfa8 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Tue, 12 Aug 2025 20:15:07 +0300 Subject: [PATCH 02/10] deps: use latest nginx chart --- helm/Chart.lock | 2 +- helm/config/default.conf | 13 +++++++++++++ helm/config/log_format.conf | 6 +++--- helm/config/nginx.conf | 13 ++++++++++--- helm/templates/_helpers.tpl | 13 +++++++++++++ helm/values.yaml | 5 +++++ 6 files changed, 45 insertions(+), 7 deletions(-) diff --git a/helm/Chart.lock b/helm/Chart.lock index 561eb50..420320d 100644 --- a/helm/Chart.lock +++ b/helm/Chart.lock @@ -3,4 +3,4 @@ dependencies: repository: oci://acrarolibotnonprod.azurecr.io/helm version: 2.0.0 digest: sha256:a121ab1fbbc1465cf315c21d22afb866bd942f2cf20e59b175374ae51d74dfda -generated: "2025-08-03T16:49:25.655465433+03:00" +generated: "2025-08-12T19:15:06.506534337+03:00" diff --git a/helm/config/default.conf b/helm/config/default.conf index 182ce6a..b12fff5 100644 --- a/helm/config/default.conf +++ b/helm/config/default.conf @@ -2,6 +2,11 @@ {{- $serviceName := include "service.fullname" . -}} {{- $opalaEnabled := .Values.nginx.authorization.enabled }} +split_clients "$otel_trace_id" $ratio_sampler { + {{ .Values.nginx.opentelemetry.ratio }}% on; + * off; +} + upstream heights { server {{ $serviceName }}:{{ .Values.env.port }}; } @@ -39,6 +44,8 @@ server { } location / { + {{ include "nginx.otelTrace" . | nindent 8 }} + if ($request_method = 'OPTIONS') { add_header 'Access-Control-Allow-Origin' '*'; add_header 'Access-Control-Allow-Methods' 'POST, OPTIONS'; @@ -64,10 +71,16 @@ server { proxy_hide_header Set-Cookie; # ensures the header will not be passed back to the client proxy_ignore_headers Set-Cookie; # ensures that the header will not automatically disable caching within nginx proxy_set_header Cookie ""; # ensures that a client cannot pass any prior cookies to the webapp and spoil your cache + + set $original_method $request_method; + set $original_args $args; {{- if $opalaEnabled }} set $domain {{ .Values.nginx.authorization.domain }}; auth_request /_validate_jwt; + + otel_span_attr opa.result $opa_result; + otel_span_attr opa.reason $opa_reason; {{ end }} proxy_pass http://heights; diff --git a/helm/config/log_format.conf b/helm/config/log_format.conf index fca7265..bf44a2f 100644 --- a/helm/config/log_format.conf +++ b/helm/config/log_format.conf @@ -2,7 +2,7 @@ map $msec $nanosec { ~(.*)\.(.*) $1$2000000; } -log_format main escape=json +log_format json escape=json '{' '"Timestamp":"$nanosec",' '"Attributes":{' @@ -35,8 +35,8 @@ log_format main escape=json '"service.name":"{{ .Values.image.repository }}",' '"service.version":"{{ .Values.image.tag }}"' '},' - #'"TraceId":"$opentelemetry_trace_id",' ## this is a byte sequence (hex-encoded in JSON) - #'"SpanId":"$opentelemetry_span_id",' + '"TraceId":"$otel_trace_id",' ## this is a byte sequence (hex-encoded in JSON) + '"SpanId":"$otel_span_id",' '"SeverityText":"INFO",' '"SeverityNumber":"9",' '"InstrumentationScope":"access.log",' diff --git a/helm/config/nginx.conf b/helm/config/nginx.conf index 7489370..93a1125 100644 --- a/helm/config/nginx.conf +++ b/helm/config/nginx.conf @@ -1,6 +1,6 @@ #user nginx; load_module modules/ngx_http_js_module.so; -load_module /otel/otel_ngx_module.so; +load_module modules/ngx_otel_module.so; worker_processes 4; error_log /var/log/nginx/error.log notice; @@ -11,13 +11,20 @@ events { } http { - opentelemetry_config /otel/otel-nginx.toml; + otel_service_name {{ .Values.nginx.opentelemetry.serviceName }}; + otel_exporter { + endpoint {{ .Values.nginx.opentelemetry.exporterHost }}:{{ .Values.nginx.opentelemetry.exporterPort }}; + } + include /etc/nginx/mime.types; default_type application/octet-stream; {{ if .Values.nginx.authorization.enabled }} js_import auth from /etc/nginx/auth.js; - js_set $jwt_payload_sub auth.jwt_payload_sub; + js_set $jwt_payload_sub auth.jwtPayloadSub; + + js_var $opa_result; + js_var $opa_reason; {{ end }} include /etc/nginx/log_format.conf; diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index b1af178..166668b 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -150,3 +150,16 @@ Returns metrics url from global if exists or from chart's values {{- .Values.env.metrics.url -}} {{- end -}} {{- end -}} + +{{/* +Generate OpenTelemetry trace configuration +*/}} +{{- define "nginx.otelTrace" -}} +{{- if eq .Values.nginx.opentelemetry.samplerMethod "AlwaysOn" -}} +otel_trace on; +{{- else if eq .Values.nginx.opentelemetry.samplerMethod "TraceIdRatioBased" -}} +otel_trace $ratio_sampler; +{{- else -}} +otel_trace off; +{{- end -}} +{{- end -}} diff --git a/helm/values.yaml b/helm/values.yaml index 7676073..fb3a50a 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -88,6 +88,11 @@ nginx: domain: "dem" url: http://opa-int-opa-service.infra-services/v1/data/http/authz/decision # url: http://opa.service/v1/data/http/authz/decision + opentelemetry: + serviceName: nginx + samplerMethod: "AlwaysOff" + exporterPort: 4317 + ratio: 10 additionalPodAnnotations: co.elastic.logs/enabled: 'true' co.elastic.logs/module: 'nginx' From aa93505c2b6d3eee208dd682d67cf692cb88bb4c Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Tue, 2 Sep 2025 18:41:48 +0300 Subject: [PATCH 03/10] fix: use unique name for nginx otelTrace helper --- helm/config/default.conf | 2 +- helm/templates/_helpers.tpl | 2 +- helm/values.yaml | 9 +++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/helm/config/default.conf b/helm/config/default.conf index b12fff5..1768ed2 100644 --- a/helm/config/default.conf +++ b/helm/config/default.conf @@ -44,7 +44,7 @@ server { } location / { - {{ include "nginx.otelTrace" . | nindent 8 }} + {{ include "heightsNginx.otelTrace" . | nindent 8 }} if ($request_method = 'OPTIONS') { add_header 'Access-Control-Allow-Origin' '*'; diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 166668b..4305d32 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -154,7 +154,7 @@ Returns metrics url from global if exists or from chart's values {{/* Generate OpenTelemetry trace configuration */}} -{{- define "nginx.otelTrace" -}} +{{- define "heightsNginx.otelTrace" -}} {{- if eq .Values.nginx.opentelemetry.samplerMethod "AlwaysOn" -}} otel_trace on; {{- else if eq .Values.nginx.opentelemetry.samplerMethod "TraceIdRatioBased" -}} diff --git a/helm/values.yaml b/helm/values.yaml index fb3a50a..b490346 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -82,15 +82,16 @@ nginx: port: 8080 targetPort: 80 nodePort: 30001 - maxAge: '3600' + nginx: + maxAge: '3600' authorization: enabled: true - domain: "dem" + domain: dem url: http://opa-int-opa-service.infra-services/v1/data/http/authz/decision # url: http://opa.service/v1/data/http/authz/decision opentelemetry: serviceName: nginx - samplerMethod: "AlwaysOff" + samplerMethod: AlwaysOff exporterPort: 4317 ratio: 10 additionalPodAnnotations: @@ -130,7 +131,7 @@ nginx: timeout: enabled: false # defaults to 30s by openshift duration: 60s # supported units (us, ms, s, m, h, d) - rewriteTarget: "" + rewriteTarget: / tls: enabled: false useCert: false From 02c980c4e43459240140195d9a35bcf5c903ba21 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Wed, 3 Sep 2025 14:15:01 +0300 Subject: [PATCH 04/10] deps: use new common helm chart version 2.1.0 --- helm/Chart.lock | 8 ++++---- helm/Chart.yaml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/helm/Chart.lock b/helm/Chart.lock index 420320d..7e66de4 100644 --- a/helm/Chart.lock +++ b/helm/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: nginx - repository: oci://acrarolibotnonprod.azurecr.io/helm - version: 2.0.0 -digest: sha256:a121ab1fbbc1465cf315c21d22afb866bd942f2cf20e59b175374ae51d74dfda -generated: "2025-08-12T19:15:06.506534337+03:00" + repository: oci://acrarolibotnonprod.azurecr.io/helm/common + version: 2.1.0 +digest: sha256:18e8ad9d643309745bcdd30df03b0e01ec004d295ce2d5a7a81fc8c40e0e9206 +generated: "2025-09-02T12:36:32.560619638+03:00" diff --git a/helm/Chart.yaml b/helm/Chart.yaml index dbf5b31..2182c71 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -6,5 +6,5 @@ version: 2.1.7 appVersion: 2.1.7 dependencies: - name: nginx - version: 2.0.0 - repository: oci://acrarolibotnonprod.azurecr.io/helm \ No newline at end of file + version: 2.1.0 + repository: oci://acrarolibotnonprod.azurecr.io/helm/common \ No newline at end of file From 9483a10ce9ac45f988a45e797768934b26c91a9d Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Mon, 8 Sep 2025 13:44:02 +0300 Subject: [PATCH 05/10] style: run prettier --- .release-please-manifest.json | 4 +--- helm/Chart.yaml | 2 +- release-please-config.json | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index ddc217b..0a18a31 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1 @@ -{ - ".": "2.1.7" -} \ No newline at end of file +{ ".": "2.1.7" } diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 2182c71..0437dee 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -7,4 +7,4 @@ appVersion: 2.1.7 dependencies: - name: nginx version: 2.1.0 - repository: oci://acrarolibotnonprod.azurecr.io/helm/common \ No newline at end of file + repository: oci://acrarolibotnonprod.azurecr.io/helm/common diff --git a/release-please-config.json b/release-please-config.json index 9357255..41e607b 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -38,4 +38,4 @@ { "type": "test", "section": "Tests", "hidden": true }, { "type": "ci", "section": "Continuous Integration", "hidden": true } ] -} \ No newline at end of file +} From bd50fe5868ee7dcf8e423821c1a26257b2ecb65a Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 11 Sep 2025 14:30:07 +0300 Subject: [PATCH 06/10] fix: use nginx maxAge value in default.conf --- helm/config/default.conf | 4 +++- helm/values.yaml | 5 ++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/helm/config/default.conf b/helm/config/default.conf index 1768ed2..84b96d5 100644 --- a/helm/config/default.conf +++ b/helm/config/default.conf @@ -1,6 +1,7 @@ {{- $serviceName := include "service.fullname" . -}} {{- $opalaEnabled := .Values.nginx.authorization.enabled }} +{{- $accessControlMaxAge := .Values.nginx.nginx.maxAge }} split_clients "$otel_trace_id" $ratio_sampler { {{ .Values.nginx.opentelemetry.ratio }}% on; @@ -56,7 +57,7 @@ server { # # Tell client that this pre-flight info is valid for 20 days # - add_header 'Access-Control-Max-Age' 1728000; + add_header 'Access-Control-Max-Age' {{ $accessControlMaxAge }}; add_header 'Content-Type' 'text/plain charset=UTF-8'; add_header 'Content-Length' 0; return 204; @@ -66,6 +67,7 @@ server { add_header 'Access-Control-Allow-Origin' '*' always; add_header 'Access-Control-Allow-Methods' 'POST, OPTIONS' always; add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type' always; + add_header 'Access-Control-Max-Age' {{ $accessControlMaxAge }}; } proxy_hide_header Set-Cookie; # ensures the header will not be passed back to the client diff --git a/helm/values.yaml b/helm/values.yaml index b490346..ecea36a 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -83,12 +83,11 @@ nginx: targetPort: 80 nodePort: 30001 nginx: - maxAge: '3600' + maxAge: '1728000' # 20 days authorization: enabled: true domain: dem - url: http://opa-int-opa-service.infra-services/v1/data/http/authz/decision - # url: http://opa.service/v1/data/http/authz/decision + url: http://opa.service/v1/data/http/authz/decision opentelemetry: serviceName: nginx samplerMethod: AlwaysOff From e20463158c8926a0071754fe1b983ec05b2117d8 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 11 Sep 2025 14:37:31 +0300 Subject: [PATCH 07/10] chore: remove nginx values that are the same as the default in the original chart --- helm/values.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/helm/values.yaml b/helm/values.yaml index ecea36a..c919ffb 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -88,11 +88,6 @@ nginx: enabled: true domain: dem url: http://opa.service/v1/data/http/authz/decision - opentelemetry: - serviceName: nginx - samplerMethod: AlwaysOff - exporterPort: 4317 - ratio: 10 additionalPodAnnotations: co.elastic.logs/enabled: 'true' co.elastic.logs/module: 'nginx' From 0890acbb2e380dfb4489c2eeb53b6988fe63befd Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Tue, 16 Sep 2025 16:58:24 +0300 Subject: [PATCH 08/10] chore: remove values matching nginx chart defaults --- helm/values.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/helm/values.yaml b/helm/values.yaml index c919ffb..f6a15ce 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -81,7 +81,6 @@ nginx: tag: latest port: 8080 targetPort: 80 - nodePort: 30001 nginx: maxAge: '1728000' # 20 days authorization: @@ -122,9 +121,6 @@ nginx: routesMapping: - host: path: /api/heights/v1 - timeout: - enabled: false # defaults to 30s by openshift - duration: 60s # supported units (us, ms, s, m, h, d) rewriteTarget: / tls: enabled: false @@ -135,7 +131,6 @@ nginx: ingress: enabled: false - type: nginx-org # supported values: 'nginx-org' | 'nginx-kubernetes' ingressMapping: - host: path: /dem-heights From e3fd266fac2e9a739d94619be04b5a086e21e2a2 Mon Sep 17 00:00:00 2001 From: Shimon Cohen <33935191+shimoncohen@users.noreply.github.com> Date: Wed, 17 Sep 2025 17:43:10 +0300 Subject: [PATCH 09/10] Update helm/config/nginx.conf Co-authored-by: Netanel Cohen <34451523+netanelC@users.noreply.github.com> --- helm/config/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/config/nginx.conf b/helm/config/nginx.conf index 93a1125..6651bac 100644 --- a/helm/config/nginx.conf +++ b/helm/config/nginx.conf @@ -3,7 +3,7 @@ load_module modules/ngx_http_js_module.so; load_module modules/ngx_otel_module.so; worker_processes 4; -error_log /var/log/nginx/error.log notice; +error_log /var/log/nginx/error.log warn; pid /tmp/nginx.pid; events { From e40c80bb71cdca2aa88f6761d01c2856b7015020 Mon Sep 17 00:00:00 2001 From: shimoncohen Date: Thu, 18 Sep 2025 10:13:04 +0300 Subject: [PATCH 10/10] chore: remove nginx ingress default values --- helm/values.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/helm/values.yaml b/helm/values.yaml index f6a15ce..a0e5d8b 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -128,12 +128,3 @@ nginx: certificate: "" key: "" caCertificate: "" - - ingress: - enabled: false - ingressMapping: - - host: - path: /dem-heights - tls: - enabled: false - secretName: ingress-tls-secret