feat: support oxia metadata store

charts/sn-platform-slim/templates/_helpers.tpl

@@ -189,6 +189,99 @@ Pulsar Cluster Name.
 {{- end }}
 {{- end }}
 
+{{/*
+Metadata provider selection.
+*/}}
+{{- define "pulsar.metadata.provider" -}}
+{{- default "zookeeper" .Values.pulsar_metadata.provider -}}
+{{- end -}}
+
+{{- define "pulsar.metadata.isZookeeper" -}}
+{{- if eq (include "pulsar.metadata.provider" .) "zookeeper" -}}true{{- end -}}
+{{- end -}}
+
+{{- define "pulsar.metadata.isOxia" -}}
+{{- if eq (include "pulsar.metadata.provider" .) "oxia" -}}true{{- end -}}
+{{- end -}}
+
+{{- define "pulsar.oxia.cluster" -}}
+{{- template "pulsar.fullname" . -}}
+{{- end -}}
+
+{{- define "pulsar.oxia.service" -}}
+{{- template "pulsar.oxia.cluster" . -}}-oxia
+{{- end -}}
+
+{{- define "pulsar.oxia.service.address" -}}
+{{- template "pulsar.oxia.service" . -}}.{{ template "pulsar.namespace" . }}.svc.cluster.local:6648
+{{- end -}}
+
+{{- define "pulsar.oxia.broker.namespace" -}}
+broker
+{{- end -}}
+
+{{- define "pulsar.oxia.bookkeeper.namespace" -}}
+bookkeeper
+{{- end -}}
+
+{{- define "pulsar.oxia.schema.namespace" -}}
+pulsar-schema
+{{- end -}}
+
+{{- define "pulsar.oxia.function.namespace" -}}
+function
+{{- end -}}
+
+{{- define "pulsar.oxia.broker.url" -}}
+oxia://{{ template "pulsar.oxia.service.address" . }}/{{ template "pulsar.oxia.broker.namespace" . }}
+{{- end -}}
+
+{{- define "pulsar.oxia.bookkeeper.url" -}}
+oxia://{{ template "pulsar.oxia.service.address" . }}/{{ template "pulsar.oxia.bookkeeper.namespace" . }}
+{{- end -}}
+
+{{- define "pulsar.oxia.schema.url" -}}
+oxia://{{ template "pulsar.oxia.service.address" . }}/{{ template "pulsar.oxia.schema.namespace" . }}
+{{- end -}}
+
+{{- define "pulsar.oxia.metadata.url" -}}
+{{- template "pulsar.oxia.broker.url" . -}}
+{{- end -}}
+
+{{- define "pulsar.oxia.bookkeeper.metadataServiceUri" -}}
+metadata-store:{{ template "pulsar.oxia.bookkeeper.url" . }}
+{{- end -}}
+
+{{- define "pulsar.oxia.podMonitor.enabled" -}}
+{{- if and .Values.monitoring.prometheus (or (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/PodMonitor") (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1")) -}}
+true
+{{- else -}}
+false
+{{- end -}}
+{{- end -}}
+
+{{- define "pulsar.validate.metadata" -}}
+{{- $provider := include "pulsar.metadata.provider" . -}}
+{{- if and (ne $provider "zookeeper") (ne $provider "oxia") -}}
+{{- fail "pulsar_metadata.provider must be one of: zookeeper, oxia" -}}
+{{- end -}}
+{{- if and (eq $provider "oxia") .Values.metadataPrefix -}}
+{{- fail "metadataPrefix is only supported with pulsar_metadata.provider=zookeeper" -}}
+{{- end -}}
+{{- if and (eq $provider "oxia") .Values.pulsar_metadata.userProvidedZookeepers -}}
+{{- fail "pulsar_metadata.userProvidedZookeepers is only supported with pulsar_metadata.provider=zookeeper" -}}
+{{- end -}}
+{{- if and (eq $provider "oxia") .Values.pulsar_metadata.configurationStoreServers -}}
+{{- fail "pulsar_metadata.configurationStoreServers is only supported with pulsar_metadata.provider=zookeeper" -}}
+{{- end -}}
+{{- if and (eq $provider "oxia") .Values.components.sql_worker -}}
+{{- fail "components.sql_worker is not supported with pulsar_metadata.provider=oxia" -}}
+{{- end -}}
+{{- if and (eq $provider "oxia") (lt (int .Values.oxia.replicaCount) 3) -}}
+{{- fail "oxia.replicaCount must be at least 3 when pulsar_metadata.provider=oxia" -}}
+{{- end -}}
+{{- end -}}
+
 {{/*
 Istio gateway selector
 */}}
@@ -213,6 +306,15 @@ prometheus.istio.io/merge-metrics: "false"
 {{- end -}}
 {{- end -}}
 
+{{/*
+Whether Prometheus should use sidecar-issued mTLS certs to scrape workload metrics.
+*/}}
+{{- define "pulsar.istio.prometheus.sidecarMTLS" -}}
+{{- if and .Values.istio.enabled (not .Values.istio.mergeMetrics) (ne (default "" .Values.istio.dataplaneMode) "ambient") -}}
+true
+{{- end -}}
+{{- end -}}
+
 {{/*
 Define TLS CA secret name
 */}}

charts/sn-platform-slim/templates/bookkeeper/_bookkeeper.tpl

@@ -134,8 +134,12 @@ Define bookie tls certs volumes
 Define bookie common config
 */}}
 {{- define "pulsar.bookkeeper.config.common" -}}
+{{- if include "pulsar.metadata.isOxia" . }}
+metadataServiceUri: "{{ template "pulsar.oxia.bookkeeper.metadataServiceUri" . }}"
+{{- else }}
 zkServers: "{{ template "pulsar.zookeeper.connect" . }}"
 zkLedgersRootPath: "{{ .Values.metadataPrefix }}/ledgers"
+{{- end }}
 # enable bookkeeper http server
 httpServerEnabled: "true"
 httpServerPort: "{{ .Values.bookkeeper.ports.http }}"

charts/sn-platform-slim/templates/bookkeeper/bookkeeper-cluster.yaml

@@ -29,7 +29,11 @@ spec:
 {{- if not .Values.initialize }}
   initialized: true
 {{- end }}
+{{- if include "pulsar.metadata.isOxia" . }}
+  metadataServiceUri: "{{ template "pulsar.oxia.bookkeeper.metadataServiceUri" . }}"
+{{- else }}
   zkServers: "{{ template "pulsar.zookeeper.connect" . }}"
+{{- end }}
   replicas: {{ .Values.bookkeeper.replicaCount }}
   {{- if not (and .Values.components.pulsar_coordinator .Values.images.coordinator) }}
   image: "{{ .Values.images.bookie.repository }}:{{ .Values.images.bookie.tag }}"
@@ -199,7 +203,11 @@ spec:
 {{- with .Values.bookkeeper.configData }}
 {{ toYaml . | indent 6 }}
 {{- end }}
+      {{- if include "pulsar.metadata.isZookeeper" . }}
       zkLedgersRootPath: "{{ .Values.metadataPrefix }}/ledgers"
+      {{- else }}
+      metadataServiceUri: "{{ template "pulsar.oxia.bookkeeper.metadataServiceUri" . }}"
+      {{- end }}
       # enable bookkeeper http server
       httpServerEnabled: "true"
       httpServerPort: "{{ .Values.bookkeeper.ports.http }}"
@@ -351,7 +359,7 @@ spec:
   {{- with .Values.bookkeeper.customization }}
   {{- toYaml . | nindent 2 -}}
   {{- end }}
-  {{- if .Values.zookeeper.customTools.restore.enable}}
+  {{- if and .Values.zookeeper.customTools.restore.enable (include "pulsar.metadata.isZookeeper" .) }}
     - match:
         groupVersionKinds:
           - kind: Job

charts/sn-platform-slim/templates/broker/broker-cluster.yaml

@@ -30,8 +30,14 @@ spec:
 {{- if not .Values.initialize }}
   initialized: true
 {{- end }}
+{{- if include "pulsar.metadata.isOxia" . }}
+  metadataStoreUrl: "{{ template "pulsar.oxia.metadata.url" . }}"
+  configurationMetadataStoreUrl: "{{ template "pulsar.oxia.metadata.url" . }}"
+  bkMetadataServiceUri: "{{ template "pulsar.oxia.bookkeeper.metadataServiceUri" . }}"
+{{- else }}
   zkServers: "{{ template "pulsar.zookeeper.connect" . }}{{ .Values.metadataPrefix }}"
   configurationStoreServers: "{{ template "pulsar.configurationStore.connect" . }}{{ .Values.metadataPrefix }}"
+{{- end }}
 {{- if not .Values.broker.autoScaling.enabled }}
   replicas: {{ .Values.broker.replicaCount }}
 {{- end }}
@@ -226,6 +232,12 @@ spec:
     {{- if .Values.pulsar_metadata.clusterName }}
     clusterName: {{ .Values.pulsar_metadata.clusterName }}
     {{- end }}
+    {{- if include "pulsar.metadata.isOxia" . }}
+    oxiaBasedSystemTopic:
+      enabled: true
+      schemaStorageUrl: "{{ template "pulsar.oxia.schema.url" . }}"
+      topicPoliciesServiceClassName: "io.streamnative.pulsar.OxiaTopicPoliciesService"
+    {{- end }}
     {{- if .Values.broker.functionmesh.enabled }}
     function:
       enabled: true
@@ -324,6 +336,11 @@ spec:
     {{- end }}
     custom:
       PULSAR_PREFIX_additionalServletDirectory: "./brokerAdditionalServlet"
+      {{- if include "pulsar.metadata.isOxia" . }}
+      PULSAR_PREFIX_metadataStoreUrl: "{{ template "pulsar.oxia.metadata.url" . }}"
+      PULSAR_PREFIX_configurationMetadataStoreUrl: "{{ template "pulsar.oxia.metadata.url" . }}"
+      PULSAR_PREFIX_enablePackagesManagement: "false"
+      {{- end }}
 {{- with .Values.broker.configData }}
 {{ toYaml . | indent 6 }}
 {{- end }}

charts/sn-platform-slim/templates/broker/function-worker-configfile-configmap.yaml

@@ -65,7 +65,11 @@ data:
     pulsarServiceUrl: {{template "pulsar.function.broker.service.url" . }}
     pulsarWebServiceUrl: {{template "pulsar.function.web.service.url" . }}
     {{- end }}
+    {{- if include "pulsar.metadata.isOxia" . }}
+    configurationMetadataStoreUrl: "{{ template "pulsar.oxia.metadata.url" . }}"
+    {{- else }}
     configurationStoreServers: "{{ template "pulsar.configurationStore.connect" . }}{{ .Values.metadataPrefix }}"
+    {{- end }}
     {{- if .Values.auth.authorization.enabled }}
     authorizationEnabled: "true"
     authorizationProvider: {{ .Values.functions.authorizationProvider }}

charts/sn-platform-slim/templates/detector/pulsar-detector-deployment.yaml

@@ -56,6 +56,7 @@ spec:
       terminationGracePeriodSeconds: {{ .Values.pulsar_detector.gracePeriod }}
       {{- if not .Values.istio.enabled }}
       initContainers:
+      {{- if include "pulsar.metadata.isZookeeper" . }}
       # This init container will wait for zookeeper to be ready before
       # deploying the bookies
       - name: wait-zookeeper-ready
@@ -73,6 +74,7 @@ spec:
         volumeMounts:
         - name: tmp
           mountPath: /pulsar/logs
+      {{- end }}
       # This init container will wait for at least one broker to be ready before
       # deploying the pulsar-detector
       - name: wait-broker-ready

charts/sn-platform-slim/templates/extra.yaml

@@ -1,6 +1,7 @@
 #
 # Copyright (c) 2019 - 2024 StreamNative, Inc.. All Rights Reserved.
 #
+{{- include "pulsar.validate.metadata" . }}
 {{- range .Values.extraResources }}
 {{ toYaml . }}
 ---

charts/sn-platform-slim/templates/openshift/scc-rolebinding.yaml

@@ -3,8 +3,10 @@
 #
 
 {{- if and .Values.openshift.enabled .Values.openshift.scc.enabled -}}
-{{- $sas := list (include "pulsar.zookeeper.serviceAccount" .) -}}
-{{- $sas = append $sas (include "pulsar.bookkeeper.serviceAccount" .) -}}
+{{- $sas := list (include "pulsar.bookkeeper.serviceAccount" .) -}}
+{{- if include "pulsar.metadata.isZookeeper" . -}}
+{{- $sas = append $sas (include "pulsar.zookeeper.serviceAccount" .) -}}
+{{- end -}}
 {{- $sas = append $sas (include "pulsar.broker.serviceAccount" .) -}}
 {{- $sas = append $sas (include "pulsar.proxy.serviceAccount" .) -}}
 {{- $sas = append $sas (include "pulsar.streamnative_console.serviceAccount" .) -}}
@@ -24,4 +26,4 @@ subjects:
     name: {{ . }}
     namespace: {{ include "pulsar.namespace" $ }}
     {{- end -}}
-{{- end -}}
+{{- end -}}

charts/sn-platform-slim/templates/oxia/oxia-cluster.yaml

@@ -0,0 +1,56 @@
+#
+# Copyright (c) 2019 - 2024 StreamNative, Inc.. All Rights Reserved.
+#
+# deploy oxia only when `pulsar_metadata.provider` is oxia
+{{- if include "pulsar.metadata.isOxia" . }}
+apiVersion: k8s.streamnative.io/v1alpha1
+kind: OxiaCluster
+metadata:
+  name: "{{ template "pulsar.oxia.cluster" . }}"
+  namespace: {{ template "pulsar.namespace" . }}
+{{- with .Values.oxia.annotations }}
+  annotations:
+{{ toYaml . | indent 4 }}
+{{- end }}
+  labels:
+    {{- include "pulsar.standardLabels" . | nindent 4 }}
+    component: oxia
+{{- with .Values.oxia.labels }}
+{{ toYaml . | indent 4 }}
+{{- end }}
+{{- if .Values.components.pulsar_coordinator }}
+    k8s.streamnative.io/coordinator-name: "{{ template "pulsar.fullname" . }}-coordinator"
+{{- end }}
+spec:
+  image: "{{ .Values.images.oxia.repository }}:{{ .Values.images.oxia.tag }}"
+  imagePullPolicy: {{ .Values.images.oxia.pullPolicy }}
+  monitoringEnabled: {{ include "pulsar.oxia.podMonitor.enabled" . }}
+  {{- with .Values.oxia.labels }}
+  labels:
+{{ toYaml . | indent 4 }}
+  {{- end }}
+  {{- with .Values.oxia.coordinator }}
+  {{- with .resources }}
+  coordinator:
+    resources:
+{{ toYaml . | indent 6 }}
+  {{- end }}
+  {{- end }}
+  server:
+    replicas: {{ .Values.oxia.replicaCount }}
+    {{- with .Values.oxia.resources }}
+    resources:
+{{ toYaml . | indent 6 }}
+    {{- end }}
+  {{- if .Values.istio.enabled }}
+  istio:
+    enabled: true
+    {{- if .Values.istio.dataplaneMode }}
+    dataplaneMode: {{ .Values.istio.dataplaneMode }}
+    {{- end }}
+    {{- if .Values.istio.migration }}
+    mtls:
+      mode: permissive
+    {{- end }}
+  {{- end }}
+{{- end }}

charts/sn-platform-slim/templates/oxia/oxia-namespace.yaml

@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2019 - 2024 StreamNative, Inc.. All Rights Reserved.
+#
+# deploy oxia namespaces only when `pulsar_metadata.provider` is oxia
+{{- if include "pulsar.metadata.isOxia" . }}
+{{- $brokerNamespace := dict "name" (include "pulsar.oxia.broker.namespace" .) "type" "broker" }}
+{{- $bookkeeperNamespace := dict "name" (include "pulsar.oxia.bookkeeper.namespace" .) "type" "bookkeeper" }}
+{{- $schemaNamespace := dict "name" (include "pulsar.oxia.schema.namespace" .) "type" "pulsar-schema" }}
+{{- $functionNamespace := dict "name" (include "pulsar.oxia.function.namespace" .) "type" "function" }}
+{{- $namespaces := list $brokerNamespace $bookkeeperNamespace $schemaNamespace $functionNamespace }}
+{{- range $index, $oxiaNamespace := $namespaces }}
+{{- if $index }}
+---
+{{- end }}
+apiVersion: k8s.streamnative.io/v1alpha1
+kind: OxiaNamespace
+metadata:
+  name: "{{ template "pulsar.oxia.cluster" $ }}-{{ $oxiaNamespace.type }}"
+  namespace: {{ template "pulsar.namespace" $ }}
+  labels:
+    {{- include "pulsar.standardLabels" $ | nindent 4 }}
+    component: oxia
+spec:
+  namespaceConfig:
+    name: "{{ $oxiaNamespace.name }}"
+    initialShardCount: 1
+    replicationFactor: 3
+    notificationsEnabled: true
+  clusterRef:
+    name: "{{ template "pulsar.oxia.cluster" $ }}"
+    namespace: {{ template "pulsar.namespace" $ }}
+  ownerRef:
+    name: "{{ template "pulsar.fullname" $ }}"
+    namespace: {{ template "pulsar.namespace" $ }}
+{{- end }}
+{{- end }}