feat(observability): OTel-native operator metrics (Phase 1)

Dockerfile

@@ -13,7 +13,7 @@ COPY go.mod go.sum ./
 RUN go mod download
 
 # Copy the go source
-COPY cmd/team-operator/main.go cmd/team-operator/main.go
+COPY cmd/team-operator/ cmd/team-operator/
 COPY api/ api/
 COPY internal/ ./internal/
 
@@ -29,7 +29,7 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \
     -ldflags="-X 'github.com/posit-dev/team-operator/internal.VersionString=${VERSION}'"\
     -a \
     -o team-operator \
-    cmd/team-operator/main.go
+    ./cmd/team-operator/
 
 # Use distroless as minimal base image to package the team-operator binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

Justfile

@@ -28,7 +28,7 @@ deps-up:
 
 # Run team-operator directly from source
 run:
-  go run cmd/team-operator/main.go
+  go run ./cmd/team-operator/
 
 # Run team-operator via the Makefile target
 mrun:
@@ -40,7 +40,7 @@ build:
     -ldflags="-X 'github.com/posit-dev/team-operator/internal.VersionString={{ VERSION }}'" \
     -a \
     -o ./bin/team-operator \
-    cmd/team-operator/main.go
+    ./cmd/team-operator/
 
 # Build ./bin/team-operator via the Makefile target
 mbuild:

Makefile

@@ -190,7 +190,7 @@ test-integration: go-test test-kind ## Run all tests (unit + integration).
 
 .PHONY: build
 build: copy-crds generate-all fmt vet ## Build manager binary.
-	go build -o bin/team-operator ./cmd/team-operator/main.go
+	go build -o bin/team-operator ./cmd/team-operator/
 
 .PHONY: docker-build
 docker-build: build ## Build the operator Docker image.
@@ -203,7 +203,7 @@ distclean:
 
 .PHONY: run
 run: manifests generate-all fmt vet ## Run a controller from your host.
-	go run ./cmd/team-operator/main.go
+	go run ./cmd/team-operator/
 
 ##@ Deployment
 

api/core/v1beta1/chronicle_types.go

@@ -119,10 +119,13 @@ func (c *Chronicle) KubernetesLabels() map[string]string {
 }
 
 func (c *Chronicle) OwnerReferencesForChildren() []metav1.OwnerReference {
+	// APIVersion/Kind are hardcoded because controller-runtime's client.Get
+	// strips TypeMeta from typed-object responses, leaving c.APIVersion and
+	// c.Kind empty in the reconcile path.
 	return []metav1.OwnerReference{
 		{
-			APIVersion: c.APIVersion,
-			Kind:       c.Kind,
+			APIVersion: GroupVersion.String(),
+			Kind:       "Chronicle",
 			Name:       c.Name,
 			UID:        c.UID,
 		},

api/core/v1beta1/connect_types.go

@@ -275,10 +275,13 @@ func (c *Connect) GetAwsAccountId() string {
 }
 
 func (c *Connect) OwnerReferencesForChildren() []metav1.OwnerReference {
+	// APIVersion/Kind are hardcoded because controller-runtime's client.Get
+	// strips TypeMeta from typed-object responses, leaving c.APIVersion and
+	// c.Kind empty in the reconcile path.
 	return []metav1.OwnerReference{
 		{
-			APIVersion: c.APIVersion,
-			Kind:       c.Kind,
+			APIVersion: GroupVersion.String(),
+			Kind:       "Connect",
 			Name:       c.Name,
 			UID:        c.UID,
 		},

api/core/v1beta1/packagemanager_types.go

@@ -414,10 +414,13 @@ func (pm *PackageManager) CreateSecretVolumeFactory() *product.SecretVolumeFacto
 }
 
 func (pm *PackageManager) OwnerReferencesForChildren() []metav1.OwnerReference {
+	// APIVersion/Kind are hardcoded because controller-runtime's client.Get
+	// strips TypeMeta from typed-object responses, leaving pm.APIVersion and
+	// pm.Kind empty in the reconcile path.
 	return []metav1.OwnerReference{
 		{
-			APIVersion: pm.APIVersion,
-			Kind:       pm.Kind,
+			APIVersion: GroupVersion.String(),
+			Kind:       "PackageManager",
 			Name:       pm.Name,
 			UID:        pm.UID,
 		},

api/core/v1beta1/site_types.go

@@ -729,10 +729,13 @@ func (s *Site) GetSecretType() product.SiteSecretType {
 }
 
 func (s *Site) OwnerReferencesForChildren() []metav1.OwnerReference {
+	// APIVersion/Kind are hardcoded because controller-runtime's client.Get
+	// strips TypeMeta from typed-object responses, leaving s.APIVersion and
+	// s.Kind empty in the reconcile path.
 	return []metav1.OwnerReference{
 		{
-			APIVersion: s.APIVersion,
-			Kind:       s.Kind,
+			APIVersion: GroupVersion.String(),
+			Kind:       "Site",
 			Name:       s.Name,
 			UID:        s.UID,
 		},

api/core/v1beta1/workbench_types.go

@@ -236,10 +236,13 @@ func init() {
 }
 
 func (w *Workbench) OwnerReferencesForChildren() []metav1.OwnerReference {
+	// APIVersion/Kind are hardcoded because controller-runtime's client.Get
+	// strips TypeMeta from typed-object responses, leaving w.APIVersion and
+	// w.Kind empty in the reconcile path.
 	return []metav1.OwnerReference{
 		{
-			APIVersion: w.APIVersion,
-			Kind:       w.Kind,
+			APIVersion: GroupVersion.String(),
+			Kind:       "Workbench",
 			Name:       w.Name,
 			UID:        w.UID,
 		},

cmd/team-operator/main.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/posit-dev/team-operator/api/keycloak/v2alpha1"
 	"github.com/posit-dev/team-operator/api/product"
+	"github.com/posit-dev/team-operator/internal/observability"
 	"github.com/traefik/traefik/v3/pkg/provider/kubernetes/crd/traefikio/v1alpha1"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
@@ -111,6 +112,20 @@ func main() {
 			"configurable Workbench session pod field and writes one numbered label per "+
 			"match onto the pod. Per-site config lives in the Workbench CR's sessionLabels field.")
 
+	var (
+		obsMetricsOTLPEndpoint   string
+		obsMetricsExportInterval time.Duration
+		obsClusterName           string
+	)
+
+	flag.StringVar(&obsMetricsOTLPEndpoint, "observability-metrics-otlp-endpoint", "",
+		"gRPC OTLP endpoint for metric push (e.g. otel-collector:4317). "+
+			"Falls back to OTEL_EXPORTER_OTLP_METRICS_ENDPOINT then OTEL_EXPORTER_OTLP_ENDPOINT.")
+	flag.DurationVar(&obsMetricsExportInterval, "observability-metrics-export-interval", 30*time.Second,
+		"Cadence for OTLP metric export and async gauge collection")
+	flag.StringVar(&obsClusterName, "observability-cluster-name", "",
+		"Value for the k8s.cluster.name resource attribute")
+
 	opts := zap.Options{Development: true}
 
 	opts.BindFlags(flag.CommandLine)
@@ -124,6 +139,26 @@ func main() {
 
 	zl.Info("team-operator version", "version", internal.VersionString)
 
+	instanceID := os.Getenv("POD_NAME")
+	if instanceID == "" {
+		setupLog.Info("POD_NAME env var not set; service.instance.id resource attribute will be empty. " +
+			"Wire POD_NAME from the downward API (metadata.name) for per-pod metric aggregation.")
+	}
+
+	obsProvider := observability.NewProvider(context.Background(), observability.Config{
+		OTLPEndpoint:          obsMetricsOTLPEndpoint,
+		MetricsExportInterval: obsMetricsExportInterval,
+		ClusterName:           obsClusterName,
+		InstanceID:            instanceID,
+	})
+	defer func() {
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		if err := obsProvider.Shutdown(shutdownCtx); err != nil {
+			setupLog.Error(err, "error shutting down observability provider")
+		}
+	}()
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme: scheme,
 		Metrics: server.Options{
@@ -171,62 +206,69 @@ func main() {
 	}
 
 	if err = (&corecontroller.SiteReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
-		Log:    setupLog,
+		Client:      mgr.GetClient(),
+		Scheme:      mgr.GetScheme(),
+		Log:         setupLog,
+		Instruments: observability.NewInstruments(obsProvider.Meter("team-operator/site")),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Site")
 		os.Exit(1)
 	}
 
 	if err = (&corecontroller.PostgresDatabaseReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
-		Log:    setupLog,
+		Client:      mgr.GetClient(),
+		Scheme:      mgr.GetScheme(),
+		Log:         setupLog,
+		Instruments: observability.NewInstruments(obsProvider.Meter("team-operator/postgres-database")),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "PostgresDatabase")
 		os.Exit(1)
 	}
 
 	if err = (&corecontroller.ConnectReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
-		Log:    setupLog,
+		Client:      mgr.GetClient(),
+		Scheme:      mgr.GetScheme(),
+		Log:         setupLog,
+		Instruments: observability.NewInstruments(obsProvider.Meter("team-operator/connect")),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "ImplConnect")
 		os.Exit(1)
 	}
 
 	if err = (&corecontroller.WorkbenchReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:      mgr.GetClient(),
+		Scheme:      mgr.GetScheme(),
+		Instruments: observability.NewInstruments(obsProvider.Meter("team-operator/workbench")),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Workbench")
 		os.Exit(1)
 	}
 
 	if err = (&corecontroller.PackageManagerReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
-		Log:    setupLog,
+		Client:      mgr.GetClient(),
+		Scheme:      mgr.GetScheme(),
+		Log:         setupLog,
+		Instruments: observability.NewInstruments(obsProvider.Meter("team-operator/package-manager")),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "PackageManager")
 		os.Exit(1)
 	}
 
 	if err = (&corecontroller.ChronicleReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
-		Log:    setupLog,
+		Client:      mgr.GetClient(),
+		Scheme:      mgr.GetScheme(),
+		Log:         setupLog,
+		Instruments: observability.NewInstruments(obsProvider.Meter("team-operator/chronicle")),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Chronicle")
 		os.Exit(1)
 	}
 
 	if err = (&corecontroller.FlightdeckReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
-		Log:    setupLog,
+		Client:      mgr.GetClient(),
+		Scheme:      mgr.GetScheme(),
+		Log:         setupLog,
+		Instruments: observability.NewInstruments(obsProvider.Meter("team-operator/flightdeck")),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Flightdeck")
 		os.Exit(1)
@@ -248,6 +290,14 @@ func main() {
 
 	//+kubebuilder:scaffold:builder
 
+	lister := &multiKindLister{client: mgr.GetClient(), log: setupLog}
+	if err := observability.RegisterResourceCountGauge(
+		obsProvider.Meter("team-operator/resource-count"),
+		lister,
+	); err != nil {
+		setupLog.Error(err, "failed to register resource count gauge; continuing without it")
+	}
+
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")
 		os.Exit(1)