From 0f0fba5750e854de5cc5c4ab11e155ae46d56dc7 Mon Sep 17 00:00:00 2001 From: Gianluca Mardente Date: Tue, 14 Jul 2026 09:27:16 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20add=20metrics=20=E2=80=94=20fix=20cardi?= =?UTF-8?q?nality=20bug,=20add=20outcome/adoption=20tracking?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Metrics (all projectsveltos_-prefixed): - **reconcile_duration_seconds** (histogram): cluster_type, cluster_namespace, cluster_name. No feature label (unlike addon-controller's metric of the same name) — EventTrigger processing isn't split by feature type. - **reconcile_outcome_total** (counter): cluster_type, cluster_namespace, cluster_name, status, event_trigger_name. - **matching_clusters** (gauge): event_trigger_name. New — clusters currently matching an EventTrigger's sourceClusterSelector. - **matching_resources** (gauge): event_trigger_name, cluster_type, cluster_namespace, cluster_name. Resources matched by the most recently processed EventReport, independent of whether ClusterProfile creation subsequently succeeds. - **reconcile_last_success_timestamp_seconds** (gauge): cluster_type, cluster_namespace, cluster_name, event_trigger_name. New. Also: - Fix metric name prefix - Enable auth-proxy RBAC resources in config/rbac/kustomization.yaml; regenerate manifest.yaml. - Add manifest/service_monitor.yaml --- .github/workflows/codeql.yaml | 51 ++++++ .github/workflows/scorecard.yaml | 45 +++++ README.md | 11 +- SECURITY.md | 47 ++++++ config/rbac/kustomization.yaml | 8 +- controllers/eventreport_collection.go | 3 + controllers/eventtrigger_controller.go | 4 +- controllers/eventtrigger_deployer.go | 44 +++-- controllers/metrics.go | 217 +++++++++++++++++++------ manifest/manifest.yaml | 57 +++++++ manifest/service_monitor.yaml | 18 ++ renovate.json | 11 ++ 12 files changed, 447 insertions(+), 69 deletions(-) create mode 100644 .github/workflows/codeql.yaml create mode 100644 .github/workflows/scorecard.yaml create mode 100644 SECURITY.md create mode 100644 manifest/service_monitor.yaml create mode 100644 renovate.json diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml new file mode 100644 index 0000000..3e20059 --- /dev/null +++ b/.github/workflows/codeql.yaml @@ -0,0 +1,51 @@ +name: CodeQL +on: + push: + branches: + - 'main' + pull_request: + branches: + - 'main' + schedule: + - cron: '21 3 * * 3' + +permissions: read-all + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + security-events: write + packages: read + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + language: ['go'] + + steps: + - name: checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Set up Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version: 1.26.5 + + - name: Initialize CodeQL + uses: github/codeql-action/init@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4.36.3 + with: + languages: ${{ matrix.language }} + + - name: Build + # Autobuild's heuristic only traced a fraction of the module (79 of 189 files per the + # code-scanning diagnostic). Build the whole module explicitly so CodeQL sees every package. + run: go build ./... + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4.36.3 + with: + category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/scorecard.yaml b/.github/workflows/scorecard.yaml new file mode 100644 index 0000000..58dc5be --- /dev/null +++ b/.github/workflows/scorecard.yaml @@ -0,0 +1,45 @@ +name: Scorecard supply-chain security +on: + branch_protection_rule: + schedule: + - cron: '30 2 * * 1' + push: + branches: + - 'main' + +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + permissions: + security-events: write + id-token: write + contents: read + actions: read + + steps: + - name: checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Run analysis + uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3 + with: + results_file: results.sarif + results_format: sarif + publish_results: true + + - name: Upload artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + - name: Upload to code-scanning + uses: github/codeql-action/upload-sarif@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4.36.3 + with: + sarif_file: results.sarif diff --git a/README.md b/README.md index ad8a6fe..8b5e872 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![CI](https://github.com/projectsveltos/event-manager/actions/workflows/main.yaml/badge.svg)](https://github.com/projectsveltos/event-manager/actions) -[![Go Report Card](https://goreportcard.com/badge/github.com/projectsveltos/event-manager)](https://goreportcard.com/report/github.com/projectsveltos/event-manager) +[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/projectsveltos/event-manager/badge)](https://scorecard.dev/viewer/?uri=github.com/projectsveltos/event-manager) +[![CodeQL](https://github.com/projectsveltos/event-manager/actions/workflows/codeql.yaml/badge.svg)](https://github.com/projectsveltos/event-manager/actions/workflows/codeql.yaml) [![Release](https://img.shields.io/github/v/release/projectsveltos/event-manager)](https://github.com/projectsveltos/event-manager/releases) [![License](https://img.shields.io/badge/license-Apache-blue.svg)](LICENSE) [![Slack](https://img.shields.io/badge/join%20slack-%23projectsveltos-brighteen)](https://join.slack.com/t/projectsveltos/shared_invite/zt-1hraownbr-W8NTs6LTimxLPB8Erj8Q6Q) @@ -52,8 +53,8 @@ spec: value: fv ``` -Sveltos supports custom events written in [Lua](https://www.lua.org/). -Following EventSource instance again defines an Event as the creation/deletion of a Service with label *sveltos: fv* but using a Lua script. +Sveltos supports custom events written in [Lua](https://www.lua.org/). +Following EventSource instance again defines an Event as the creation/deletion of a Service with label *sveltos: fv* but using a Lua script. ```yaml apiVersion: lib.projectsveltos.io/v1beta1 @@ -89,11 +90,11 @@ spec: Event manager is a Sveltos micro service in charge of deploying add-ons when certain events happen in managed clusters. -## Contributing +## Contributing ❤️ Your contributions are always welcome! If you want to contribute, have questions, noticed any bug or want to get the latest project news, you can connect with us in the following ways: -1. Open a bug/feature enhancement on github [![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/projectsveltos/addon-controller/issues) +1. Open a bug/feature enhancement on github [![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/projectsveltos/event-manager/issues) 2. Chat with us on the Slack in the #projectsveltos channel [![Slack](https://img.shields.io/badge/join%20slack-%23projectsveltos-brighteen)](https://join.slack.com/t/projectsveltos/shared_invite/zt-1hraownbr-W8NTs6LTimxLPB8Erj8Q6Q) 3. [Contact Us](mailto:support@projectsveltos.io) diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..1e24d31 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,47 @@ +# Security Policy + +## Supported Versions + +We release security fixes for the latest minor version. We encourage all users to stay on the latest release. + +| Version | Supported | +|----------------|--------------------| +| latest release | :white_check_mark: | +| older releases | :x: | + +## Reporting a Vulnerability + +**Please do not report security vulnerabilities through public GitHub issues.** + +If you believe you have found a security vulnerability in any Sveltos repository, please report it responsibly by sending an email to: + +**support@projectsveltos.io** + +Please include as much of the following information as possible to help us understand and resolve the issue quickly: + +- A description of the vulnerability and its potential impact +- The affected component(s) and version(s) +- Step-by-step instructions to reproduce the issue +- Any proof-of-concept or exploit code (if applicable) +- Suggested remediation (if any) + +## Response Process + +- You will receive an acknowledgement within **2 business days** +- We will investigate and keep you informed of our progress +- Once the issue is confirmed, we will work on a fix and coordinate a release +- We will publicly disclose the vulnerability after a fix is available, giving you credit unless you prefer to remain anonymous + +## Scope + +This policy covers all projects under the [projectsveltos](https://github.com/projectsveltos) GitHub organization. + +## Out of Scope + +- Vulnerabilities in dependencies (please report those to the upstream project) +- Issues in non-production branches or unreleased code +- Social engineering attacks + +## Thank You + +We appreciate responsible disclosure and the work of the security community in keeping Sveltos and its users safe. diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 5d9a44e..4483ced 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -10,10 +10,10 @@ resources: # Comment the following 4 lines if you want to disable # the auth proxy (https://github.com/brancz/kube-rbac-proxy) # which protects your /metrics endpoint. -# - auth_proxy_service.yaml -# - auth_proxy_role.yaml -# - auth_proxy_role_binding.yaml -# - auth_proxy_client_clusterrole.yaml +- auth_proxy_service.yaml +- auth_proxy_role.yaml +- auth_proxy_role_binding.yaml +- auth_proxy_client_clusterrole.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines diff --git a/controllers/eventreport_collection.go b/controllers/eventreport_collection.go index 315ec75..c9a8798 100644 --- a/controllers/eventreport_collection.go +++ b/controllers/eventreport_collection.go @@ -840,6 +840,9 @@ func updateAllClusterProfiles(ctx context.Context, mgmtClient client.Client, clu continue } + trackMatchingResources(eventTriggers[i].Name, cluster.Namespace, cluster.Name, clusterType, + len(er.Spec.MatchingResources), l) + l.V(logs.LogDebug).Info("updating ClusterProfile") err = updateClusterProfiles(ctx, mgmtClient, cluster.Namespace, cluster.Name, clusterType, eventTriggers[i], er, l) diff --git a/controllers/eventtrigger_controller.go b/controllers/eventtrigger_controller.go index f652576..a021a13 100644 --- a/controllers/eventtrigger_controller.go +++ b/controllers/eventtrigger_controller.go @@ -277,7 +277,9 @@ func (r *EventTriggerReconciler) reconcileNormal( matchingCluster = append(matchingCluster, clusterSetClusters...) - eventTriggerScope.SetMatchingClusterRefs(removeDuplicates(matchingCluster)) + matchingClusters := removeDuplicates(matchingCluster) + eventTriggerScope.SetMatchingClusterRefs(matchingClusters) + trackMatchingClusters(eventTriggerScope.EventTrigger.Name, len(matchingClusters), logger) err = r.updateClusterInfo(ctx, eventTriggerScope) if err != nil { diff --git a/controllers/eventtrigger_deployer.go b/controllers/eventtrigger_deployer.go index b8ad707..8fcfca7 100644 --- a/controllers/eventtrigger_deployer.go +++ b/controllers/eventtrigger_deployer.go @@ -161,6 +161,7 @@ func (r *EventTriggerReconciler) deployEventTrigger(ctx context.Context, eScope allProcessed = false } + r.trackClusterInfoOutcome(clusterInfo, resource.Name, l) } } } @@ -191,6 +192,27 @@ func (r *EventTriggerReconciler) deployEventTrigger(ctx context.Context, eScope return nil } +// trackClusterInfoOutcome records reconcile-outcome metrics for a terminal clusterInfo.Status. Non-terminal +// statuses (Provisioning, Removing) are intentionally not tracked. +func (r *EventTriggerReconciler) trackClusterInfoOutcome(clusterInfo *libsveltosv1beta1.ClusterInfo, + eventTriggerName string, logger logr.Logger) { + + clusterType := clusterproxy.GetClusterType(&clusterInfo.Cluster) + + switch clusterInfo.Status { + case libsveltosv1beta1.SveltosStatusProvisioned, libsveltosv1beta1.SveltosStatusRemoved: + trackReconcileOutcome(clusterInfo.Cluster.Namespace, clusterInfo.Cluster.Name, clusterType, + true, eventTriggerName, logger) + trackLastSuccess(clusterInfo.Cluster.Namespace, clusterInfo.Cluster.Name, clusterType, + time.Now(), eventTriggerName, logger) + case libsveltosv1beta1.SveltosStatusFailed, libsveltosv1beta1.SveltosStatusFailedNonRetriable: + trackReconcileOutcome(clusterInfo.Cluster.Namespace, clusterInfo.Cluster.Name, clusterType, + false, eventTriggerName, logger) + case libsveltosv1beta1.SveltosStatusProvisioning, libsveltosv1beta1.SveltosStatusRemoving: + // Non-terminal; nothing to track yet. + } +} + // undeployEventBasedAddon clean resources in managed clusters func (r *EventTriggerReconciler) undeployEventTrigger(ctx context.Context, eScope *scope.EventTriggerScope, clusterInfo []libsveltosv1beta1.ClusterInfo, logger logr.Logger) error { @@ -1412,6 +1434,15 @@ func updateClusterProfiles(ctx context.Context, c client.Client, clusterNamespac if eventTrigger.Spec.OneForEvent { logger.V(logs.LogDebug).Info("updating one clusterProfile per resource") + // Instantiate ConfigMap/Secrets from ConfigMapGenerator/SecretGenerator first: the + // ClusterProfile is created/updated only once the resources it may depend on are ready, + // so a generator failure does not leave behind a ClusterProfile for this reconcile pass. + fromGenerators, err = instantiateFromGeneratorsPerResource(ctx, c, eventTrigger, er, clusterNamespace, + clusterName, clusterType, logger) + if err != nil { + logger.V(logs.LogInfo).Error(err, "failed to instantiate from generators") + return err + } clusterProfiles, err = instantiateOneClusterProfilePerResource(ctx, c, clusterNamespace, clusterName, clusterType, eventTrigger, er, logger) if err != nil { @@ -1419,15 +1450,14 @@ func updateClusterProfiles(ctx context.Context, c client.Client, clusterNamespac "failed to create one clusterProfile instance per matching resource") return err } - // Instantiate ConfigMap/Secrets from ConfigMapGenerator/SecretGenerator - fromGenerators, err = instantiateFromGeneratorsPerResource(ctx, c, eventTrigger, er, clusterNamespace, + } else { + logger.V(logs.LogDebug).Info("updating one clusterProfile for all resources") + fromGenerators, err = instantiateFromGeneratorsPerAllResource(ctx, c, eventTrigger, er, clusterNamespace, clusterName, clusterType, logger) if err != nil { logger.V(logs.LogInfo).Error(err, "failed to instantiate from generators") return err } - } else { - logger.V(logs.LogDebug).Info("updating one clusterProfile for all resources") clusterProfiles, err = instantiateOneClusterProfilePerAllResource(ctx, c, clusterNamespace, clusterName, clusterType, eventTrigger, er, logger) if err != nil { @@ -1435,12 +1465,6 @@ func updateClusterProfiles(ctx context.Context, c client.Client, clusterNamespac "failed to create one clusterProfile instance per matching resource") return err } - fromGenerators, err = instantiateFromGeneratorsPerAllResource(ctx, c, eventTrigger, er, clusterNamespace, - clusterName, clusterType, logger) - if err != nil { - logger.V(logs.LogInfo).Error(err, "failed to instantiate from generators") - return err - } } // ClusterProfiles created because of CloudEvents are removed when CloudEventAction is set to Delete. diff --git a/controllers/metrics.go b/controllers/metrics.go index b08f6c6..b8e65cb 100644 --- a/controllers/metrics.go +++ b/controllers/metrics.go @@ -15,9 +15,7 @@ limitations under the License. package controllers import ( - "errors" "fmt" - "strings" "time" "github.com/go-logr/logr" @@ -30,73 +28,194 @@ import ( ) const ( - eventTriggerMetricName = "program_eventtrigger_time_seconds" - eventTriggerMetricHelp = "Program EventTrigger on a workload cluster duration distribution" + // metricNamespace prefixes every custom metric in this file, purely to avoid name collisions with + // unrelated tools scraped by the same Prometheus instance. Deliberately a fixed literal, not derived + // from getSveltosNamespace() (the Kubernetes namespace this component happens to be deployed into, + // which is user-configurable per install, and which is also set too late to be usable here: package-level + // var initializers run before main(), and SetSveltosNamespace() is only called from inside main()). + // A metric name prefix should stay stable across every install regardless. + metricNamespace = "projectsveltos" + + metricClusterNameLabel = "cluster_name" + metricClusterNamespaceLabel = "cluster_namespace" + metricClusterTypeLabel = "cluster_type" + metricStatusLabel = "status" + metricEventTriggerNameLabel = "event_trigger_name" + + statusSuccess = "success" + statusFailure = "failure" ) var ( - programEventTriggerDurationHistogram = prometheus.NewHistogram( + // reconcileDurationHistogram tracks how long it takes to process an EventTrigger for a cluster + // (evaluate matching events, create/update the resulting ClusterProfile(s)). Labeled by cluster so + // it can be filtered to a single cluster, aggregated to a namespace, or averaged fleet-wide. + // + // Not labeled by EventTrigger name: this is observed from programDuration, which is invoked as a + // libsveltos/lib/deployer.MetricHandler callback — a type shared with several other components + // (addon-controller, healthcheck-manager, classifier, etc.). That callback only carries cluster + // identity and featureID, not the EventTrigger that requested the work, so adding that label here + // would require changing the shared MetricHandler signature across all of them. reconcileOutcomeCounter + // and reconcileLastSuccessTimestampGauge below get the EventTrigger name instead, since both are + // recorded from event-manager-local code (deployEventTrigger) that already has it on hand. + // + // Replaces the previous programEventTriggerDurationHistogram/newEventTriggerHistogram pair, which had + // the same per-cluster metric-name cardinality bug fixed in addon-controller: baking cluster identity + // into the Prometheus metric Namespace (and thus its exported name) instead of using a label, producing + // one distinct, never-unregistered metric per cluster. + reconcileDurationHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Namespace: getSveltosNamespace(), - Name: eventTriggerMetricName, - Help: eventTriggerMetricHelp, - Buckets: []float64{0.1, 0.5, 1, 5, 10, 20, 30}, + Namespace: metricNamespace, + Name: "reconcile_duration_seconds", + Help: "Duration distribution of processing an EventTrigger for a cluster", + Buckets: []float64{0.1, 0.5, 1, 2, 5, 10, 20, 30, 60, 120}, + }, + []string{metricClusterTypeLabel, metricClusterNamespaceLabel, metricClusterNameLabel}, + ) + + // reconcileOutcomeCounter tracks terminal reconcile outcomes (success/failure) per EventTrigger and + // cluster. Incremented from deployEventTrigger, the one place that already knows whether processing + // a cluster ended up Provisioned/Removed (success) or Failed/FailedNonRetriable (failure). + reconcileOutcomeCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Name: "reconcile_outcome_total", + Help: "Total number of terminal reconcile outcomes for an EventTrigger on a cluster, by outcome", }, + []string{metricClusterTypeLabel, metricClusterNamespaceLabel, metricClusterNameLabel, metricStatusLabel, + metricEventTriggerNameLabel}, + ) + + // matchingClustersGauge tracks how many clusters currently match a given EventTrigger's + // sourceClusterSelector. Set from EventTriggerReconciler right after it computes the matching set, + // independent of any per-cluster reconcile activity. + matchingClustersGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricNamespace, + Name: "matching_clusters", + Help: "Number of clusters currently matching an EventTrigger's sourceClusterSelector", + }, + []string{metricEventTriggerNameLabel}, + ) + + // matchingResourcesGauge tracks how many resources the most recently processed EventReport matched + // for a given EventTrigger on a given cluster (er.Spec.MatchingResources — not CloudEvents, a + // separate matching mechanism on EventReport). Set unconditionally once an EventTrigger is confirmed + // to apply to the cluster, independent of whether ClusterProfile creation subsequently succeeds: + // this is a fact about the EventReport, not about what Sveltos did with it. + matchingResourcesGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricNamespace, + Name: "matching_resources", + Help: "Number of resources in the most recently processed EventReport matching an EventTrigger on a cluster", + }, + []string{metricEventTriggerNameLabel, metricClusterTypeLabel, metricClusterNamespaceLabel, metricClusterNameLabel}, + ) + + // reconcileLastSuccessTimestampGauge records when an EventTrigger last reached a successful terminal + // state (Provisioned/Removed) for a cluster. Only ever moves forward on success and is left untouched + // on failure, so it answers "how long has it been since this last worked" even for something that + // fails intermittently rather than in a tight consecutive streak. + // + // No equivalent "consecutive failures" gauge: unlike addon-controller's ClusterSummary, which already + // tracks FeatureSummary.ConsecutiveFailures, libsveltos's ClusterInfo (what EventTrigger.Status.ClusterInfo + // is built from) has no running failure-streak field to mirror. Adding one would mean adding new state + // to the CRD, not just exporting what's already computed — out of scope here. + reconcileLastSuccessTimestampGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricNamespace, + Name: "reconcile_last_success_timestamp_seconds", + Help: "Unix timestamp of the last successful (Provisioned/Removed) terminal outcome for an EventTrigger on a cluster", + }, + []string{metricClusterTypeLabel, metricClusterNamespaceLabel, metricClusterNameLabel, metricEventTriggerNameLabel}, ) ) //nolint:gochecknoinits // forced pattern, can't workaround func init() { // Register custom metrics with the global prometheus registry - metrics.Registry.MustRegister(programEventTriggerDurationHistogram) + metrics.Registry.MustRegister(reconcileDurationHistogram, reconcileOutcomeCounter, matchingClustersGauge, + reconcileLastSuccessTimestampGauge, matchingResourcesGauge) } -func newEventTriggerHistogram(clusterNamespace, clusterName string, clusterType libsveltosv1beta1.ClusterType, - logger logr.Logger) prometheus.Histogram { +func programDuration(elapsed time.Duration, clusterNamespace, clusterName, featureID string, + clusterType libsveltosv1beta1.ClusterType, logger logr.Logger) { - clusterInfo := strings.ReplaceAll(fmt.Sprintf("%s_%s_%s", clusterType, clusterNamespace, clusterName), "-", "_") - histogram := prometheus.NewHistogram( - prometheus.HistogramOpts{ - Namespace: clusterInfo, - Name: eventTriggerMetricName, - Help: eventTriggerMetricHelp, - Buckets: []float64{0.1, 0.5, 1, 5, 10, 20, 30}, - }, - ) + if featureID != string(v1beta1.FeatureEventTrigger) { + return + } - err := metrics.Registry.Register(histogram) - if err != nil { - var registrationError *prometheus.AlreadyRegisteredError - ok := errors.As(err, ®istrationError) - if ok { - _, ok = registrationError.ExistingCollector.(prometheus.Histogram) - if ok { - return registrationError.ExistingCollector.(prometheus.Histogram) - } - logCollectorError(err, logger) - return nil - } - logCollectorError(err, logger) - return nil + reconcileDurationHistogram.With(prometheus.Labels{ + metricClusterTypeLabel: string(clusterType), + metricClusterNamespaceLabel: clusterNamespace, + metricClusterNameLabel: clusterName, + }).Observe(elapsed.Seconds()) + + logger.V(logs.LogVerbose).Info(fmt.Sprintf("recorded duration for %s/%s %s: %s", + clusterNamespace, clusterName, featureID, elapsed)) +} + +// trackReconcileOutcome records a terminal reconcile outcome for an EventTrigger on a cluster. success is +// true for SveltosStatusProvisioned/SveltosStatusRemoved, false for SveltosStatusFailed/SveltosStatusFailedNonRetriable. +// Non-terminal statuses (Provisioning, Removing) must not call this. +func trackReconcileOutcome(clusterNamespace, clusterName string, clusterType libsveltosv1beta1.ClusterType, + success bool, eventTriggerName string, logger logr.Logger) { + + status := statusFailure + if success { + status = statusSuccess } - return histogram + reconcileOutcomeCounter.With(prometheus.Labels{ + metricClusterTypeLabel: string(clusterType), + metricClusterNamespaceLabel: clusterNamespace, + metricClusterNameLabel: clusterName, + metricStatusLabel: status, + metricEventTriggerNameLabel: eventTriggerName, + }).Inc() + + logger.V(logs.LogVerbose).Info(fmt.Sprintf("Tracking reconcile outcome for %s %s/%s eventTrigger %s: %s", + clusterType, clusterNamespace, clusterName, eventTriggerName, status)) } -func logCollectorError(err error, logger logr.Logger) { - logger.V(logs.LogDebug).Error(err, "failed to register collector") +// trackLastSuccess records the timestamp of the most recent successful (Provisioned/Removed) terminal +// outcome for an EventTrigger on a cluster. Only called from the success branch. +func trackLastSuccess(clusterNamespace, clusterName string, clusterType libsveltosv1beta1.ClusterType, + timestamp time.Time, eventTriggerName string, logger logr.Logger) { + + reconcileLastSuccessTimestampGauge.With(prometheus.Labels{ + metricClusterTypeLabel: string(clusterType), + metricClusterNamespaceLabel: clusterNamespace, + metricClusterNameLabel: clusterName, + metricEventTriggerNameLabel: eventTriggerName, + }).Set(float64(timestamp.Unix())) + + logger.V(logs.LogVerbose).Info(fmt.Sprintf("Tracking last success for %s %s/%s eventTrigger %s: %s", + clusterType, clusterNamespace, clusterName, eventTriggerName, timestamp)) } -func programDuration(elapsed time.Duration, clusterNamespace, clusterName, featureID string, - clusterType libsveltosv1beta1.ClusterType, logger logr.Logger) { +// trackMatchingClusters records how many clusters currently match an EventTrigger's sourceClusterSelector. +func trackMatchingClusters(eventTriggerName string, count int, logger logr.Logger) { + matchingClustersGauge.With(prometheus.Labels{ + metricEventTriggerNameLabel: eventTriggerName, + }).Set(float64(count)) - if featureID == string(v1beta1.FeatureEventTrigger) { - programEventTriggerDurationHistogram.Observe(elapsed.Seconds()) - clusterHistogram := newEventTriggerHistogram(clusterNamespace, clusterName, clusterType, logger) - if clusterHistogram != nil { - logger.V(logs.LogVerbose).Info(fmt.Sprintf("register data for %s/%s %s", - clusterNamespace, clusterName, featureID)) - clusterHistogram.Observe(elapsed.Seconds()) - } - } + logger.V(logs.LogVerbose).Info(fmt.Sprintf("Tracking matching clusters for eventTrigger %s: %d", + eventTriggerName, count)) +} + +// trackMatchingResources records how many resources the most recently processed EventReport matched +// for an EventTrigger on a cluster. +func trackMatchingResources(eventTriggerName, clusterNamespace, clusterName string, + clusterType libsveltosv1beta1.ClusterType, count int, logger logr.Logger) { + + matchingResourcesGauge.With(prometheus.Labels{ + metricEventTriggerNameLabel: eventTriggerName, + metricClusterTypeLabel: string(clusterType), + metricClusterNamespaceLabel: clusterNamespace, + metricClusterNameLabel: clusterName, + }).Set(float64(count)) + + logger.V(logs.LogVerbose).Info(fmt.Sprintf("Tracking matching resources for eventTrigger %s %s %s/%s: %d", + eventTriggerName, clusterType, clusterNamespace, clusterName, count)) } diff --git a/manifest/manifest.yaml b/manifest/manifest.yaml index 5f9a921..7fd98c4 100644 --- a/manifest/manifest.yaml +++ b/manifest/manifest.yaml @@ -2571,6 +2571,34 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: event-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: event-proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: event-manager-rolebinding @@ -2583,6 +2611,35 @@ subjects: name: event-manager namespace: projectsveltos --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: event-proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: event-proxy-role +subjects: +- kind: ServiceAccount + name: event-manager + namespace: projectsveltos +--- +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: event-manager + name: event-manager-metrics-service + namespace: projectsveltos +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + selector: + control-plane: event-manager +--- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/manifest/service_monitor.yaml b/manifest/service_monitor.yaml new file mode 100644 index 0000000..dc7792b --- /dev/null +++ b/manifest/service_monitor.yaml @@ -0,0 +1,18 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: event-manager + name: manager-metrics-monitor + namespace: projectsveltos +spec: + endpoints: + - path: /metrics + port: metrics + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + insecureSkipVerify: true + selector: + matchLabels: + control-plane: event-manager diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..a85e67e --- /dev/null +++ b/renovate.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": ["config:recommended", ":dependencyDashboard"], + "prHourlyLimit": 0, + "prConcurrentLimit": 0, + "branchConcurrentLimit": 0, + "ignorePaths": ["**/test/**"], + "sveltos": { + "fileMatch": ["^.*\\.yaml$"] + } +}