diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8b89e26d19f6..6776d99e422b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -439,6 +439,7 @@ /comp/haagent @DataDog/network-device-monitoring-core /comp/languagedetection/client @DataDog/container-platform /comp/logonduration @DataDog/windows-products +/comp/metriclookback @DataDog/q-branch /comp/networkconfigmanagement @DataDog/ndm-integrations /comp/notableevents @DataDog/windows-products /comp/offlinereporter @DataDog/agent-metric-pipelines @@ -486,6 +487,8 @@ # pkg /pkg/ @DataDog/agent-runtimes +/pkg/metriclookback/ @DataDog/q-branch +/pkg/metriclookback/dogstatsd/ @DataDog/q-branch @DataDog/agent-metric-pipelines /pkg/api/ @DataDog/agent-runtimes /pkg/api/security/cert/cert_getter_dca.go @DataDog/agent-runtimes @DataDog/container-platform /pkg/aggregator/ @DataDog/agent-metric-pipelines diff --git a/cmd/agent/subcommands/run/command.go b/cmd/agent/subcommands/run/command.go index 93c113eb0425..484b4f1f924d 100644 --- a/cmd/agent/subcommands/run/command.go +++ b/cmd/agent/subcommands/run/command.go @@ -151,6 +151,7 @@ import ( runner "github.com/DataDog/datadog-agent/comp/metadata/runner/def" securityagentmetadata "github.com/DataDog/datadog-agent/comp/metadata/securityagent/def" systemprobemetadata "github.com/DataDog/datadog-agent/comp/metadata/systemprobe/def" + metriclookbackdef "github.com/DataDog/datadog-agent/comp/metriclookback/def" "github.com/DataDog/datadog-agent/comp/ndmtmp" "github.com/DataDog/datadog-agent/comp/netflow" netflowServer "github.com/DataDog/datadog-agent/comp/netflow/server/def" @@ -285,6 +286,7 @@ func run(log log.Component, rcclient rcclient.Component, _ runner.Component, demultiplexer demultiplexer.Component, + metricLookback metriclookbackdef.Component, _ serializer.MetricSerializer, _ option.Option[logsAgent.Component], _ statsd.Component, @@ -371,6 +373,7 @@ func run(log log.Component, ac, rcclient, demultiplexer, + metricLookback, invChecks, logReceiver, collector, @@ -469,6 +472,7 @@ func getSharedFxOption() fx.Option { grpcAgentfx.Module(), commonendpoints.Module(), filterlist.Module(), + metriclookbackModule(), demultiplexerimpl.Module(demultiplexerimpl.NewDefaultParams(demultiplexerimpl.WithDogstatsdNoAggregationPipelineConfig())), demultiplexerendpointfx.Module(), dogstatsd.Bundle(dogstatsdServer.Params{Serverless: false}), @@ -639,6 +643,7 @@ func startAgent( ac autodiscovery.Component, rcclient rcclient.Component, demultiplexer demultiplexer.Component, + metricLookback metriclookbackdef.Component, invChecks inventorychecks.Component, logReceiver option.Option[integrations.Component], collectorComponent collector.Component, @@ -742,7 +747,9 @@ func startAgent( // Set up check collector commonchecks.RegisterChecks(wmeta, filterStore, tagger, cfg, tlm, rcclient, flare, snmpScanManager, traceroute, ncmComp) - ac.AddScheduler("check", pkgcollector.InitCheckScheduler(option.New(collectorComponent), demultiplexer, logReceiver, tagger, filterStore), true) + checkScheduler := pkgcollector.InitCheckScheduler(option.New(collectorComponent), demultiplexer, logReceiver, tagger, filterStore) + checkScheduler.SetMetricLookbackShadowSenderManager(metricLookback.NewSenderManager(ctx, hostnameDetected)) + ac.AddScheduler("check", checkScheduler, true) demultiplexer.AddAgentStartupTelemetry(version.AgentVersion) diff --git a/cmd/agent/subcommands/run/command_windows.go b/cmd/agent/subcommands/run/command_windows.go index 707765ee361f..0cf67e08ba4f 100644 --- a/cmd/agent/subcommands/run/command_windows.go +++ b/cmd/agent/subcommands/run/command_windows.go @@ -76,6 +76,7 @@ import ( inventoryhost "github.com/DataDog/datadog-agent/comp/metadata/inventoryhost/def" packagesigning "github.com/DataDog/datadog-agent/comp/metadata/packagesigning/def" runner "github.com/DataDog/datadog-agent/comp/metadata/runner/def" + metriclookbackdef "github.com/DataDog/datadog-agent/comp/metriclookback/def" netflowServer "github.com/DataDog/datadog-agent/comp/netflow/server/def" otelcollector "github.com/DataDog/datadog-agent/comp/otelcol/collector/def" processAgent "github.com/DataDog/datadog-agent/comp/process/agent/def" @@ -124,6 +125,7 @@ func StartAgentWithDefaults(ctxChan <-chan context.Context) (<-chan error, error _ serializer.MetricSerializer, _ otelcollector.Component, demultiplexer demultiplexer.Component, + metricLookback metriclookbackdef.Component, _ host.Component, _ inventoryagent.Component, _ inventoryhost.Component, @@ -165,6 +167,7 @@ func StartAgentWithDefaults(ctxChan <-chan context.Context) (<-chan error, error ac, rcclient, demultiplexer, + metricLookback, invChecks, logsReceiver, collector, diff --git a/cmd/agent/subcommands/run/metriclookback.go b/cmd/agent/subcommands/run/metriclookback.go new file mode 100644 index 000000000000..a93caa25ef33 --- /dev/null +++ b/cmd/agent/subcommands/run/metriclookback.go @@ -0,0 +1,18 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build python || test + +package run + +import ( + "go.uber.org/fx" + + metriclookbackfx "github.com/DataDog/datadog-agent/comp/metriclookback/fx" +) + +func metriclookbackModule() fx.Option { + return metriclookbackfx.Module() +} diff --git a/cmd/agent/subcommands/run/metriclookback_nopython.go b/cmd/agent/subcommands/run/metriclookback_nopython.go new file mode 100644 index 000000000000..3f2c660e39ec --- /dev/null +++ b/cmd/agent/subcommands/run/metriclookback_nopython.go @@ -0,0 +1,31 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build !python && !test + +package run + +import ( + "context" + + "go.uber.org/fx" + + metriclookbackdef "github.com/DataDog/datadog-agent/comp/metriclookback/def" + "github.com/DataDog/datadog-agent/pkg/aggregator/sender" +) + +type unavailableMetricLookback struct{} + +func (unavailableMetricLookback) NewSenderManager(context.Context, string) sender.SenderManager { + return nil +} + +func metriclookbackModule() fx.Option { + // Keep concrete metric lookback retention out of non-Python Agent binaries, + // while satisfying shared Agent startup wiring with an unavailable component. + return fx.Provide(func() metriclookbackdef.Component { + return unavailableMetricLookback{} + }) +} diff --git a/comp/README.md b/comp/README.md index 5e8307cd5d28..6cdf5cedd069 100644 --- a/comp/README.md +++ b/comp/README.md @@ -891,6 +891,12 @@ Package client implements a component to send process metadata to the Cluster-Ag Package logonduration provides a component that monitors the duration of a user logon after boot and forwards them to the Datadog Event Management v2 API. +### [comp/metriclookback](https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/metriclookback) + +*Datadog Team*: q-branch + +Package metriclookback defines the metric lookback component. + ### [comp/networkconfigmanagement](https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/networkconfigmanagement) *Datadog Team*: ndm-integrations diff --git a/comp/aggregator/demultiplexer/impl/BUILD.bazel b/comp/aggregator/demultiplexer/impl/BUILD.bazel index 1bb48b72c2ef..06bf9c274475 100644 --- a/comp/aggregator/demultiplexer/impl/BUILD.bazel +++ b/comp/aggregator/demultiplexer/impl/BUILD.bazel @@ -90,7 +90,9 @@ dd_agent_go_test( "//comp/serializer/logscompression/fx-mock", "//comp/serializer/metricscompression/def", "//comp/serializer/metricscompression/fx-mock", + "//pkg/aggregator", "//pkg/aggregator/mocksender", + "//pkg/serializer", "//pkg/util/fxutil", "@com_github_stretchr_testify//require", "@org_uber_go_fx//:fx", diff --git a/comp/aggregator/demultiplexer/impl/demultiplexer.go b/comp/aggregator/demultiplexer/impl/demultiplexer.go index ea18e20d307e..a4b6aaa573a0 100644 --- a/comp/aggregator/demultiplexer/impl/demultiplexer.go +++ b/comp/aggregator/demultiplexer/impl/demultiplexer.go @@ -40,18 +40,19 @@ func Module(params Params) fxutil.Module { // Dependencies defines the dependencies required by the demultiplexer component. type Dependencies struct { compdef.In - Lc compdef.Lifecycle - Config config.Component - Log log.Component - SharedForwarder defaultforwarder.Component - OrchestratorForwarder orchestratorforwarder.Component - EventPlatformForwarder eventplatform.Component - HaAgent haagent.Component - Compressor compression.Component - Tagger tagger.Component - Hostname hostnameinterface.Component - FilterList filterlist.Component - Observer observer.Component `optional:"true"` + Lc compdef.Lifecycle + Config config.Component + Log log.Component + SharedForwarder defaultforwarder.Component + OrchestratorForwarder orchestratorforwarder.Component + EventPlatformForwarder eventplatform.Component + HaAgent haagent.Component + Compressor compression.Component + Tagger tagger.Component + Hostname hostnameinterface.Component + FilterList filterlist.Component + Observer observer.Component `optional:"true"` + DogStatsDLookbackFactory aggregator.DogStatsDLookbackFactory `optional:"true"` Params Params } @@ -81,7 +82,7 @@ func NewComponent(deps Dependencies) (Provides, error) { return Provides{}, deps.Log.Errorf("Error while getting hostname, exiting: %v", err) } } - options := createAgentDemultiplexerOptions(deps.Config, deps.Params) + options := createAgentDemultiplexerOptions(deps.Config, deps.Params, deps.DogStatsDLookbackFactory) agentDemultiplexer := aggregator.InitAndStartAgentDemultiplexer( deps.Log, deps.SharedForwarder, @@ -113,7 +114,7 @@ func NewComponent(deps Dependencies) (Provides, error) { }, nil } -func createAgentDemultiplexerOptions(config config.Component, params Params) aggregator.AgentDemultiplexerOptions { +func createAgentDemultiplexerOptions(config config.Component, params Params, dogStatsDLookbackFactory aggregator.DogStatsDLookbackFactory) aggregator.AgentDemultiplexerOptions { options := aggregator.DefaultAgentDemultiplexerOptions() if params.useDogstatsdNoAggregationPipelineConfig { if config.GetBool("dogstatsd_no_aggregation_pipeline") { @@ -124,6 +125,8 @@ func createAgentDemultiplexerOptions(config config.Component, params Params) agg } } + options.DogStatsDLookbackFactory = dogStatsDLookbackFactory + // Override FlushInterval only if flushInterval is set by the user if v, ok := params.flushInterval.Get(); ok { options.FlushInterval = v diff --git a/comp/aggregator/demultiplexer/impl/demultiplexer_options_test.go b/comp/aggregator/demultiplexer/impl/demultiplexer_options_test.go index d3564eb3ae57..3539e1bef435 100644 --- a/comp/aggregator/demultiplexer/impl/demultiplexer_options_test.go +++ b/comp/aggregator/demultiplexer/impl/demultiplexer_options_test.go @@ -12,6 +12,8 @@ import ( "github.com/stretchr/testify/require" configmock "github.com/DataDog/datadog-agent/comp/core/config" + "github.com/DataDog/datadog-agent/pkg/aggregator" + "github.com/DataDog/datadog-agent/pkg/serializer" ) func TestCreateAgentDemultiplexerOptionsNoAggWorkerCountNotReadWithoutConfigOption(t *testing.T) { @@ -20,7 +22,7 @@ func TestCreateAgentDemultiplexerOptionsNoAggWorkerCountNotReadWithoutConfigOpti "dogstatsd_no_aggregation_pipeline_workers_count": 4, }) - options := createAgentDemultiplexerOptions(cfg, NewDefaultParams()) + options := createAgentDemultiplexerOptions(cfg, NewDefaultParams(), nil) require.Equal(t, 0, options.NoAggregationPipelineWorkersCount) } @@ -31,7 +33,7 @@ func TestCreateAgentDemultiplexerOptionsNoAggWorkerCountFromConfig(t *testing.T) "dogstatsd_no_aggregation_pipeline_workers_count": 4, }) - options := createAgentDemultiplexerOptions(cfg, NewDefaultParams(WithDogstatsdNoAggregationPipelineConfig())) + options := createAgentDemultiplexerOptions(cfg, NewDefaultParams(WithDogstatsdNoAggregationPipelineConfig()), nil) require.Equal(t, 4, options.NoAggregationPipelineWorkersCount) } @@ -41,7 +43,7 @@ func TestCreateAgentDemultiplexerOptionsNoAggWorkerCountDefaultsToOneWhenEnabled "dogstatsd_no_aggregation_pipeline": true, }) - options := createAgentDemultiplexerOptions(cfg, NewDefaultParams(WithDogstatsdNoAggregationPipelineConfig())) + options := createAgentDemultiplexerOptions(cfg, NewDefaultParams(WithDogstatsdNoAggregationPipelineConfig()), nil) require.Equal(t, 1, options.NoAggregationPipelineWorkersCount) } @@ -52,7 +54,7 @@ func TestCreateAgentDemultiplexerOptionsNoAggWorkerCountDisabled(t *testing.T) { "dogstatsd_no_aggregation_pipeline_workers_count": 4, }) - options := createAgentDemultiplexerOptions(cfg, NewDefaultParams(WithDogstatsdNoAggregationPipelineConfig())) + options := createAgentDemultiplexerOptions(cfg, NewDefaultParams(WithDogstatsdNoAggregationPipelineConfig()), nil) require.Equal(t, 0, options.NoAggregationPipelineWorkersCount) } @@ -65,9 +67,20 @@ func TestCreateAgentDemultiplexerOptionsNoAggWorkerCountFallsBackToOne(t *testin "dogstatsd_no_aggregation_pipeline_workers_count": configured, }) - options := createAgentDemultiplexerOptions(cfg, NewDefaultParams(WithDogstatsdNoAggregationPipelineConfig())) + options := createAgentDemultiplexerOptions(cfg, NewDefaultParams(WithDogstatsdNoAggregationPipelineConfig()), nil) require.Equal(t, 1, options.NoAggregationPipelineWorkersCount) }) } } + +func TestCreateAgentDemultiplexerOptionsStoresLookbackFactory(t *testing.T) { + cfg := configmock.NewMock(t) + factory := aggregator.DogStatsDLookbackFactory(func(serializer.MetricSerializer) aggregator.DogStatsDLookback { + return nil + }) + + options := createAgentDemultiplexerOptions(cfg, NewDefaultParams(), factory) + + require.NotNil(t, options.DogStatsDLookbackFactory) +} diff --git a/comp/metriclookback/BUILD.bazel b/comp/metriclookback/BUILD.bazel new file mode 100644 index 000000000000..9968c4af55f8 --- /dev/null +++ b/comp/metriclookback/BUILD.bazel @@ -0,0 +1 @@ +# gazelle:ignore diff --git a/comp/metriclookback/def/BUILD.bazel b/comp/metriclookback/def/BUILD.bazel new file mode 100644 index 000000000000..a35773901ac4 --- /dev/null +++ b/comp/metriclookback/def/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "def", + srcs = ["component.go"], + importpath = "github.com/DataDog/datadog-agent/comp/metriclookback/def", + visibility = ["//visibility:public"], + deps = ["//pkg/aggregator/sender"], +) diff --git a/comp/metriclookback/def/component.go b/comp/metriclookback/def/component.go new file mode 100644 index 000000000000..686cd3aa762a --- /dev/null +++ b/comp/metriclookback/def/component.go @@ -0,0 +1,23 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package metriclookback defines the metric lookback component. +package metriclookback + +import ( + "context" + + "github.com/DataDog/datadog-agent/pkg/aggregator/sender" +) + +// team: q-branch + +// Component is the metric lookback component. +type Component interface { + // NewSenderManager returns the sender manager used exclusively by metric + // lookback shadow checks. It returns nil when lookback is unavailable in the + // current Agent build. + NewSenderManager(context.Context, string) sender.SenderManager +} diff --git a/comp/metriclookback/fx/BUILD.bazel b/comp/metriclookback/fx/BUILD.bazel new file mode 100644 index 000000000000..675335634d04 --- /dev/null +++ b/comp/metriclookback/fx/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "fx", + srcs = ["fx.go"], + importpath = "github.com/DataDog/datadog-agent/comp/metriclookback/fx", + visibility = ["//visibility:public"], + deps = [ + "//comp/metriclookback/impl", + "//pkg/util/fxutil", + ], +) diff --git a/comp/metriclookback/fx/fx.go b/comp/metriclookback/fx/fx.go new file mode 100644 index 000000000000..5567fa071061 --- /dev/null +++ b/comp/metriclookback/fx/fx.go @@ -0,0 +1,19 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package fx provides the fx module for the metric lookback component. +package fx + +import ( + metriclookbackimpl "github.com/DataDog/datadog-agent/comp/metriclookback/impl" + "github.com/DataDog/datadog-agent/pkg/util/fxutil" +) + +// Module defines the fx options for this component. +func Module() fxutil.Module { + return fxutil.Component( + fxutil.ProvideComponentConstructor(metriclookbackimpl.NewComponent), + ) +} diff --git a/comp/metriclookback/impl/BUILD.bazel b/comp/metriclookback/impl/BUILD.bazel new file mode 100644 index 000000000000..86da713de21f --- /dev/null +++ b/comp/metriclookback/impl/BUILD.bazel @@ -0,0 +1,43 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "impl", + srcs = ["metriclookback.go"], + importpath = "github.com/DataDog/datadog-agent/comp/metriclookback/impl", + visibility = ["//visibility:public"], + deps = [ + "//comp/core/config", + "//comp/core/log/def", + "//comp/metriclookback/def", + "//pkg/aggregator", + "//pkg/aggregator/sender", + "//pkg/collector/metriclookback", + "//pkg/metriclookback", + "//pkg/metriclookback/dogstatsd", + "//pkg/metriclookback/monitor", + "//pkg/metriclookback/ringbuffer", + "//pkg/serializer", + ], +) + +go_test( + name = "impl_test", + srcs = ["metriclookback_test.go"], + embed = [":impl"], + gotags = ["test"], + deps = [ + "//comp/core/config", + "//comp/core/log/mock", + "//pkg/aggregator", + "//pkg/aggregator/ckey", + "//pkg/collector/check/id", + "//pkg/collector/metriclookback", + "//pkg/metriclookback", + "//pkg/metriclookback/dogstatsd", + "//pkg/metrics", + "//pkg/serializer/mocks", + "//pkg/tagset", + "@com_github_stretchr_testify//mock", + "@com_github_stretchr_testify//require", + ], +) diff --git a/comp/metriclookback/impl/metriclookback.go b/comp/metriclookback/impl/metriclookback.go new file mode 100644 index 000000000000..e2c2675bf139 --- /dev/null +++ b/comp/metriclookback/impl/metriclookback.go @@ -0,0 +1,264 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package metriclookbackimpl implements metric lookback wiring for the Agent. +package metriclookbackimpl + +import ( + "context" + "fmt" + "sort" + "time" + + "github.com/DataDog/datadog-agent/comp/core/config" + log "github.com/DataDog/datadog-agent/comp/core/log/def" + metriclookbackdef "github.com/DataDog/datadog-agent/comp/metriclookback/def" + "github.com/DataDog/datadog-agent/pkg/aggregator" + "github.com/DataDog/datadog-agent/pkg/aggregator/sender" + collectormetriclookback "github.com/DataDog/datadog-agent/pkg/collector/metriclookback" + "github.com/DataDog/datadog-agent/pkg/metriclookback" + metriclookbackdogstatsd "github.com/DataDog/datadog-agent/pkg/metriclookback/dogstatsd" + "github.com/DataDog/datadog-agent/pkg/metriclookback/monitor" + "github.com/DataDog/datadog-agent/pkg/metriclookback/ringbuffer" + "github.com/DataDog/datadog-agent/pkg/serializer" +) + +// Requires defines the metric lookback component dependencies. +type Requires struct { + Config config.Component + Log log.Component +} + +// Provides defines the values provided by the metric lookback component. +type Provides struct { + Comp metriclookbackdef.Component + DogStatsDLookbackFactory aggregator.DogStatsDLookbackFactory +} + +type component struct { + retention *metriclookback.Retention +} + +// NewSenderManager returns a shadow-check sender manager backed by the shared +// metric lookback retention ring. +func (c component) NewSenderManager(ctx context.Context, defaultHostname string) sender.SenderManager { + return collectormetriclookback.NewSenderManager(ctx, defaultHostname, c.retention) +} + +// NewComponent creates the metric lookback component. +func NewComponent(req Requires) (Provides, error) { + retention := newMetricLookbackRetention(req.Config) + factory, err := newMetricLookbackDogStatsDFactory(req.Config, req.Log, retention) + if err != nil { + return Provides{}, err + } + return Provides{Comp: component{retention: retention}, DogStatsDLookbackFactory: factory}, nil +} + +const ( + metricLookbackMonitorModeDisabled = "disabled" + metricLookbackMonitorModeDryRun = "dry_run" + metricLookbackMonitorModeEnabled = "enabled" +) + +func newMetricLookbackRetention(cfg config.Component) *metriclookback.Retention { + // Construct the retention owner eagerly, but keep the backing rings lazy. This + // lets per-instance shadow-check opt-in share the same retention path without + // reserving ring memory when no selected metric is written. + return metriclookback.NewRetention(ringbuffer.Options{ + Capacity: cfg.GetInt("metric_lookback.capacity"), + ShardCount: cfg.GetInt("metric_lookback.shard_count"), + }) +} + +func newMetricLookbackDogStatsDFactory(cfg config.Component, logger log.Component, retention *metriclookback.Retention) (aggregator.DogStatsDLookbackFactory, error) { + if err := validateMetricLookbackMonitorConfig(cfg); err != nil { + return nil, err + } + + return func(metricSerializer serializer.MetricSerializer) aggregator.DogStatsDLookback { + if retention == nil || !cfg.GetBool("metric_lookback.enabled") { + return nil + } + + metricNames := cfg.GetStringSlice("metric_lookback.dogstatsd.metric_names") + monitorMode := metricLookbackMonitorMode(cfg) + monitorEnabled := monitorMode != metricLookbackMonitorModeDisabled + if len(metricNames) == 0 && !monitorEnabled { + return nil + } + + var egressController *metriclookback.EgressController + if monitorEnabled { + egressController = metriclookback.NewEgressController(retention, metricSerializer, metriclookback.EgressControllerOptions{ + PreTriggerWindow: cfg.GetDuration("metric_lookback.egress.pre_trigger_window"), + PostRecoveryWindow: cfg.GetDuration("metric_lookback.egress.post_recovery_window"), + DryRun: monitorMode == metricLookbackMonitorModeDryRun, + MonitorStateTransitionLogger: metricLookbackMonitorStateTransitionLogger(logger), + }) + } + watcher, err := newMetricLookbackMonitor(cfg, logger, retention, egressController) + if err != nil { + logger.Errorf("invalid metric_lookback monitor configuration: %v", err) + return nil + } + retention.SetMonitor(watcher) + materializer := metriclookbackdogstatsd.NewDogStatsDBucketMaterializer(retention, metriclookbackdogstatsd.DogStatsDBucketMaterializerOptions{ + Monitor: watcher, + }) + + adapter := metriclookbackdogstatsd.NewDogStatsDAdapter(retention, metriclookbackdogstatsd.DogStatsDOptions{ + MetricNames: metricNames, + Monitor: watcher, + BucketMaterializer: materializer, + EgressController: egressController, + }) + if adapter == nil { + logger.Warn("metric_lookback is enabled but no DogStatsD metric names or monitor metric are configured; lookback inactive") + return nil + } + if egressController != nil && watcher != nil { + egressController.Start() + } + return adapter + }, nil +} + +func metricLookbackMonitorStateTransitionLogger(logger log.Component) metriclookback.MonitorStateTransitionLogger { + return func(transition metriclookback.MonitorStateTransition) { + if logger == nil { + return + } + decision := transition.Decision + if transition.Initial { + logger.Infof("metric_lookback monitor state initialized: metric_name=%q state=%s dry_run=%t egress_mode=%s window_from=%s window_to=%s point_count=%d min=%v max=%v range=%v range_epsilon=%v partition_tags=%v partition_key=%q partition_count=%d", + transition.MetricName, + transition.To.String(), + transition.DryRun, + transition.EgressMode.String(), + decision.WindowFrom.Format(time.RFC3339Nano), + decision.WindowTo.Format(time.RFC3339Nano), + decision.PointCount, + decision.Min, + decision.Max, + decision.Range, + decision.RangeEpsilon, + decision.PartitionTags, + decision.PartitionKey, + decision.PartitionCount, + ) + return + } + logger.Infof("metric_lookback monitor state transition: metric_name=%q from=%s to=%s dry_run=%t egress_mode=%s window_from=%s window_to=%s point_count=%d min=%v max=%v range=%v range_epsilon=%v partition_tags=%v partition_key=%q partition_count=%d", + transition.MetricName, + transition.From.String(), + transition.To.String(), + transition.DryRun, + transition.EgressMode.String(), + decision.WindowFrom.Format(time.RFC3339Nano), + decision.WindowTo.Format(time.RFC3339Nano), + decision.PointCount, + decision.Min, + decision.Max, + decision.Range, + decision.RangeEpsilon, + decision.PartitionTags, + decision.PartitionKey, + decision.PartitionCount, + ) + } +} + +func metricLookbackMonitorMode(cfg config.Component) string { + if cfg == nil { + return metricLookbackMonitorModeDisabled + } + mode := cfg.GetString("metric_lookback.monitor.mode") + if mode == "" { + return metricLookbackMonitorModeDisabled + } + return mode +} + +func validateMetricLookbackMonitorConfig(cfg config.Component) error { + if cfg == nil || !cfg.GetBool("metric_lookback.enabled") { + return nil + } + mode := metricLookbackMonitorMode(cfg) + switch mode { + case metricLookbackMonitorModeDisabled: + return nil + case metricLookbackMonitorModeDryRun, metricLookbackMonitorModeEnabled: + if cfg.GetString("metric_lookback.monitor.metric_name") == "" { + return fmt.Errorf("metric_lookback.monitor.metric_name is required when metric_lookback.monitor.mode is %q", mode) + } + if rangeEpsilon := cfg.GetFloat64("metric_lookback.monitor.range_epsilon"); rangeEpsilon < 0 { + return fmt.Errorf("metric_lookback.monitor.range_epsilon must be non-negative, got %v", rangeEpsilon) + } + if evaluationInterval := cfg.GetDuration("metric_lookback.monitor.evaluation_interval"); evaluationInterval < 0 { + return fmt.Errorf("metric_lookback.monitor.evaluation_interval must be non-negative, got %v", evaluationInterval) + } + if preTriggerWindow := cfg.GetDuration("metric_lookback.egress.pre_trigger_window"); preTriggerWindow < 0 { + return fmt.Errorf("metric_lookback.egress.pre_trigger_window must be non-negative, got %v", preTriggerWindow) + } + if postRecoveryWindow := cfg.GetDuration("metric_lookback.egress.post_recovery_window"); postRecoveryWindow < 0 { + return fmt.Errorf("metric_lookback.egress.post_recovery_window must be non-negative, got %v", postRecoveryWindow) + } + return nil + default: + return fmt.Errorf("metric_lookback.monitor.mode must be one of %q, %q, or %q; got %q", metricLookbackMonitorModeDisabled, metricLookbackMonitorModeDryRun, metricLookbackMonitorModeEnabled, mode) + } +} + +func newMetricLookbackMonitor(cfg config.Component, logger log.Component, retention *metriclookback.Retention, sink monitor.DecisionSink) (*monitor.Watcher, error) { + if metricLookbackMonitorMode(cfg) == metricLookbackMonitorModeDisabled { + return nil, nil + } + if sink == nil { + logger.Warn("metric_lookback.monitor.mode is active but egress controller is not available; monitor inactive") + return nil, nil + } + + metricName := cfg.GetString("metric_lookback.monitor.metric_name") + monitorSources := []ringbuffer.Source{ + {Kind: ringbuffer.SourceDogStatsDBucketed}, + {Kind: ringbuffer.SourceDogStatsDNoAggregation}, + {Kind: ringbuffer.SourceCheckShadow}, + } + reader := monitor.PointReaderFunc(func(metricName string, from, to time.Time) []monitor.Point { + points := retention.PointsBetweenSources(monitorSources, metricName, from, to) + // PlaceholderAverageSketchProjection is intentionally isolated from retention + // and egress serialization. It only lets the current scalar monitor evaluate + // selected distribution sketches until a sketch-aware monitor design is chosen. + sketchPoints := retention.ProjectedSketchPointsBetweenSources(monitorSources, metricName, from, to, metriclookback.PlaceholderAverageSketchProjection{}) + out := make([]monitor.Point, 0, len(points)+len(sketchPoints)) + for _, point := range points { + out = append(out, monitor.Point{Ts: point.Ts, Value: point.Value, Tags: point.Tags}) + } + for _, point := range sketchPoints { + out = append(out, monitor.Point{Ts: point.Ts, Value: point.Value, Tags: point.Tags}) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Ts.Equal(out[j].Ts) { + return out[i].Value < out[j].Value + } + return out[i].Ts.Before(out[j].Ts) + }) + return out + }) + watcher, err := monitor.New(monitor.Config{ + MetricName: metricName, + RangeEpsilon: cfg.GetFloat64("metric_lookback.monitor.range_epsilon"), + PartitionTags: cfg.GetStringSlice("metric_lookback.monitor.partition_tags"), + EvaluationInterval: cfg.GetDuration("metric_lookback.monitor.evaluation_interval"), + }, reader, sink) + if err != nil { + return nil, err + } + if watcher == nil { + logger.Warn("metric_lookback.monitor.mode is active but metric_lookback.monitor.metric_name is empty; monitor inactive") + } + return watcher, nil +} diff --git a/comp/metriclookback/impl/metriclookback_test.go b/comp/metriclookback/impl/metriclookback_test.go new file mode 100644 index 000000000000..ae16f79087b9 --- /dev/null +++ b/comp/metriclookback/impl/metriclookback_test.go @@ -0,0 +1,432 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build test + +package metriclookbackimpl + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + configmock "github.com/DataDog/datadog-agent/comp/core/config" + logmock "github.com/DataDog/datadog-agent/comp/core/log/mock" + "github.com/DataDog/datadog-agent/pkg/aggregator" + "github.com/DataDog/datadog-agent/pkg/aggregator/ckey" + checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" + collectormetriclookback "github.com/DataDog/datadog-agent/pkg/collector/metriclookback" + "github.com/DataDog/datadog-agent/pkg/metriclookback" + metriclookbackdogstatsd "github.com/DataDog/datadog-agent/pkg/metriclookback/dogstatsd" + "github.com/DataDog/datadog-agent/pkg/metrics" + serializermocks "github.com/DataDog/datadog-agent/pkg/serializer/mocks" + "github.com/DataDog/datadog-agent/pkg/tagset" +) + +func TestMetricLookbackDogStatsDFactoryDisabled(t *testing.T) { + cfg := configmock.NewMockWithOverrides(t, map[string]interface{}{ + "metric_lookback.enabled": false, + }) + factory := requireMetricLookbackDogStatsDFactory(t, cfg, newMetricLookbackRetention(cfg)) + + lookback := factory(serializermocks.NewMetricSerializer(t)) + + require.Nil(t, lookback) +} + +func TestMetricLookbackComponentProvidesDogStatsDFactory(t *testing.T) { + cfg := configmock.NewMockWithOverrides(t, map[string]interface{}{ + "metric_lookback.enabled": true, + "metric_lookback.capacity": 256, + "metric_lookback.shard_count": 1, + "metric_lookback.dogstatsd.metric_names": []string{"target.metric"}, + "metric_lookback.monitor.mode": "disabled", + }) + + provides, err := NewComponent(Requires{Config: cfg, Log: logmock.New(t)}) + + require.NoError(t, err) + require.NotNil(t, provides.Comp) + require.NotNil(t, provides.DogStatsDLookbackFactory) + + shadowManager := provides.Comp.NewSenderManager(context.Background(), "default-host") + require.NotNil(t, shadowManager) + shadowSender, err := shadowManager.GetSender(checkid.ID("gpu:shadow")) + require.NoError(t, err) + shadowSender.Gauge("shadow.metric", 42, "", nil) + shadowSender.Commit() + retained := provides.Comp.(component).retention.Series() + require.Len(t, retained, 1) + require.Equal(t, "shadow.metric", retained[0].Name) + require.Equal(t, "default-host", retained[0].Host) + + lookback := provides.DogStatsDLookbackFactory(serializermocks.NewMetricSerializer(t)) + require.NotNil(t, lookback) + stopLookbackOnCleanup(t, lookback) + require.True(t, lookback.WantsDogStatsDMetric("target.metric")) +} + +func TestMetricLookbackDogStatsDFactoryMetricNamesEnableDogStatsDLookback(t *testing.T) { + cfg := configmock.NewMockWithOverrides(t, map[string]interface{}{ + "metric_lookback.enabled": true, + "metric_lookback.capacity": 256, + "metric_lookback.shard_count": 1, + "metric_lookback.dogstatsd.metric_names": []string{"target.metric"}, + "metric_lookback.monitor.mode": "disabled", + }) + factory := requireMetricLookbackDogStatsDFactory(t, cfg, newMetricLookbackRetention(cfg)) + + lookback := factory(serializermocks.NewMetricSerializer(t)) + + require.NotNil(t, lookback) + stopLookbackOnCleanup(t, lookback) + require.True(t, lookback.WantsDogStatsDMetric("target.metric")) + require.False(t, lookback.WantsDogStatsDMetric("other.metric")) +} + +func TestMetricLookbackDogStatsDFactoryBuildsMonitorEgressAdapterFromNoAggSeries(t *testing.T) { + start := time.Unix(100, 0) + cfg := metricLookbackMonitorFactoryConfig(t) + serializer := serializermocks.NewMetricSerializer(t) + forwarded := make(chan struct{}, 1) + serializer.On("SendIterableSeries", mock.Anything).Run(func(args mock.Arguments) { + source := args.Get(0).(metrics.SerieSource) + require.Greater(t, source.Count(), uint64(0)) + found := false + for source.MoveNext() { + if source.Current().Name == "target.metric" { + found = true + } + } + require.True(t, found) + forwarded <- struct{}{} + }).Return(nil).Maybe() + + factory := requireMetricLookbackDogStatsDFactory(t, cfg, newMetricLookbackRetention(cfg)) + lookback := factory(serializer) + require.NotNil(t, lookback) + stopLookbackOnCleanup(t, lookback) + + appendFactoryNoAggTestWindow(lookback, start, 0, 30, 2) + + select { + case <-forwarded: + case <-time.After(time.Second): + require.FailNow(t, "timed out waiting for monitor egress") + } +} + +func TestMetricLookbackDogStatsDFactoryDryRunForwardsHealthyMonitorWindow(t *testing.T) { + start := time.Unix(100, 0) + cfg := configmock.NewMockWithOverrides(t, map[string]interface{}{ + "metric_lookback.enabled": true, + "metric_lookback.capacity": 256, + "metric_lookback.shard_count": 1, + "metric_lookback.dogstatsd.metric_names": []string{}, + "metric_lookback.monitor.mode": "dry_run", + "metric_lookback.monitor.metric_name": "target.metric", + "metric_lookback.monitor.range_epsilon": 0.05, + }) + serializer := serializermocks.NewMetricSerializer(t) + forwarded := make(chan struct{}, 1) + serializer.On("SendIterableSeries", mock.Anything).Run(func(args mock.Arguments) { + source := args.Get(0).(metrics.SerieSource) + require.Greater(t, source.Count(), uint64(0)) + forwarded <- struct{}{} + }).Return(nil).Maybe() + + factory := requireMetricLookbackDogStatsDFactory(t, cfg, newMetricLookbackRetention(cfg)) + lookback := factory(serializer) + require.NotNil(t, lookback) + stopLookbackOnCleanup(t, lookback) + + for second := 0; second <= 30; second++ { + lookback.AppendDogStatsDNoAggSerie(&metrics.Serie{ + Name: "target.metric", + Points: []metrics.Point{{Ts: float64(start.Add(time.Duration(second) * time.Second).Unix()), Value: 2}}, + Tags: tagset.CompositeTagsFromSlice(nil), + MType: metrics.APIGaugeType, + Interval: 10, + }) + } + + select { + case <-forwarded: + case <-time.After(time.Second): + require.FailNow(t, "timed out waiting for dry-run monitor egress") + } +} + +func TestMetricLookbackDogStatsDFactoryBuildsMonitorEgressAdapterFromBucketedSamples(t *testing.T) { + start := time.Unix(100, 0) + cfg := metricLookbackMonitorFactoryConfig(t) + serializer := serializermocks.NewMetricSerializer(t) + forwarded := make(chan struct{}, 1) + serializer.On("SendIterableSeries", mock.Anything).Run(func(args mock.Arguments) { + source := args.Get(0).(metrics.SerieSource) + require.Greater(t, source.Count(), uint64(0)) + foundBucketedPoint := false + for source.MoveNext() { + serie := source.Current() + if serie.Name == "target.metric" && serie.Interval == 1 { + foundBucketedPoint = true + } + } + require.True(t, foundBucketedPoint) + forwarded <- struct{}{} + }).Return(nil).Maybe() + + factory := requireMetricLookbackDogStatsDFactory(t, cfg, newMetricLookbackRetention(cfg)) + lookback := factory(serializer) + require.NotNil(t, lookback) + stopLookbackOnCleanup(t, lookback) + + appendFactoryBucketedTestWindow(lookback, start, 0, 30, 2) + + select { + case <-forwarded: + case <-time.After(time.Second): + require.FailNow(t, "timed out waiting for monitor egress") + } +} + +func TestMetricLookbackDogStatsDFactoryBuildsMonitorEgressAdapterFromDistributionSamples(t *testing.T) { + start := time.Unix(100, 0) + cfg := metricLookbackMonitorFactoryConfig(t) + serializer := serializermocks.NewMetricSerializer(t) + forwarded := make(chan struct{}, 1) + serializer.On("SendSketch", mock.Anything).Run(func(args mock.Arguments) { + source := args.Get(0).(metrics.SketchesSource) + require.Greater(t, source.Count(), uint64(0)) + foundSketch := false + for source.MoveNext() { + series, ok := source.Current().(*metrics.SketchSeries) + require.True(t, ok) + if series.Name == "target.metric" { + foundSketch = true + } + } + require.True(t, foundSketch) + forwarded <- struct{}{} + }).Return(nil).Maybe() + + factory := requireMetricLookbackDogStatsDFactory(t, cfg, newMetricLookbackRetention(cfg)) + lookback := factory(serializer) + require.NotNil(t, lookback) + stopLookbackOnCleanup(t, lookback) + + appendFactoryDistributionTestWindow(lookback, start, 0, 30, 2) + + select { + case <-forwarded: + case <-time.After(time.Second): + require.FailNow(t, "timed out waiting for monitor egress") + } +} + +func TestMetricLookbackMonitorEgressAdapterFromShadowSenderSamples(t *testing.T) { + start := time.Unix(100, 0) + cfg := metricLookbackMonitorFactoryConfig(t) + retention := newMetricLookbackRetention(cfg) + serializer := serializermocks.NewMetricSerializer(t) + forwarded := make(chan struct{}, 1) + serializer.On("SendIterableSeries", mock.Anything).Run(func(args mock.Arguments) { + source := args.Get(0).(metrics.SerieSource) + require.Greater(t, source.Count(), uint64(0)) + foundShadowPoint := false + for source.MoveNext() { + serie := source.Current() + if serie.Name == "target.metric" && serie.Host == "default-host" { + foundShadowPoint = true + } + } + require.True(t, foundShadowPoint) + forwarded <- struct{}{} + }).Return(nil).Maybe() + + factory := requireMetricLookbackDogStatsDFactory(t, cfg, retention) + lookback := factory(serializer) + // The DogStatsD adapter is still created because the monitor metric is auto-admitted, + // but this test only writes through the shadow-check sender manager. + require.NotNil(t, lookback) + stopLookbackOnCleanup(t, lookback) + + manager := collectormetriclookback.NewSenderManager(context.Background(), "default-host", retention) + sender, err := manager.GetSender(checkid.ID("cpu:shadow")) + require.NoError(t, err) + for second := 0; second <= 30; second++ { + ts := float64(start.Add(time.Duration(second) * time.Second).Unix()) + value := float64(2) + if second == 30 { + value = 3 + } + require.NoError(t, sender.GaugeWithTimestamp("target.metric", value, "", nil, ts)) + sender.Commit() + } + + select { + case <-forwarded: + case <-time.After(time.Second): + require.FailNow(t, "timed out waiting for shadow sender monitor egress") + } +} + +func TestMetricLookbackDogStatsDFactoryRejectsNegativeRangeEpsilon(t *testing.T) { + cfg := configmock.NewMockWithOverrides(t, map[string]interface{}{ + "metric_lookback.enabled": true, + "metric_lookback.monitor.mode": "enabled", + "metric_lookback.monitor.metric_name": "target.metric", + "metric_lookback.monitor.range_epsilon": -0.01, + }) + + factory, err := newMetricLookbackDogStatsDFactory(cfg, logmock.New(t), newMetricLookbackRetention(cfg)) + + require.Nil(t, factory) + require.ErrorContains(t, err, "metric_lookback.monitor.range_epsilon") +} + +func TestMetricLookbackDogStatsDFactoryRejectsNegativeWindowDurations(t *testing.T) { + for _, key := range []string{ + "metric_lookback.monitor.evaluation_interval", + "metric_lookback.egress.pre_trigger_window", + "metric_lookback.egress.post_recovery_window", + } { + t.Run(key, func(t *testing.T) { + cfg := configmock.NewMockWithOverrides(t, map[string]interface{}{ + "metric_lookback.enabled": true, + "metric_lookback.monitor.mode": "enabled", + "metric_lookback.monitor.metric_name": "target.metric", + "metric_lookback.monitor.range_epsilon": 0.05, + key: -time.Second, + }) + + factory, err := newMetricLookbackDogStatsDFactory(cfg, logmock.New(t), newMetricLookbackRetention(cfg)) + + require.Nil(t, factory) + require.ErrorContains(t, err, key) + }) + } +} + +func TestMetricLookbackDogStatsDFactoryRejectsInvalidMonitorMode(t *testing.T) { + cfg := configmock.NewMockWithOverrides(t, map[string]interface{}{ + "metric_lookback.enabled": true, + "metric_lookback.monitor.mode": "observe", + }) + + factory, err := newMetricLookbackDogStatsDFactory(cfg, logmock.New(t), newMetricLookbackRetention(cfg)) + + require.Nil(t, factory) + require.ErrorContains(t, err, "metric_lookback.monitor.mode") +} + +func TestMetricLookbackDogStatsDFactoryRequiresMetricNameWhenMonitorModeActive(t *testing.T) { + cfg := configmock.NewMockWithOverrides(t, map[string]interface{}{ + "metric_lookback.enabled": true, + "metric_lookback.monitor.mode": "enabled", + }) + + factory, err := newMetricLookbackDogStatsDFactory(cfg, logmock.New(t), newMetricLookbackRetention(cfg)) + + require.Nil(t, factory) + require.ErrorContains(t, err, "metric_lookback.monitor.metric_name") +} + +func requireMetricLookbackDogStatsDFactory(t testing.TB, cfg configmock.Component, retention *metriclookback.Retention) aggregator.DogStatsDLookbackFactory { + t.Helper() + factory, err := newMetricLookbackDogStatsDFactory(cfg, logmock.New(t), retention) + require.NoError(t, err) + require.NotNil(t, factory) + return factory +} + +func stopLookbackOnCleanup(t testing.TB, lookback aggregator.DogStatsDLookback) { + t.Helper() + stopper, ok := lookback.(aggregator.DogStatsDLookbackStopper) + if !ok { + return + } + t.Cleanup(stopper.Stop) +} + +func metricLookbackMonitorFactoryConfig(t testing.TB) configmock.Component { + return configmock.NewMockWithOverrides(t, map[string]interface{}{ + "metric_lookback.enabled": true, + "metric_lookback.capacity": 256, + "metric_lookback.shard_count": 1, + "metric_lookback.dogstatsd.metric_names": []string{}, + "metric_lookback.monitor.mode": "enabled", + "metric_lookback.monitor.metric_name": "target.metric", + "metric_lookback.monitor.evaluation_interval": 30 * time.Second, + "metric_lookback.monitor.range_epsilon": 0.05, + "metric_lookback.egress.pre_trigger_window": 0 * time.Second, + "metric_lookback.egress.post_recovery_window": 30 * time.Second, + }) +} + +func appendFactoryNoAggTestWindow(lookback interface { + AppendDogStatsDNoAggSerie(*metrics.Serie) +}, start time.Time, fromSecond, toSecond int, value float64) { + for second := fromSecond; second <= toSecond; second++ { + sampleValue := value + if second == toSecond { + sampleValue = value + 1 + } + lookback.AppendDogStatsDNoAggSerie(&metrics.Serie{ + Name: "target.metric", + Points: []metrics.Point{{Ts: float64(start.Add(time.Duration(second) * time.Second).Unix()), Value: sampleValue}}, + Tags: tagset.CompositeTagsFromSlice(nil), + MType: metrics.APIGaugeType, + Interval: 10, + }) + } +} + +func appendFactoryBucketedTestWindow(lookback aggregator.DogStatsDLookback, start time.Time, fromSecond, toSecond int, value float64) { + ctx := aggregator.DogStatsDLookbackContext{ + ContextKey: ckey.ContextKey(1), + Name: "target.metric", + Host: "host", + } + for second := fromSecond; second <= toSecond; second++ { + ts := float64(start.Add(time.Duration(second) * time.Second).Unix()) + sampleValue := value + if second == toSecond { + sampleValue = value + 1 + } + lookback.ObserveDogStatsDSample(&metrics.MetricSample{ + Name: "target.metric", + Value: sampleValue, + Mtype: metrics.GaugeType, + SampleRate: 1, + }, ts, ctx) + lookback.FlushDogStatsDBuckets(ts+metriclookbackdogstatsd.DefaultDogStatsDSealDelay.Seconds()+1, false) + } +} + +func appendFactoryDistributionTestWindow(lookback aggregator.DogStatsDLookback, start time.Time, fromSecond, toSecond int, value float64) { + ctx := aggregator.DogStatsDLookbackContext{ + ContextKey: ckey.ContextKey(2), + Name: "target.metric", + Host: "host", + } + for second := fromSecond; second <= toSecond; second++ { + ts := float64(start.Add(time.Duration(second) * time.Second).Unix()) + sampleValue := value + if second == toSecond { + sampleValue = value + 1 + } + lookback.ObserveDogStatsDSample(&metrics.MetricSample{ + Name: "target.metric", + Value: sampleValue, + Mtype: metrics.DistributionType, + SampleRate: 1, + }, ts, ctx) + lookback.FlushDogStatsDBuckets(ts+metriclookbackdogstatsd.DefaultDogStatsDSealDelay.Seconds()+1, false) + } +} diff --git a/pkg/aggregator/BUILD.bazel b/pkg/aggregator/BUILD.bazel index 0b86e02c65aa..317769beaa6f 100644 --- a/pkg/aggregator/BUILD.bazel +++ b/pkg/aggregator/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "demultiplexer_mock.go", "demultiplexer_senders.go", "demultiplexer_serverless.go", + "lookback.go", "no_aggregation_stream_worker.go", "no_op_sender_manager.go", "no_orchestrator.go", @@ -163,6 +164,7 @@ go_test( "//pkg/metrics", "//pkg/metrics/event", "//pkg/metrics/servicecheck", + "//pkg/serializer", "//pkg/tagger/types", "//pkg/tagset", "//pkg/util/flavor", diff --git a/pkg/aggregator/demultiplexer_agent.go b/pkg/aggregator/demultiplexer_agent.go index 74348d801838..eb0aecfc763e 100644 --- a/pkg/aggregator/demultiplexer_agent.go +++ b/pkg/aggregator/demultiplexer_agent.go @@ -87,6 +87,9 @@ type AgentDemultiplexerOptions struct { NoAggregationPipelineWorkersCount int + DogStatsDLookback DogStatsDLookback + DogStatsDLookbackFactory DogStatsDLookbackFactory + DontStartForwarders bool // unit tests don't need the forwarders to be instanciated UseDogstatsdContextLimiter bool @@ -168,6 +171,9 @@ func initAgentDemultiplexer(log log.Component, // ---------------------- sharedSerializer := serializer.NewSerializer(sharedForwarder, orchestratorForwarder, compressor, pkgconfigsetup.Datadog(), log, hostname) + if options.DogStatsDLookback == nil && options.DogStatsDLookbackFactory != nil { + options.DogStatsDLookback = options.DogStatsDLookbackFactory(sharedSerializer) + } // prepare the embedded aggregator // -- @@ -189,6 +195,7 @@ func initAgentDemultiplexer(log log.Component, tagsStore := tags.NewStore(pkgconfigsetup.Datadog().GetBool("aggregator_use_tags_store"), fmt.Sprintf("timesampler #%d", i)) statsdSampler := NewTimeSampler(TimeSamplerID(i), bucketSize, tagsStore, tagger, agg.hostname) + statsdSampler.dogStatsDLookback = options.DogStatsDLookback // its worker (process loop + flush/serialization mechanism) @@ -213,6 +220,7 @@ func initAgentDemultiplexer(log log.Component, noAggSerializers[i], agg.flushAndSerializeInParallel, tagger, + options.DogStatsDLookback, ) } } @@ -484,6 +492,9 @@ func (d *AgentDemultiplexer) Stop() { d.aggregator.Stop() } d.aggregator = nil + if stopper, ok := d.options.DogStatsDLookback.(DogStatsDLookbackStopper); ok { + stopper.Stop() + } // forwarders diff --git a/pkg/aggregator/demultiplexer_agent_test.go b/pkg/aggregator/demultiplexer_agent_test.go index cc3a9f378fa3..dfdc64b403e5 100644 --- a/pkg/aggregator/demultiplexer_agent_test.go +++ b/pkg/aggregator/demultiplexer_agent_test.go @@ -41,6 +41,7 @@ import ( configmock "github.com/DataDog/datadog-agent/pkg/config/mock" "github.com/DataDog/datadog-agent/pkg/metrics" "github.com/DataDog/datadog-agent/pkg/metrics/event" + "github.com/DataDog/datadog-agent/pkg/serializer" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) @@ -73,10 +74,36 @@ func testDemuxSamples(_ *testing.T) metrics.MetricSampleBatch { // the option is NOT enabled, this metric should go into the first // timesampler of the statsd stack. +type recordingDogStatsDNoAggLookback struct { + calls int + stops int + series []*metrics.Serie +} + +func (r *recordingDogStatsDNoAggLookback) WantsDogStatsDMetric(string) bool { return false } + +func (r *recordingDogStatsDNoAggLookback) ObserveDogStatsDSample(*metrics.MetricSample, float64, DogStatsDLookbackContext) { +} + +func (r *recordingDogStatsDNoAggLookback) FlushDogStatsDBuckets(float64, bool) {} + +func (r *recordingDogStatsDNoAggLookback) AppendDogStatsDNoAggSerie(serie *metrics.Serie) { + r.calls++ + copySerie := *serie + copySerie.Points = append([]metrics.Point(nil), serie.Points...) + r.series = append(r.series, ©Serie) +} + +func (r *recordingDogStatsDNoAggLookback) Stop() { + r.stops++ +} + func TestDemuxNoAggOptionDisabled(t *testing.T) { require := require.New(t) + lookback := &recordingDogStatsDNoAggLookback{} opts := demuxTestOptions() + opts.DogStatsDLookback = lookback deps := createDemultiplexerAgentTestDeps(t) demux := initAgentDemultiplexer(deps.Log, NewForwarderTest(deps.Log), deps.OrchestratorFwd, opts, deps.EventPlatform, deps.HaAgent, deps.Compressor, deps.Tagger, deps.FilterList, "") @@ -87,6 +114,7 @@ func TestDemuxNoAggOptionDisabled(t *testing.T) { require.Len(demux.statsd.workers[0].samplesChan, 1) read := <-demux.statsd.workers[0].samplesChan require.Len(read, 3) + require.Equal(0, lookback.calls, "lookback should not receive samples when no-aggregation is disabled and samples are redirected to normal aggregation") } // the option is enabled, these metrics will go through the no aggregation pipeline. @@ -95,7 +123,9 @@ func TestDemuxNoAggOptionEnabled(t *testing.T) { noAggWorkerStreamCheckFrequency = 100 * time.Millisecond + lookback := &recordingDogStatsDNoAggLookback{} opts := demuxTestOptions() + opts.DogStatsDLookback = lookback mockSerializer := &MockSerializerIterableSerie{} mockSerializer.On("AreSeriesEnabled").Return(true) mockSerializer.On("AreSketchesEnabled").Return(true) @@ -115,12 +145,20 @@ func TestDemuxNoAggOptionEnabled(t *testing.T) { // nothing should be in the time sampler require.Len(demux.statsd.workers[0].samplesChan, 0) require.Len(mockSerializer.series, 3) + require.Equal(3, lookback.calls) + require.Equal(1, lookback.stops) + require.Len(lookback.series, 3) for i := 0; i < len(batch); i++ { require.Equal(batch[i].Name, mockSerializer.series[i].Name) require.Len(mockSerializer.series[i].Points, 1) require.Equal(batch[i].Timestamp, mockSerializer.series[i].Points[0].Ts) require.ElementsMatch(batch[i].Tags, mockSerializer.series[i].Tags.UnsafeToReadOnlySliceString()) + require.Equal(mockSerializer.series[i].Name, lookback.series[i].Name) + require.Equal(mockSerializer.series[i].Points, lookback.series[i].Points) + require.Equal(mockSerializer.series[i].Tags.UnsafeToReadOnlySliceString(), lookback.series[i].Tags.UnsafeToReadOnlySliceString()) + require.Equal(mockSerializer.series[i].MType, lookback.series[i].MType) + require.Equal(mockSerializer.series[i].Interval, lookback.series[i].Interval) } } @@ -214,9 +252,44 @@ func TestDemuxNoAggWorkersUseSharedQueue(t *testing.T) { } } +func TestDemuxNoAggLookbackDoesNotReceiveNormalAggregationBatches(t *testing.T) { + lookback := &recordingDogStatsDNoAggLookback{} + opts := demuxTestOptions() + opts.NoAggregationPipelineWorkersCount = 1 + opts.DogStatsDLookback = lookback + deps := createDemultiplexerAgentTestDeps(t) + + demux := initAgentDemultiplexer(deps.Log, NewForwarderTest(deps.Log), deps.OrchestratorFwd, opts, deps.EventPlatform, deps.HaAgent, deps.Compressor, deps.Tagger, deps.FilterList, "") + + demux.AggregateSamples(TimeSamplerID(0), testDemuxSamples(t)) + + require.Equal(t, 0, lookback.calls) + require.Len(t, demux.statsd.workers[0].samplesChan, 1) +} + +func TestDemuxNoAggLookbackFactoryReceivesSharedSerializer(t *testing.T) { + lookback := &recordingDogStatsDNoAggLookback{} + called := false + opts := demuxTestOptions() + opts.NoAggregationPipelineWorkersCount = 1 + opts.DogStatsDLookbackFactory = func(metricSerializer serializer.MetricSerializer) DogStatsDLookback { + called = true + require.NotNil(t, metricSerializer) + return lookback + } + deps := createDemultiplexerAgentTestDeps(t) + + demux := initAgentDemultiplexer(deps.Log, NewForwarderTest(deps.Log), deps.OrchestratorFwd, opts, deps.EventPlatform, deps.HaAgent, deps.Compressor, deps.Tagger, deps.FilterList, "") + + require.True(t, called) + require.Same(t, lookback, demux.options.DogStatsDLookback) +} + func TestSendSamplesWithoutAggregationDropsEmptyBatch(t *testing.T) { + lookback := &recordingDogStatsDNoAggLookback{} opts := demuxTestOptions() opts.NoAggregationPipelineWorkersCount = 1 + opts.DogStatsDLookback = lookback deps := createDemultiplexerAgentTestDeps(t) demux := initAgentDemultiplexer(deps.Log, NewForwarderTest(deps.Log), deps.OrchestratorFwd, opts, deps.EventPlatform, deps.HaAgent, deps.Compressor, deps.Tagger, deps.FilterList, "") @@ -224,6 +297,7 @@ func TestSendSamplesWithoutAggregationDropsEmptyBatch(t *testing.T) { demux.SendSamplesWithoutAggregation(metrics.MetricSampleBatch{}) require.Len(t, demux.statsd.noAggSamplesChan, 0) + require.Equal(t, 0, lookback.calls) } func TestAddAgentStartupTelemetrySendsShutdownEventOnFinalStop(t *testing.T) { diff --git a/pkg/aggregator/lookback.go b/pkg/aggregator/lookback.go new file mode 100644 index 000000000000..c71e0ea1ea93 --- /dev/null +++ b/pkg/aggregator/lookback.go @@ -0,0 +1,67 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package aggregator + +import ( + "github.com/DataDog/datadog-agent/pkg/aggregator/ckey" + "github.com/DataDog/datadog-agent/pkg/metrics" + "github.com/DataDog/datadog-agent/pkg/serializer" +) + +// DogStatsDLookbackContext carries the effective series identity resolved by +// the normal DogStatsD aggregation path. Lookback implementations should use +// this context instead of reimplementing DogStatsD tagger enrichment, host +// extraction, metric tag filtering, source, or no-index behavior. +type DogStatsDLookbackContext struct { + // ContextKey is the same context key used by the TimeSampler for the normal + // DogStatsD aggregation path. It lets lookback implementations avoid + // rebuilding a selected series key on every matching sample. + ContextKey ckey.ContextKey + + Name string + Host string + Tags []string + NoIndex bool + Source metrics.MetricSource +} + +// DogStatsDLookback receives selected semantic DogStatsD observations from the +// existing aggregation pipeline. Concrete implementations live outside +// pkg/aggregator so binaries that do not enable metric lookback do not link the +// retention ring or trigger code. +type DogStatsDLookback interface { + // WantsDogStatsDMetric is the hot-path exact-name admission check. It must not + // allocate for nonmatching samples. + WantsDogStatsDMetric(name string) bool + + // ObserveDogStatsDSample receives a normal DogStatsD sample after the existing + // TimeSampler context resolver has computed the effective series identity and + // after normal aggregation has accepted the sample. + ObserveDogStatsDSample(sample *metrics.MetricSample, timestamp float64, ctx DogStatsDLookbackContext) + + // FlushDogStatsDBuckets gives the lookback materializer a chance to seal + // buckets using the same clock cadence as normal DogStatsD flushing. When + // forceFlushAll is true, open buckets should be sealed even if they are newer + // than the normal lookback seal delay. + FlushDogStatsDBuckets(timestamp float64, forceFlushAll bool) + + // AppendDogStatsDNoAggSerie receives the same DogStatsD no-aggregation series + // that the worker appends to the serializer. + AppendDogStatsDNoAggSerie(*metrics.Serie) +} + +// DogStatsDLookbackStopper is optionally implemented by DogStatsDLookback +// adapters that own background resources. The demultiplexer calls Stop during +// shutdown when this interface is present. +type DogStatsDLookbackStopper interface { + Stop() +} + +// DogStatsDLookbackFactory builds a DogStatsD lookback adapter after the +// demultiplexer has constructed its serializer. This keeps dump-trigger wiring +// out of pkg/aggregator while still allowing concrete lookback code to close +// over the serializer it should dump through. +type DogStatsDLookbackFactory func(serializer.MetricSerializer) DogStatsDLookback diff --git a/pkg/aggregator/no_aggregation_stream_worker.go b/pkg/aggregator/no_aggregation_stream_worker.go index 7eb72e0b9084..848441242fb8 100644 --- a/pkg/aggregator/no_aggregation_stream_worker.go +++ b/pkg/aggregator/no_aggregation_stream_worker.go @@ -62,6 +62,11 @@ type noAggregationStreamWorker struct { // observerHandle is set when the observer component is wired in. // Nil when the feature is disabled or the observer is not included in the binary. observerHandle observer.Handle + + // lookback receives the final no-aggregation series after the worker has + // applied the same tag enrichment, type mapping, and value normalization used + // for normal serialization. + lookback DogStatsDLookback } // noAggWorkerStreamCheckFrequency is the frequency at which the no agg worker @@ -97,6 +102,7 @@ func newNoAggregationStreamWorker(maxMetricsPerPayload int, metricSamplePool *me samplesChan chan metrics.MetricSampleBatch, serializer serializer.MetricSerializer, flushConfig FlushAndSerializeInParallel, tagger tagger.Component, + lookback DogStatsDLookback, ) *noAggregationStreamWorker { return &noAggregationStreamWorker{ serializer: serializer, @@ -119,7 +125,8 @@ func newNoAggregationStreamWorker(maxMetricsPerPayload int, metricSamplePool *me // every 5 minutes. logThrottling: util.NewSimpleThrottler(200, 5*time.Minute, "Pausing the unsupported metric type warning message for 5m"), - tagger: tagger, + tagger: tagger, + lookback: lookback, } } @@ -221,6 +228,9 @@ func (w *noAggregationStreamWorker) run() { serie.Host = sample.Host serie.MType = mtype serie.Interval = bucketSize + if w.lookback != nil { + w.lookback.AppendDogStatsDNoAggSerie(&serie) + } seriesSink.Append(&serie) w.taggerBuffer.Reset() diff --git a/pkg/aggregator/time_sampler.go b/pkg/aggregator/time_sampler.go index 53758dc8ba73..7d31ec164d15 100644 --- a/pkg/aggregator/time_sampler.go +++ b/pkg/aggregator/time_sampler.go @@ -50,6 +50,10 @@ type TimeSampler struct { // observerHandle is set when the observer component is wired in. // Nil when the feature is disabled or the observer is not included in the binary. observerHandle observer.Handle + + // dogStatsDLookback is set when metric lookback is wired in. Nil when the + // feature is disabled, so default DogStatsD hot-path overhead is zero. + dogStatsDLookback DogStatsDLookback } // NewTimeSampler returns a newly initialized TimeSampler @@ -101,7 +105,9 @@ func (s *TimeSampler) sample(metricSample *metrics.MetricSample, timestamp float switch metricSample.Mtype { case metrics.DistributionType: - s.sketchMap.insert(bucketStart, contextKey, metricSample.Value, metricSample.SampleRate) + if !s.sketchMap.insert(bucketStart, contextKey, metricSample.Value, metricSample.SampleRate) { + return + } default: // If it's a new bucket, initialize it bucketMetrics, ok := s.metricsByTimestamp[bucketStart] @@ -112,8 +118,33 @@ func (s *TimeSampler) sample(metricSample *metrics.MetricSample, timestamp float // Add sample to bucket if err := bucketMetrics.AddSample(contextKey, metricSample, timestamp, s.interval, nil, pkgconfigsetup.Datadog()); err != nil { log.Debugf("TimeSampler #%d Ignoring sample '%s' on host '%s' and tags '%s': %s", s.id, metricSample.Name, metricSample.Host, metricSample.Tags, err) + return } } + + s.observeDogStatsDLookback(metricSample, timestamp, contextKey) +} + +func (s *TimeSampler) observeDogStatsDLookback(metricSample *metrics.MetricSample, timestamp float64, contextKey ckey.ContextKey) { + lookback := s.dogStatsDLookback + if lookback == nil || !lookback.WantsDogStatsDMetric(metricSample.Name) { + return + } + + context, ok := s.contextResolver.get(contextKey) + if !ok { + log.Errorf("TimeSampler #%d Ignoring metric lookback sample on context key '%v': inconsistent context resolver state: the context is not tracked", s.id, contextKey) + return + } + + lookback.ObserveDogStatsDSample(metricSample, timestamp, DogStatsDLookbackContext{ + ContextKey: contextKey, + Name: context.Name, + Host: context.Host, + Tags: context.Tags().UnsafeToReadOnlySliceString(), + NoIndex: context.noIndex, + Source: context.source, + }) } func (s *TimeSampler) newSketchSeries(ck ckey.ContextKey, points []metrics.SketchPoint) *metrics.SketchSeries { @@ -250,6 +281,9 @@ func (s *TimeSampler) flush(timestamp float64, series metrics.SerieSink, sketche s.flushSeries(cutoffTime, series, filterList, forceFlushAll) s.flushSketches(cutoffTime, sketches, forceFlushAll) + if s.dogStatsDLookback != nil { + s.dogStatsDLookback.FlushDogStatsDBuckets(timestamp, forceFlushAll) + } // expiring contexts s.contextResolver.expireContexts(int64(timestamp)) s.lastCutOffTime = cutoffTime diff --git a/pkg/aggregator/time_sampler_test.go b/pkg/aggregator/time_sampler_test.go index d35b5e062af8..e84b833f1938 100644 --- a/pkg/aggregator/time_sampler_test.go +++ b/pkg/aggregator/time_sampler_test.go @@ -41,6 +41,42 @@ func testTimeSampler(store *tags.Store) *TimeSampler { return sampler } +type recordingDogStatsDLookback struct { + wanted map[string]struct{} + observations []recordedDogStatsDLookbackObservation + flushes []recordedDogStatsDLookbackFlush +} + +type recordedDogStatsDLookbackObservation struct { + sample metrics.MetricSample + timestamp float64 + ctx DogStatsDLookbackContext +} + +type recordedDogStatsDLookbackFlush struct { + timestamp float64 + forceFlushAll bool +} + +func (r *recordingDogStatsDLookback) WantsDogStatsDMetric(name string) bool { + _, found := r.wanted[name] + return found +} + +func (r *recordingDogStatsDLookback) ObserveDogStatsDSample(sample *metrics.MetricSample, timestamp float64, ctx DogStatsDLookbackContext) { + r.observations = append(r.observations, recordedDogStatsDLookbackObservation{ + sample: *sample, + timestamp: timestamp, + ctx: ctx, + }) +} + +func (r *recordingDogStatsDLookback) FlushDogStatsDBuckets(timestamp float64, forceFlushAll bool) { + r.flushes = append(r.flushes, recordedDogStatsDLookbackFlush{timestamp: timestamp, forceFlushAll: forceFlushAll}) +} + +func (r *recordingDogStatsDLookback) AppendDogStatsDNoAggSerie(*metrics.Serie) {} + // TimeSampler func TestCalculateBucketStart(t *testing.T) { sampler := testTimeSampler(tags.NewStore(true, "test")) @@ -84,6 +120,130 @@ func TestBucketSampling(t *testing.T) { testWithTagsStore(t, testBucketSampling) } +func TestTimeSamplerDogStatsDLookbackReceivesSelectedResolvedContext(t *testing.T) { + samper := testTimeSampler(tags.NewStore(true, "test")) + lookback := &recordingDogStatsDLookback{wanted: map[string]struct{}{"target.metric": {}}} + samper.dogStatsDLookback = lookback + matcher := filterlist.NewNoopTagMatcher() + + samper.sample(&metrics.MetricSample{ + Name: "other.metric", + Value: 1, + Mtype: metrics.GaugeType, + Tags: []string{"env:test"}, + SampleRate: 1, + }, 10, matcher) + samper.sample(&metrics.MetricSample{ + Name: "target.metric", + Value: 2, + Mtype: metrics.GaugeType, + Tags: []string{"env:test", "role:web"}, + Host: "sample-host", + SampleRate: 1, + NoIndex: true, + Source: metrics.MetricSource(9), + }, 11, matcher) + + require.Len(t, lookback.observations, 1) + observation := lookback.observations[0] + require.Equal(t, "target.metric", observation.sample.Name) + require.Equal(t, float64(11), observation.timestamp) + require.Equal(t, "target.metric", observation.ctx.Name) + require.Equal(t, "sample-host", observation.ctx.Host) + require.ElementsMatch(t, []string{"env:test", "role:web"}, observation.ctx.Tags) + require.True(t, observation.ctx.NoIndex) + require.Equal(t, metrics.MetricSource(9), observation.ctx.Source) + require.False(t, observation.ctx.ContextKey.IsZero()) +} + +func TestTimeSamplerDogStatsDLookbackFlushesBuckets(t *testing.T) { + samper := testTimeSampler(tags.NewStore(true, "test")) + lookback := &recordingDogStatsDLookback{wanted: map[string]struct{}{"target.metric": {}}} + samper.dogStatsDLookback = lookback + + series, sketches := flushSerie(samper, 123, false) + require.Empty(t, series) + require.Empty(t, sketches) + require.Equal(t, []recordedDogStatsDLookbackFlush{{timestamp: 123}}, lookback.flushes) +} + +func TestTimeSamplerDogStatsDLookbackForceFlushesAllBuckets(t *testing.T) { + samper := testTimeSampler(tags.NewStore(true, "test")) + lookback := &recordingDogStatsDLookback{wanted: map[string]struct{}{"target.metric": {}}} + samper.dogStatsDLookback = lookback + + series, sketches := flushSerie(samper, 123, true) + require.Empty(t, series) + require.Empty(t, sketches) + require.Equal(t, []recordedDogStatsDLookbackFlush{{timestamp: 123, forceFlushAll: true}}, lookback.flushes) +} + +func TestTimeSamplerDogStatsDLookbackUsesFilteredCounterContext(t *testing.T) { + configmock.New(t).SetInTest("metric_tag_filterlist_adp_only", false) + samper := testTimeSampler(tags.NewStore(true, "test")) + lookback := &recordingDogStatsDLookback{wanted: map[string]struct{}{"counter.metric": {}}} + samper.dogStatsDLookback = lookback + matcher := filterlist.NewTagMatcher(map[string]filterlist.MetricTagList{ + "counter.metric": { + Tags: []string{"env"}, + Action: "exclude", + }, + }, logmock.New(t)) + + samper.sample(&metrics.MetricSample{ + Name: "counter.metric", + Value: 5, + Mtype: metrics.CounterType, + Tags: []string{"env:prod", "instance:a"}, + SampleRate: 1, + }, 1001, matcher) + samper.sample(&metrics.MetricSample{ + Name: "counter.metric", + Value: 7, + Mtype: metrics.CounterType, + Tags: []string{"env:dev", "instance:a"}, + SampleRate: 1, + }, 1005, matcher) + + require.Len(t, lookback.observations, 2) + require.Equal(t, lookback.observations[0].ctx.ContextKey, lookback.observations[1].ctx.ContextKey) + for _, observation := range lookback.observations { + require.Equal(t, []string{"instance:a"}, observation.ctx.Tags) + } +} + +func TestTimeSamplerDogStatsDLookbackIgnoresRejectedSamples(t *testing.T) { + samper := testTimeSampler(tags.NewStore(true, "test")) + lookback := &recordingDogStatsDLookback{wanted: map[string]struct{}{"target.metric": {}}} + samper.dogStatsDLookback = lookback + matcher := filterlist.NewNoopTagMatcher() + + samper.sample(&metrics.MetricSample{ + Name: "target.metric", + Value: math.NaN(), + Mtype: metrics.GaugeType, + SampleRate: 1, + }, 10, matcher) + + require.Empty(t, lookback.observations) +} + +func TestTimeSamplerDogStatsDLookbackIgnoresRejectedDistributionSamples(t *testing.T) { + samper := testTimeSampler(tags.NewStore(true, "test")) + lookback := &recordingDogStatsDLookback{wanted: map[string]struct{}{"target.metric": {}}} + samper.dogStatsDLookback = lookback + matcher := filterlist.NewNoopTagMatcher() + + samper.sample(&metrics.MetricSample{ + Name: "target.metric", + Value: math.NaN(), + Mtype: metrics.DistributionType, + SampleRate: 1, + }, 10, matcher) + + require.Empty(t, lookback.observations) +} + func testContextSampling(t *testing.T, store *tags.Store) { sampler := testTimeSampler(store) matcher := filterlist.NewNoopTagMatcher() diff --git a/pkg/collector/BUILD.bazel b/pkg/collector/BUILD.bazel index d24b6feea435..1b8ef3281e31 100644 --- a/pkg/collector/BUILD.bazel +++ b/pkg/collector/BUILD.bazel @@ -8,6 +8,8 @@ go_library( srcs = [ "infra_mode.go", "scheduler.go", + "scheduler_metriclookback.go", + "scheduler_metriclookback_nopython.go", "stats.go", ], importpath = "github.com/DataDog/datadog-agent/pkg/collector", @@ -25,7 +27,6 @@ go_library( "//pkg/collector/corechecks", "//pkg/collector/loaders", "//pkg/collector/metriclookback", - "//pkg/collector/metriclookback/lookbacksender", "//pkg/config/model", "//pkg/config/setup", "//pkg/util/infratags", diff --git a/pkg/collector/metriclookback/BUILD.bazel b/pkg/collector/metriclookback/BUILD.bazel index 9b051adf6ebc..dbaa7b310886 100644 --- a/pkg/collector/metriclookback/BUILD.bazel +++ b/pkg/collector/metriclookback/BUILD.bazel @@ -6,13 +6,20 @@ go_library( srcs = [ "metadata.go", "policy.go", + "shadow_writer.go", ], importpath = "github.com/DataDog/datadog-agent/pkg/collector/metriclookback", visibility = ["//visibility:public"], deps = [ "//comp/core/autodiscovery/integration", "//comp/core/workloadfilter/def", + "//pkg/aggregator/sender", + "//pkg/collector/check/id", + "//pkg/collector/metriclookback/lookbacksender", "//pkg/config/model", + "//pkg/metriclookback", + "//pkg/metriclookback/ringbuffer", + "//pkg/metrics", "@in_yaml_go_yaml_v2//:yaml", ], ) @@ -22,12 +29,18 @@ dd_agent_go_test( srcs = [ "metadata_test.go", "policy_test.go", + "shadow_writer_test.go", ], embed = [":metriclookback"], deps = [ "//comp/core/autodiscovery/integration", "//comp/core/workloadfilter/def", + "//pkg/collector/check/id", "//pkg/config/mock", + "//pkg/metriclookback", + "//pkg/metriclookback/monitor", + "//pkg/metriclookback/ringbuffer", + "//pkg/metrics", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", "@in_yaml_go_yaml_v2//:yaml", diff --git a/pkg/collector/metriclookback/lookbacksender/sender.go b/pkg/collector/metriclookback/lookbacksender/sender.go index 35ecccadc541..1a642da6a0a5 100644 --- a/pkg/collector/metriclookback/lookbacksender/sender.go +++ b/pkg/collector/metriclookback/lookbacksender/sender.go @@ -40,8 +40,13 @@ type SenderManager struct { defaultSender *noopSender } -// NewSenderManager creates a sender manager for lookback shadow checks. +// NewSenderManager creates a sender manager for lookback shadow checks. It +// returns nil when no writer is configured so selected shadows cannot silently +// discard their only output. func NewSenderManager(ctx context.Context, defaultHostname string, writer Writer, now func() float64) *SenderManager { + if writer == nil { + return nil + } if ctx == nil { ctx = context.Background() } @@ -50,10 +55,6 @@ func NewSenderManager(ctx context.Context, defaultHostname string, writer Writer return float64(time.Now().UnixNano()) / 1e9 } } - if writer == nil { - writer = noopWriter{} - } - return &SenderManager{ ctx: ctx, defaultHostname: defaultHostname, @@ -352,12 +353,6 @@ func (s *sender) OrchestratorManifest(_ []types.ProcessMessageBody, _ string) { s.recordUnsupportedDrop("OrchestratorManifest") } -type noopWriter struct{} - -func (noopWriter) Append(context.Context, checkid.ID, []metrics.MetricSample) error { - return nil -} - type noopSender struct{} func (n *noopSender) Commit() {} diff --git a/pkg/collector/metriclookback/lookbacksender/sender_test.go b/pkg/collector/metriclookback/lookbacksender/sender_test.go index a12f4d536c5c..46202e1237e8 100644 --- a/pkg/collector/metriclookback/lookbacksender/sender_test.go +++ b/pkg/collector/metriclookback/lookbacksender/sender_test.go @@ -271,20 +271,16 @@ func TestSenderManagerRejectsSenderForDifferentCheckID(t *testing.T) { assert.ErrorContains(t, err, "sender ID cpu:shadow does not match check ID disk:shadow") } -func TestSenderManagerDefaultSenderAndNoopWriter(t *testing.T) { - manager := NewSenderManager(context.Background(), "default-host", nil, nil) +func TestSenderManagerRequiresWriterAndProvidesDefaultSender(t *testing.T) { + require.Nil(t, NewSenderManager(context.Background(), "default-host", nil, nil)) + manager := NewSenderManager(context.Background(), "default-host", &recordingWriter{}, nil) + require.NotNil(t, manager) defaultSender, err := manager.GetDefaultSender() require.NoError(t, err) defaultSender.Gauge("metric.default", 1, "", nil) defaultSender.Commit() assert.Equal(t, int64(0), defaultSender.GetSenderStats().MetricSamples) - - gotSender, err := manager.GetSender(checkid.ID("cpu:shadow")) - require.NoError(t, err) - gotSender.Gauge("metric.gauge", 1, "", nil) - gotSender.Commit() - assert.Equal(t, int64(1), gotSender.GetSenderStats().MetricSamples) } type testContextKey struct{} diff --git a/pkg/collector/metriclookback/metadata.go b/pkg/collector/metriclookback/metadata.go index 07a4b6077edf..78736b14e1a4 100644 --- a/pkg/collector/metriclookback/metadata.go +++ b/pkg/collector/metriclookback/metadata.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -// Package metriclookback contains helpers for 1Hz check metric lookback. +// Package metriclookback contains collector/check integration helpers for metric lookback. package metriclookback import ( diff --git a/pkg/collector/metriclookback/shadow_writer.go b/pkg/collector/metriclookback/shadow_writer.go new file mode 100644 index 000000000000..93c881c98a4e --- /dev/null +++ b/pkg/collector/metriclookback/shadow_writer.go @@ -0,0 +1,48 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package metriclookback + +import ( + "context" + + "github.com/DataDog/datadog-agent/pkg/aggregator/sender" + checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" + "github.com/DataDog/datadog-agent/pkg/collector/metriclookback/lookbacksender" + corelookback "github.com/DataDog/datadog-agent/pkg/metriclookback" + "github.com/DataDog/datadog-agent/pkg/metriclookback/ringbuffer" + "github.com/DataDog/datadog-agent/pkg/metrics" +) + +// NewSenderManager returns a shadow-check sender manager backed by the shared +// retention ring. It is the collector/check integration point for checks that +// should populate metric lookback without emitting their samples through the +// normal check sender path. +func NewSenderManager(ctx context.Context, defaultHostname string, retention *corelookback.Retention) sender.SenderManager { + if retention == nil { + return nil + } + return lookbacksender.NewSenderManager(ctx, defaultHostname, shadowWriter{retention: retention}, nil) +} + +type shadowWriter struct { + retention *corelookback.Retention +} + +// Append stores scalar samples emitted by a lookback shadow check. It satisfies +// lookbacksender.Writer and records the shadow check ID as retention source +// provenance without adding tags or changing the metric identity that egress +// forwards later. +func (w shadowWriter) Append(ctx context.Context, checkID checkid.ID, samples []metrics.MetricSample) error { + if w.retention == nil { + return nil + } + err := w.retention.AppendSamples(ctx, ringbuffer.Source{Kind: ringbuffer.SourceCheckShadow, ID: string(checkID)}, samples) + if err != nil { + return err + } + w.retention.ObserveSamples(samples) + return nil +} diff --git a/pkg/collector/metriclookback/shadow_writer_test.go b/pkg/collector/metriclookback/shadow_writer_test.go new file mode 100644 index 000000000000..15bc35245729 --- /dev/null +++ b/pkg/collector/metriclookback/shadow_writer_test.go @@ -0,0 +1,93 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package metriclookback + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" + corelookback "github.com/DataDog/datadog-agent/pkg/metriclookback" + "github.com/DataDog/datadog-agent/pkg/metriclookback/monitor" + "github.com/DataDog/datadog-agent/pkg/metriclookback/ringbuffer" + "github.com/DataDog/datadog-agent/pkg/metrics" +) + +func TestNewSenderManagerRequiresRetention(t *testing.T) { + require.Nil(t, NewSenderManager(context.Background(), "default-host", nil)) +} + +func TestNewSenderManagerWritesShadowCheckSamples(t *testing.T) { + retention := corelookback.NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + manager := NewSenderManager(context.Background(), "default-host", retention) + require.NotNil(t, manager) + + sender, err := manager.GetSender(checkid.ID("cpu:shadow")) + require.NoError(t, err) + sender.Gauge("shadow.metric", 42, "", []string{"env:staging"}) + sender.Commit() + + points := retention.PointsBetween( + ringbuffer.Source{Kind: ringbuffer.SourceCheckShadow, ID: "cpu:shadow"}, + "shadow.metric", + time.Unix(0, 0), + time.Now().Add(time.Minute), + ) + require.Len(t, points, 1) + require.Equal(t, float64(42), points[0].Value) + + series := retention.Series() + require.Len(t, series, 1) + require.Equal(t, "shadow.metric", series[0].Name) + require.Equal(t, "default-host", series[0].Host) + require.Equal(t, []string{"env:staging"}, series[0].Tags.UnsafeToReadOnlySliceString()) + require.Equal(t, metrics.CheckNameToMetricSource("cpu"), series[0].Source) + require.Equal(t, "System", series[0].SourceTypeName) +} + +func TestNewSenderManagerSamplesNotifyMonitor(t *testing.T) { + retention := corelookback.NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + decisions := make(chan monitor.Decision, 1) + watcher, err := monitor.New(monitor.Config{ + MetricName: "shadow.metric", + RangeEpsilon: 0.05, + EvaluationInterval: 2 * time.Second, + MinPoints: 2, + }, monitor.PointReaderFunc(func(metricName string, from, to time.Time) []monitor.Point { + points := retention.PointsBetweenSources([]ringbuffer.Source{{Kind: ringbuffer.SourceCheckShadow}}, metricName, from, to) + out := make([]monitor.Point, 0, len(points)) + for _, point := range points { + out = append(out, monitor.Point{Ts: point.Ts, Value: point.Value}) + } + return out + }), monitor.DecisionSinkFunc(func(decision monitor.Decision) { + decisions <- decision + })) + require.NoError(t, err) + retention.SetMonitor(watcher) + + manager := NewSenderManager(context.Background(), "default-host", retention) + sender, err := manager.GetSender(checkid.ID("cpu:shadow")) + require.NoError(t, err) + require.NoError(t, sender.GaugeWithTimestamp("shadow.metric", 40, "", nil, 10)) + sender.Commit() + require.NoError(t, sender.GaugeWithTimestamp("shadow.metric", 40.1, "", nil, 12)) + sender.Commit() + + select { + case decision := <-decisions: + require.Equal(t, monitor.Breach, decision.State) + require.Equal(t, "shadow.metric", decision.MetricName) + require.Equal(t, float64(40), decision.Min) + require.Equal(t, 40.1, decision.Max) + require.InDelta(t, 0.1, decision.Range, 1e-12) + case <-time.After(time.Second): + require.FailNow(t, "timed out waiting for monitor decision") + } +} diff --git a/pkg/collector/scheduler.go b/pkg/collector/scheduler.go index 5551962671ca..d15979fb517c 100644 --- a/pkg/collector/scheduler.go +++ b/pkg/collector/scheduler.go @@ -7,7 +7,6 @@ package collector import ( - "context" "expvar" "fmt" "slices" @@ -24,13 +23,10 @@ import ( filter "github.com/DataDog/datadog-agent/comp/core/workloadfilter/def" integrations "github.com/DataDog/datadog-agent/comp/logs/integrations/def" "github.com/DataDog/datadog-agent/pkg/aggregator/sender" - collectoraggregator "github.com/DataDog/datadog-agent/pkg/collector/aggregator" "github.com/DataDog/datadog-agent/pkg/collector/check" checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" corecheckLoader "github.com/DataDog/datadog-agent/pkg/collector/corechecks" "github.com/DataDog/datadog-agent/pkg/collector/loaders" - "github.com/DataDog/datadog-agent/pkg/collector/metriclookback" - "github.com/DataDog/datadog-agent/pkg/collector/metriclookback/lookbacksender" "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/util/infratags" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -74,8 +70,6 @@ type CheckScheduler struct { collector option.Option[collectorcomp.Component] senderManager sender.SenderManager shadowSenderManager sender.SenderManager - shadowSenderContext context.Context - shadowSenderCancel context.CancelFunc shadowCoreLoader check.Loader infraTagger *infratags.Tagger // nil = no infra mode tagging m sync.RWMutex @@ -99,6 +93,17 @@ func InitCheckScheduler(collector option.Option[collectorcomp.Component], sender return checkScheduler } +// SetMetricLookbackShadowSenderManager sets the sender manager used by metric +// lookback shadow checks loaded by this scheduler. +func (s *CheckScheduler) SetMetricLookbackShadowSenderManager(senderManager sender.SenderManager) { + if s == nil { + return + } + s.m.Lock() + defer s.m.Unlock() + s.shadowSenderManager = senderManager +} + // Schedule schedules configs to checks func (s *CheckScheduler) Schedule(configs []integration.Config) { if coll, ok := s.collector.Get(); ok { @@ -168,17 +173,8 @@ func (s *CheckScheduler) Unschedule(configs []integration.Config) { } } -// Stop releases scheduler-owned resources. -func (s *CheckScheduler) Stop() { - s.m.Lock() - defer s.m.Unlock() - - if s.shadowSenderCancel != nil { - s.shadowSenderCancel() - s.shadowSenderCancel = nil - s.shadowSenderContext = nil - } -} +// Stop satisfies the autodiscovery scheduler interface. +func (s *CheckScheduler) Stop() {} // addLoader adds a new Loader that AutoConfig can use to load a check. func (s *CheckScheduler) addLoader(loader check.Loader) { @@ -194,7 +190,7 @@ func (s *CheckScheduler) addLoader(loader check.Loader) { func (s *CheckScheduler) getChecks(config integration.Config, includeShadowChecks bool) ([]check.Check, error) { checks := []check.Check{} numLoaders := len(s.loaders) - var shadowCandidates map[int]metriclookback.ShadowCandidate + var shadowCandidates map[int]shadowCandidate if includeShadowChecks { shadowCandidates = shadowCandidatesByInstance(config) } @@ -293,18 +289,6 @@ func (s *CheckScheduler) shadowLoaderFor(loader check.Loader) (check.Loader, boo } } -func shadowCandidatesByInstance(config integration.Config) map[int]metriclookback.ShadowCandidate { - candidates := metriclookback.SelectShadowCandidates([]integration.Config{config}, metriclookback.ShadowPolicyOptionsFromConfig(setup.Datadog())) - if len(candidates) == 0 { - return nil - } - byInstance := make(map[int]metriclookback.ShadowCandidate, len(candidates)) - for _, candidate := range candidates { - byInstance[candidate.InstanceIndex] = candidate - } - return byInstance -} - func (s *CheckScheduler) loadCheckInstance(senderManager sender.SenderManager, config integration.Config, instance integration.Data, instanceIndex int, selectedInstanceLoader string) loadInstanceResult { result := loadInstanceResult{loaderErrors: make(map[string]error, len(s.loaders))} for _, loader := range s.loaders { @@ -336,67 +320,6 @@ func (s *CheckScheduler) applyInfraTagger(senderManager sender.SenderManager, ch chkSender.SetInfraTagger(s.infraTagger) } -func (s *CheckScheduler) ensureShadowSenderContext() context.Context { - if s.shadowSenderContext == nil { - s.shadowSenderContext, s.shadowSenderCancel = context.WithCancel(context.Background()) - } - return s.shadowSenderContext -} - -func (s *CheckScheduler) loadShadowCheck(candidate metriclookback.ShadowCandidate, loader check.Loader, sourceCheckID checkid.ID) (check.Check, error) { - shadowSenderManager := s.shadowSenderManager - if shadowSenderManager == nil { - shadowSenderManager = lookbacksender.NewSenderManager(s.ensureShadowSenderContext(), "", nil, nil) - s.shadowSenderManager = shadowSenderManager - } - shadowCheckID := check.ShadowID(sourceCheckID) - checkSenderManager := &shadowCheckSenderManager{ - SenderManager: shadowSenderManager, - shadowCheckID: shadowCheckID, - } - loadedCheck, err := loader.Load(checkSenderManager, candidate.SourceConfig, candidate.Instance, candidate.InstanceIndex) - if err != nil { - checkSenderManager.DestroySender(shadowCheckID) - return nil, err - } - if !checkSenderManager.RegisterCallbackID(loadedCheck.ID()) { - log.Warnf("Unable to register metric lookback rtloader callback route for shadow check %s loaded as %s", shadowCheckID, loadedCheck.ID()) - } - s.applyInfraTagger(checkSenderManager, candidate.SourceConfig.Name, shadowCheckID) - return check.NewShadowCheckForSource(loadedCheck, sourceCheckID, candidate.ShadowInterval, checkSenderManager), nil -} - -type shadowCheckSenderManager struct { - sender.SenderManager - shadowCheckID checkid.ID - unregisterCallbacks []func() -} - -func (m shadowCheckSenderManager) GetSender(checkid.ID) (sender.Sender, error) { - return m.SenderManager.GetSender(m.shadowCheckID) -} - -func (m shadowCheckSenderManager) SetSender(s sender.Sender, _ checkid.ID) error { - return m.SenderManager.SetSender(s, m.shadowCheckID) -} - -func (m *shadowCheckSenderManager) DestroySender(checkid.ID) { - for _, unregister := range m.unregisterCallbacks { - unregister() - } - m.unregisterCallbacks = nil - m.SenderManager.DestroySender(m.shadowCheckID) -} - -func (m *shadowCheckSenderManager) RegisterCallbackID(id checkid.ID) bool { - unregister, ok := collectoraggregator.RegisterCheckSenderManager(id, m) - if !ok { - return false - } - m.unregisterCallbacks = append(m.unregisterCallbacks, unregister) - return true -} - // GetChecksByNameForConfigs returns checks matching name for passed in configs func GetChecksByNameForConfigs(checkName string, configs []integration.Config) []check.Check { var checks []check.Check diff --git a/pkg/collector/scheduler_metriclookback.go b/pkg/collector/scheduler_metriclookback.go new file mode 100644 index 000000000000..94b49c2d7e57 --- /dev/null +++ b/pkg/collector/scheduler_metriclookback.go @@ -0,0 +1,88 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build python || test + +package collector + +import ( + "errors" + + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" + "github.com/DataDog/datadog-agent/pkg/aggregator/sender" + collectoraggregator "github.com/DataDog/datadog-agent/pkg/collector/aggregator" + "github.com/DataDog/datadog-agent/pkg/collector/check" + checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" + "github.com/DataDog/datadog-agent/pkg/collector/metriclookback" + "github.com/DataDog/datadog-agent/pkg/config/setup" + "github.com/DataDog/datadog-agent/pkg/util/log" +) + +type shadowCandidate = metriclookback.ShadowCandidate + +func shadowCandidatesByInstance(config integration.Config) map[int]shadowCandidate { + candidates := metriclookback.SelectShadowCandidates([]integration.Config{config}, metriclookback.ShadowPolicyOptionsFromConfig(setup.Datadog())) + if len(candidates) == 0 { + return nil + } + byInstance := make(map[int]shadowCandidate, len(candidates)) + for _, candidate := range candidates { + byInstance[candidate.InstanceIndex] = candidate + } + return byInstance +} + +func (s *CheckScheduler) loadShadowCheck(candidate shadowCandidate, loader check.Loader, sourceCheckID checkid.ID) (check.Check, error) { + shadowSenderManager := s.shadowSenderManager + if shadowSenderManager == nil { + return nil, errors.New("metric lookback shadow sender manager is not configured") + } + shadowCheckID := check.ShadowID(sourceCheckID) + checkSenderManager := &shadowCheckSenderManager{ + SenderManager: shadowSenderManager, + shadowCheckID: shadowCheckID, + } + loadedCheck, err := loader.Load(checkSenderManager, candidate.SourceConfig, candidate.Instance, candidate.InstanceIndex) + if err != nil { + checkSenderManager.DestroySender(shadowCheckID) + return nil, err + } + if !checkSenderManager.RegisterCallbackID(loadedCheck.ID()) { + log.Warnf("Unable to register metric lookback rtloader callback route for shadow check %s loaded as %s", shadowCheckID, loadedCheck.ID()) + } + s.applyInfraTagger(checkSenderManager, candidate.SourceConfig.Name, shadowCheckID) + return check.NewShadowCheckForSource(loadedCheck, sourceCheckID, candidate.ShadowInterval, checkSenderManager), nil +} + +type shadowCheckSenderManager struct { + sender.SenderManager + shadowCheckID checkid.ID + unregisterCallbacks []func() +} + +func (m shadowCheckSenderManager) GetSender(checkid.ID) (sender.Sender, error) { + return m.SenderManager.GetSender(m.shadowCheckID) +} + +func (m shadowCheckSenderManager) SetSender(s sender.Sender, _ checkid.ID) error { + return m.SenderManager.SetSender(s, m.shadowCheckID) +} + +func (m *shadowCheckSenderManager) DestroySender(checkid.ID) { + for _, unregister := range m.unregisterCallbacks { + unregister() + } + m.unregisterCallbacks = nil + m.SenderManager.DestroySender(m.shadowCheckID) +} + +func (m *shadowCheckSenderManager) RegisterCallbackID(id checkid.ID) bool { + unregister, ok := collectoraggregator.RegisterCheckSenderManager(id, m) + if !ok { + return false + } + m.unregisterCallbacks = append(m.unregisterCallbacks, unregister) + return true +} diff --git a/pkg/collector/scheduler_metriclookback_nopython.go b/pkg/collector/scheduler_metriclookback_nopython.go new file mode 100644 index 000000000000..cc2f81df7b73 --- /dev/null +++ b/pkg/collector/scheduler_metriclookback_nopython.go @@ -0,0 +1,27 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build !python && !test + +package collector + +import ( + "errors" + + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" + "github.com/DataDog/datadog-agent/pkg/collector/check" + checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" +) + +type shadowCandidate struct{} + +func shadowCandidatesByInstance(integration.Config) map[int]shadowCandidate { + // Metric lookback shadow checks are intentionally excluded from non-Python Agent binaries. + return nil +} + +func (s *CheckScheduler) loadShadowCheck(shadowCandidate, check.Loader, checkid.ID) (check.Check, error) { + return nil, errors.New("metric lookback shadow checks are disabled in non-Python Agent builds") +} diff --git a/pkg/collector/scheduler_test.go b/pkg/collector/scheduler_test.go index 7a7f67440ad6..7f95155d1fed 100644 --- a/pkg/collector/scheduler_test.go +++ b/pkg/collector/scheduler_test.go @@ -7,7 +7,6 @@ package collector import ( "bytes" - "context" "errors" "testing" "time" @@ -163,12 +162,12 @@ func TestGetChecksFromConfigsLoadsSelectedShadowCheckWithSenderManagerOverride(t loader, err := core.NewGoCheckLoader() require.NoError(t, err) s := CheckScheduler{ - configToChecks: make(map[string][]checkid.ID), - senderManager: normalSenderManager, - shadowSenderManager: shadowSenderManager, - infraTagger: infratags.NewTagger(cfg), + configToChecks: make(map[string][]checkid.ID), + senderManager: normalSenderManager, + infraTagger: infratags.NewTagger(cfg), } s.addLoader(loader) + s.SetMetricLookbackShadowSenderManager(shadowSenderManager) config := integration.Config{ Name: "cpu", @@ -248,7 +247,7 @@ func TestGetChecksFromConfigsDoesNotLoadShadowChecksWhenCacheIsNotPopulated(t *t assert.Empty(t, shadowSenderManager.requestedIDs) } -func TestGetChecksFromConfigsUsesFallbackShadowSenderManager(t *testing.T) { +func TestGetChecksFromConfigsKeepsNormalCheckWhenShadowSenderManagerMissing(t *testing.T) { core.WithTestCatalog(t) cfg := configmock.New(t) cfg.SetInTest("metric_lookback.enabled", true) @@ -274,19 +273,13 @@ func TestGetChecksFromConfigsUsesFallbackShadowSenderManager(t *testing.T) { checks := s.GetChecksFromConfigs([]integration.Config{config}, true) - require.Len(t, checks, 2) + require.Len(t, checks, 1) assert.False(t, check.IsShadow(checks[0])) - assert.True(t, check.IsShadow(checks[1])) - assert.Equal(t, []checkid.ID{checks[0].ID(), checks[1].ID()}, s.configToChecks[config.Digest()]) - assert.Equal(t, []core.LoadMode{core.NormalLoadMode, core.ShadowLoadMode}, modes) - require.Len(t, calls, 2) + assert.Equal(t, []checkid.ID{checks[0].ID()}, s.configToChecks[config.Digest()]) + assert.Equal(t, []core.LoadMode{core.NormalLoadMode}, modes) + require.Len(t, calls, 1) assert.Same(t, normalSenderManager, calls[0].senderManager) - assert.NotNil(t, s.shadowSenderManager) - assert.NotEqual(t, normalSenderManager, calls[1].senderManager) - shadowCallSenderManager, ok := calls[1].senderManager.(*shadowCheckSenderManager) - require.True(t, ok) - assert.Equal(t, s.shadowSenderManager, shadowCallSenderManager.SenderManager) - assert.Equal(t, checks[1].ID(), shadowCallSenderManager.shadowCheckID) + assert.Nil(t, s.shadowSenderManager) } func TestGetChecksFromConfigsKeepsNormalCheckWhenShadowLoadFails(t *testing.T) { @@ -325,33 +318,6 @@ func TestGetChecksFromConfigsKeepsNormalCheckWhenShadowLoadFails(t *testing.T) { assert.Equal(t, []checkid.ID{check.ShadowID(checks[0].ID())}, shadowSenderManager.destroyedIDs) } -func TestStopCancelsShadowSenderContext(t *testing.T) { - s := CheckScheduler{} - ctx := s.ensureShadowSenderContext() - - s.Stop() - - select { - case <-ctx.Done(): - assert.ErrorIs(t, ctx.Err(), context.Canceled) - case <-time.After(time.Second): - t.Fatal("timed out waiting for shadow sender context cancellation") - } -} - -func TestInitCheckSchedulerDoesNotCreateShadowSenderContext(t *testing.T) { - s := InitCheckScheduler( - option.None[collectorcomp.Component](), - &recordingSchedulerSenderManager{}, - option.None[integrations.Component](), - nil, - nil, - ) - - assert.Nil(t, s.shadowSenderContext) - assert.Nil(t, s.shadowSenderCancel) -} - func TestGetChecksFromConfigsSkipsShadowCheckForUnsupportedLoader(t *testing.T) { cfg := configmock.New(t) cfg.SetInTest("metric_lookback.enabled", true) diff --git a/pkg/config/config_template.yaml b/pkg/config/config_template.yaml index 9c42fe8b4803..645f62226e90 100644 --- a/pkg/config/config_template.yaml +++ b/pkg/config/config_template.yaml @@ -2663,6 +2663,117 @@ api_key: # # dogstatsd_no_aggregation_pipeline_workers_count: 1 +## @param metric_lookback - custom object - optional +## Configure the local metric lookback rings. The first supported DogStatsD use +## case captures selected normal DogStatsD bucketed points/sketches and final +## no-aggregation points by exact name. An optional monitor can forward retained +## high-resolution ranges after a watched metric breaches its configured +## short-window max-minus-min range threshold. +# +# metric_lookback: + +# # @param enabled - boolean - optional - default: false +# # @env DD_METRIC_LOOKBACK_ENABLED - boolean - optional - default: false +# # Enable metric lookback globally. For shadow checks, this enables shadowing for checks allowed +# # by enabled_checks. +# +# enabled: false + +# # @param enabled_checks - list of strings - optional - default: [] +# # @env DD_METRIC_LOOKBACK_ENABLED_CHECKS - space-separated list of strings - optional - default: [] +# # Optional check-name allowlist for metric lookback shadow checks. When empty, all supported +# # checks are eligible. +# +# enabled_checks: [] + +# # @param capacity - integer - optional - default: 262144 +# # @env DD_METRIC_LOOKBACK_CAPACITY - integer - optional - default: 262144 +# # Total number of records retained across each lookback ring's shards. Series points and +# # sketches use separate rings. +# +# capacity: 262144 + +# # @param shard_count - integer - optional - default: 16 +# # @env DD_METRIC_LOOKBACK_SHARD_COUNT - integer - optional - default: 16 +# # Number of independent shards used by each lookback ring. +# +# shard_count: 16 + +# # @param dogstatsd - custom object - optional +# # Configure exact-name DogStatsD admission into metric lookback. +# +# dogstatsd: + +# # @param metric_names - list of strings - optional - default: [] +# # @env DD_METRIC_LOOKBACK_DOGSTATSD_METRIC_NAMES - space-separated list of strings - optional - default: [] +# # Additional exact DogStatsD metric names to retain in the lookback ring. The monitor metric is +# # auto-admitted from DogStatsD when monitor.mode is dry_run or enabled and does not need to be repeated here. +# +# metric_names: [] + +# # @param monitor - custom object - optional +# # Configure a metric lookback monitor for egress. +# +# monitor: + +# # @param mode - string - optional - default: "disabled" +# # @env DD_METRIC_LOOKBACK_MONITOR_MODE - string - optional - default: "disabled" +# # Controls monitor operation. Valid values are: +# # - disabled: do not run the monitor or monitor-driven egress. +# # - dry_run: run the monitor, log state transitions, and keep retained ranges eligible for intake forwarding. +# # - enabled: run the monitor and let monitor decisions control egress. +# +# mode: "disabled" + +# # @param metric_name - string - optional - default: "" +# # @env DD_METRIC_LOOKBACK_MONITOR_METRIC_NAME - string - optional - default: "" +# # Exact metric name watched by the monitor. +# +# metric_name: "" + +# # @param evaluation_interval - duration - optional - default: "30s" +# # @env DD_METRIC_LOOKBACK_MONITOR_EVALUATION_INTERVAL - duration - optional - default: "30s" +# # Approximate width of each monitor evaluation window. Shorter windows can detect +# # breaches sooner; longer windows evaluate more retained points together. +# +# evaluation_interval: "30s" + +# # @param range_epsilon - float - optional - default: 0 +# # @env DD_METRIC_LOOKBACK_MONITOR_RANGE_EPSILON - float - optional - default: 0 +# # Monitor max-minus-min threshold. The Agent fails to start when this value is negative. +# # The monitor computes a window range as the maximum retained point value minus the minimum +# # retained point value in each completed evaluation window, or in each configured tag partition. +# # Egress forwards retained ranges after that window range is greater than this epsilon. +# +# range_epsilon: 0 + +# # @param partition_tags - list of strings - optional - default: [] +# # @env DD_METRIC_LOOKBACK_MONITOR_PARTITION_TAGS - space-separated list of strings - optional - default: [] +# # Tag keys used to partition monitor range evaluation. When empty, all retained points for the +# # monitor metric are evaluated together. When set, the monitor evaluates each selected tag-value +# # tuple independently and breaches if any partition exceeds range_epsilon. +# +# partition_tags: [] + +# # @param egress - custom object - optional +# # Configure monitor-triggered egress ranges. +# +# egress: + +# # @param pre_trigger_window - duration - optional - default: "0s" +# # @env DD_METRIC_LOOKBACK_EGRESS_PRE_TRIGGER_WINDOW - duration - optional - default: "0s" +# # Extra retained time to forward before a breach or unknown monitor window. +# # This captures context before the monitor-triggering window. +# +# pre_trigger_window: "0s" + +# # @param post_recovery_window - duration - optional - default: "30s" +# # @env DD_METRIC_LOOKBACK_EGRESS_POST_RECOVERY_WINDOW - duration - optional - default: "30s" +# # Extra retained time to forward after the first healthy monitor window that +# # suppresses egress. The default approximates the previous two-healthy-window behavior. +# +# post_recovery_window: "30s" + ## @param statsd_forward_host - string - optional - default: "" ## @env DD_STATSD_FORWARD_HOST - string - optional - default: "" ## Forward every packet received by the DogStatsD server to another statsd server. diff --git a/pkg/config/schema/yaml/core_schema.yaml b/pkg/config/schema/yaml/core_schema.yaml index 0c59cc9e903e..d6026de194b8 100644 --- a/pkg/config/schema/yaml/core_schema.yaml +++ b/pkg/config/schema/yaml/core_schema.yaml @@ -2453,6 +2453,170 @@ properties: tags: - template_section:Dogstatsd comment: How many workers are used by the no-aggregation pipeline. + metric_lookback: + node_type: section + type: object + visibility: public + description: |- + Configure the local metric lookback rings. The first supported DogStatsD use + case captures selected normal DogStatsD bucketed points/sketches and final + no-aggregation points by exact name. An optional monitor can forward retained + high-resolution ranges after a watched metric breaches its configured + short-window max-minus-min range threshold. + tags: + - template_section:Dogstatsd + - full-agent-only:true + properties: + enabled: + node_type: setting + type: boolean + default: false + visibility: public + description: |- + Enable metric lookback globally. For shadow checks, this enables shadowing for checks allowed + by enabled_checks. + tags: + - full-agent-only:true + enabled_checks: + node_type: setting + type: array + default: [] + items: + type: string + visibility: public + description: |- + Optional check-name allowlist for metric lookback shadow checks. When empty, all supported + checks are eligible. + tags: + - full-agent-only:true + capacity: + node_type: setting + type: integer + default: 262144 + visibility: public + description: |- + Total number of records retained across each lookback ring's shards. Series points and + sketches use separate rings. + shard_count: + node_type: setting + type: integer + default: 16 + visibility: public + description: Number of independent shards used by each lookback ring. + dogstatsd: + node_type: section + type: object + visibility: public + description: Configure exact-name DogStatsD admission into metric lookback. + tags: + - template_section:Dogstatsd + properties: + metric_names: + node_type: setting + type: array + default: [] + items: + type: string + visibility: public + description: |- + Additional exact DogStatsD metric names to retain in the lookback ring. The monitor metric is + auto-admitted from DogStatsD when monitor.mode is dry_run or enabled and does not need to be repeated here. + monitor: + node_type: section + type: object + visibility: public + description: Configure a metric lookback monitor for egress. + tags: + - template_section:Dogstatsd + properties: + mode: + node_type: setting + type: string + default: disabled + visibility: public + description: |- + Controls monitor operation. Valid values are: + - disabled: do not run the monitor or monitor-driven egress. + - dry_run: run the monitor, log state transitions, and keep retained ranges eligible for intake forwarding. + - enabled: run the monitor and let monitor decisions control egress. + metric_name: + node_type: setting + type: string + default: '' + visibility: public + description: Exact metric name watched by the monitor. + evaluation_interval: + node_type: setting + type: string + default: 30s + format: duration + visibility: public + description: |- + Approximate width of each monitor evaluation window. Shorter windows can detect + breaches sooner; longer windows evaluate more retained points together. + tags: + - golang_type:duration + range_epsilon: + node_type: setting + type: number + default: 0 + visibility: public + description: |- + Monitor max-minus-min threshold. The Agent fails to start when this value is negative. + The monitor computes a window range as the maximum retained point value minus the minimum + retained point value in each completed evaluation window, or in each configured tag partition. + Egress forwards retained ranges after that window range is greater than this epsilon. + tags: + - golang_type:float64 + partition_tags: + node_type: setting + type: array + default: [] + items: + type: string + visibility: public + description: |- + Tag keys used to partition monitor range evaluation. When empty, all retained points for the + monitor metric are evaluated together. When set, the monitor evaluates each selected tag-value + tuple independently and breaches if any partition exceeds range_epsilon. + egress: + node_type: section + type: object + visibility: public + description: Configure monitor-triggered egress ranges. + tags: + - template_section:Dogstatsd + properties: + pre_trigger_window: + node_type: setting + type: string + default: 0s + format: duration + visibility: public + description: |- + Extra retained time to forward before a breach or unknown monitor window. + This captures context before the monitor-triggering window. + tags: + - golang_type:duration + post_recovery_window: + node_type: setting + type: string + default: 30s + format: duration + visibility: public + description: |- + Extra retained time to forward after the first healthy monitor window that + suppresses egress. The default approximates the previous two-healthy-window behavior. + tags: + - golang_type:duration + collection_interval: + node_type: setting + type: string + default: 1s + format: duration + tags: + - golang_type:duration + - full-agent-only:true statsd_forward_host: node_type: setting type: string @@ -9249,34 +9413,6 @@ properties: default: false tags: - full-agent-only:true - metric_lookback: - node_type: section - type: object - properties: - collection_interval: - node_type: setting - type: string - default: 1s - format: duration - tags: - - golang_type:duration - - full-agent-only:true - enabled: - node_type: setting - type: boolean - default: false - tags: - - full-agent-only:true - enabled_checks: - node_type: setting - type: array - default: [] - items: - type: string - tags: - - full-agent-only:true - tags: - - full-agent-only:true metric_tag_filterlist: node_type: setting type: array diff --git a/pkg/config/setup/common_settings.go b/pkg/config/setup/common_settings.go index 3cf9699977e1..9b5d78ac4df7 100644 --- a/pkg/config/setup/common_settings.go +++ b/pkg/config/setup/common_settings.go @@ -37,6 +37,16 @@ func initCoreAgentFull(config pkgconfigmodel.Setup) { config.BindEnvAndSetDefault("metric_lookback.enabled", false) config.BindEnvAndSetDefault("metric_lookback.enabled_checks", []string{}) config.BindEnvAndSetDefault("metric_lookback.collection_interval", time.Second) + config.BindEnvAndSetDefault("metric_lookback.capacity", 262144) + config.BindEnvAndSetDefault("metric_lookback.shard_count", 16) + config.BindEnvAndSetDefault("metric_lookback.dogstatsd.metric_names", []string{}) + config.BindEnvAndSetDefault("metric_lookback.monitor.mode", "disabled") + config.BindEnvAndSetDefault("metric_lookback.monitor.metric_name", "") + config.BindEnvAndSetDefault("metric_lookback.monitor.evaluation_interval", 30*time.Second) + config.BindEnvAndSetDefault("metric_lookback.monitor.range_epsilon", 0.0) + config.BindEnvAndSetDefault("metric_lookback.monitor.partition_tags", []string{}) + config.BindEnvAndSetDefault("metric_lookback.egress.pre_trigger_window", 0*time.Second) + config.BindEnvAndSetDefault("metric_lookback.egress.post_recovery_window", 30*time.Second) config.BindEnvAndSetDefault("host_aliases", []string{}) config.BindEnvAndSetDefault("collect_ccrid", true) diff --git a/pkg/config/setup/config_test.go b/pkg/config/setup/config_test.go index 0d783ab2cb1d..b4af5aa0be51 100644 --- a/pkg/config/setup/config_test.go +++ b/pkg/config/setup/config_test.go @@ -110,18 +110,36 @@ func TestMetricLookbackDefaults(t *testing.T) { assert.False(t, config.GetBool("metric_lookback.enabled")) assert.Empty(t, config.GetStringSlice("metric_lookback.enabled_checks")) assert.Equal(t, time.Second, config.GetDuration("metric_lookback.collection_interval")) + assert.Equal(t, "disabled", config.GetString("metric_lookback.monitor.mode")) + assert.Equal(t, 30*time.Second, config.GetDuration("metric_lookback.monitor.evaluation_interval")) + assert.Equal(t, 0.0, config.GetFloat64("metric_lookback.monitor.range_epsilon")) + assert.Empty(t, config.GetStringSlice("metric_lookback.monitor.partition_tags")) + assert.Equal(t, 0*time.Second, config.GetDuration("metric_lookback.egress.pre_trigger_window")) + assert.Equal(t, 30*time.Second, config.GetDuration("metric_lookback.egress.post_recovery_window")) } func TestMetricLookbackEnvOverride(t *testing.T) { t.Setenv("DD_METRIC_LOOKBACK_ENABLED", "true") t.Setenv("DD_METRIC_LOOKBACK_ENABLED_CHECKS", `["cpu","disk"]`) t.Setenv("DD_METRIC_LOOKBACK_COLLECTION_INTERVAL", "3s") + t.Setenv("DD_METRIC_LOOKBACK_MONITOR_MODE", "dry_run") + t.Setenv("DD_METRIC_LOOKBACK_MONITOR_EVALUATION_INTERVAL", "10s") + t.Setenv("DD_METRIC_LOOKBACK_MONITOR_RANGE_EPSILON", "0.05") + t.Setenv("DD_METRIC_LOOKBACK_MONITOR_PARTITION_TAGS", `["az","instance_type"]`) + t.Setenv("DD_METRIC_LOOKBACK_EGRESS_PRE_TRIGGER_WINDOW", "5s") + t.Setenv("DD_METRIC_LOOKBACK_EGRESS_POST_RECOVERY_WINDOW", "20s") config := newTestConf(t) assert.True(t, config.GetBool("metric_lookback.enabled")) assert.Equal(t, []string{"cpu", "disk"}, config.GetStringSlice("metric_lookback.enabled_checks")) assert.Equal(t, 3*time.Second, config.GetDuration("metric_lookback.collection_interval")) + assert.Equal(t, "dry_run", config.GetString("metric_lookback.monitor.mode")) + assert.Equal(t, 10*time.Second, config.GetDuration("metric_lookback.monitor.evaluation_interval")) + assert.Equal(t, 0.05, config.GetFloat64("metric_lookback.monitor.range_epsilon")) + assert.Equal(t, []string{"az", "instance_type"}, config.GetStringSlice("metric_lookback.monitor.partition_tags")) + assert.Equal(t, 5*time.Second, config.GetDuration("metric_lookback.egress.pre_trigger_window")) + assert.Equal(t, 20*time.Second, config.GetDuration("metric_lookback.egress.post_recovery_window")) } func TestMetricLookbackYAML(t *testing.T) { @@ -132,11 +150,27 @@ metric_lookback: enabled_checks: - cpu - disk + monitor: + mode: dry_run + evaluation_interval: 15s + range_epsilon: 0.05 + partition_tags: + - az + - instance_type + egress: + pre_trigger_window: 7s + post_recovery_window: 21s `) assert.True(t, cfg.GetBool("metric_lookback.enabled")) assert.Equal(t, []string{"cpu", "disk"}, cfg.GetStringSlice("metric_lookback.enabled_checks")) assert.Equal(t, 5*time.Second, cfg.GetDuration("metric_lookback.collection_interval")) + assert.Equal(t, "dry_run", cfg.GetString("metric_lookback.monitor.mode")) + assert.Equal(t, 15*time.Second, cfg.GetDuration("metric_lookback.monitor.evaluation_interval")) + assert.Equal(t, 0.05, cfg.GetFloat64("metric_lookback.monitor.range_epsilon")) + assert.Equal(t, []string{"az", "instance_type"}, cfg.GetStringSlice("metric_lookback.monitor.partition_tags")) + assert.Equal(t, 7*time.Second, cfg.GetDuration("metric_lookback.egress.pre_trigger_window")) + assert.Equal(t, 21*time.Second, cfg.GetDuration("metric_lookback.egress.post_recovery_window")) } func TestUnexpectedUnicode(t *testing.T) { diff --git a/pkg/metriclookback/BUILD.bazel b/pkg/metriclookback/BUILD.bazel new file mode 100644 index 000000000000..595c029ed064 --- /dev/null +++ b/pkg/metriclookback/BUILD.bazel @@ -0,0 +1,41 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "metriclookback", + srcs = [ + "egress_controller.go", + "egress_policy.go", + "retention.go", + "sketch_projection.go", + ], + importpath = "github.com/DataDog/datadog-agent/pkg/metriclookback", + visibility = ["//visibility:public"], + deps = [ + "//comp/core/telemetry/impl", + "//pkg/metriclookback/monitor", + "//pkg/metriclookback/ringbuffer", + "//pkg/metrics", + "//pkg/serializer", + ], +) + +go_test( + name = "metriclookback_test", + srcs = [ + "egress_controller_test.go", + "egress_policy_test.go", + "retention_test.go", + ], + embed = [":metriclookback"], + gotags = ["test"], + deps = [ + "//pkg/metriclookback/monitor", + "//pkg/metriclookback/ringbuffer", + "//pkg/metrics", + "//pkg/serializer/mocks", + "//pkg/tagset", + "//pkg/util/quantile", + "@com_github_stretchr_testify//mock", + "@com_github_stretchr_testify//require", + ], +) diff --git a/pkg/metriclookback/dogstatsd/BUILD.bazel b/pkg/metriclookback/dogstatsd/BUILD.bazel new file mode 100644 index 000000000000..12e25cd49196 --- /dev/null +++ b/pkg/metriclookback/dogstatsd/BUILD.bazel @@ -0,0 +1,44 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "dogstatsd", + srcs = [ + "adapter.go", + "bucket.go", + ], + importpath = "github.com/DataDog/datadog-agent/pkg/metriclookback/dogstatsd", + visibility = ["//visibility:public"], + deps = [ + "//comp/core/telemetry/impl", + "//pkg/aggregator", + "//pkg/aggregator/ckey", + "//pkg/config/setup", + "//pkg/metriclookback", + "//pkg/metriclookback/monitor", + "//pkg/metriclookback/ringbuffer", + "//pkg/metrics", + "//pkg/tagset", + "//pkg/util/quantile", + ], +) + +go_test( + name = "dogstatsd_test", + srcs = [ + "adapter_test.go", + "bucket_test.go", + ], + embed = [":dogstatsd"], + gotags = ["test"], + deps = [ + "//pkg/aggregator", + "//pkg/aggregator/ckey", + "//pkg/config/setup", + "//pkg/metriclookback", + "//pkg/metriclookback/monitor", + "//pkg/metriclookback/ringbuffer", + "//pkg/metrics", + "//pkg/tagset", + "@com_github_stretchr_testify//require", + ], +) diff --git a/pkg/metriclookback/dogstatsd/adapter.go b/pkg/metriclookback/dogstatsd/adapter.go new file mode 100644 index 000000000000..1de6eb72a0fb --- /dev/null +++ b/pkg/metriclookback/dogstatsd/adapter.go @@ -0,0 +1,155 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package dogstatsd adapts DogStatsD aggregation streams to metric lookback retention. +package dogstatsd + +import ( + "context" + "time" + + "github.com/DataDog/datadog-agent/pkg/aggregator" + "github.com/DataDog/datadog-agent/pkg/metriclookback" + "github.com/DataDog/datadog-agent/pkg/metriclookback/monitor" + "github.com/DataDog/datadog-agent/pkg/metriclookback/ringbuffer" + "github.com/DataDog/datadog-agent/pkg/metrics" +) + +// DogStatsDOptions controls selected DogStatsD lookback admission. +type DogStatsDOptions struct { + // MetricNames is the exact-name allow-list retained from DogStatsD. Non- + // matching series/samples are ignored without allocation. + MetricNames []string + + // Monitor is an optional materialized view over the same admitted stream. Its + // metric name is automatically admitted even if MetricNames is empty. + Monitor *monitor.Watcher + + // BucketMaterializer receives selected normal DogStatsD samples. It is nil + // when only no-aggregation final series should be retained. + BucketMaterializer *DogStatsDBucketMaterializer + + // EgressController is stopped with the adapter when the owning demultiplexer + // shuts down. It is nil when monitor egress is disabled. + EgressController *metriclookback.EgressController +} + +// DogStatsDAdapter admits selected DogStatsD observations into a shared +// Retention ring and updates an optional monitor view from the same admitted +// stream. Normal DogStatsD samples flow through a bucket materializer before +// retention; no-aggregation series are already final and are retained directly. +type DogStatsDAdapter struct { + retention *metriclookback.Retention + materializer *DogStatsDBucketMaterializer + single string + names map[string]struct{} + monitor *monitor.Watcher + egressController *metriclookback.EgressController +} + +// NewDogStatsDAdapter creates an exact-name DogStatsD lookback adapter. It +// returns nil when retention is nil or no names are selected. +func NewDogStatsDAdapter(retention *metriclookback.Retention, opts DogStatsDOptions) *DogStatsDAdapter { + if retention == nil { + return nil + } + + seen := make(map[string]struct{}, len(opts.MetricNames)+1) + for _, name := range opts.MetricNames { + if name == "" { + continue + } + seen[name] = struct{}{} + } + if opts.Monitor != nil { + if monitorName := opts.Monitor.MetricName(); monitorName != "" { + seen[monitorName] = struct{}{} + } + } + if len(seen) == 0 { + return nil + } + + adapter := &DogStatsDAdapter{retention: retention, materializer: opts.BucketMaterializer, monitor: opts.Monitor, egressController: opts.EgressController} + if len(seen) == 1 { + for name := range seen { + adapter.single = name + } + return adapter + } + + adapter.names = seen + return adapter +} + +// WantsDogStatsDMetric returns whether name is admitted by the exact-name +// selection used by DogStatsD lookback. It does not allocate for nonmatches. +func (a *DogStatsDAdapter) WantsDogStatsDMetric(name string) bool { + return a != nil && a.match(name) +} + +// ObserveDogStatsDSample admits selected normal DogStatsD samples into the +// bucket materializer. Non-matching samples do not allocate and do not update +// monitor state. +func (a *DogStatsDAdapter) ObserveDogStatsDSample(sample *metrics.MetricSample, timestamp float64, ctx aggregator.DogStatsDLookbackContext) { + if a == nil || sample == nil || a.materializer == nil || !a.match(sample.Name) { + return + } + a.materializer.Observe(sample, timestamp, ctx) +} + +// FlushDogStatsDBuckets seals any selected normal-DogStatsD buckets that are old +// enough to become immutable lookback points. +func (a *DogStatsDAdapter) FlushDogStatsDBuckets(timestamp float64, forceFlushAll bool) { + if a == nil || a.materializer == nil { + return + } + if forceFlushAll { + a.materializer.FlushAll(timestamp) + return + } + a.materializer.Flush(timestamp) +} + +// Stop stops background resources owned by the adapter. +func (a *DogStatsDAdapter) Stop() { + if a == nil || a.egressController == nil { + return + } + a.egressController.Stop() +} + +// AppendDogStatsDNoAggSerie admits selected no-aggregation series after the +// worker has applied the same tag enrichment, type mapping, and value +// normalization used for normal serialization. Non-matching series do not +// allocate and do not update monitor state. +func (a *DogStatsDAdapter) AppendDogStatsDNoAggSerie(serie *metrics.Serie) { + if a == nil || serie == nil || len(serie.Points) == 0 || !a.match(serie.Name) { + return + } + + _ = a.retention.AppendSerie(context.Background(), ringbuffer.Source{Kind: ringbuffer.SourceDogStatsDNoAggregation}, serie) + if a.monitor == nil { + return + } + for i := range serie.Points { + a.monitor.Observe(serie.Name, pointObservedAt(serie.Points[i])) + } +} + +func pointObservedAt(point metrics.Point) time.Time { + if point.Ts > 0 { + return time.UnixMicro(int64(point.Ts * 1e6)) + } + return time.Now() +} + +func (a *DogStatsDAdapter) match(name string) bool { + if a.single != "" { + return name == a.single + } + _, found := a.names[name] + return found +} diff --git a/pkg/metriclookback/dogstatsd/adapter_test.go b/pkg/metriclookback/dogstatsd/adapter_test.go new file mode 100644 index 000000000000..7b2193bc2e6d --- /dev/null +++ b/pkg/metriclookback/dogstatsd/adapter_test.go @@ -0,0 +1,140 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package dogstatsd + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/pkg/aggregator" + "github.com/DataDog/datadog-agent/pkg/aggregator/ckey" + "github.com/DataDog/datadog-agent/pkg/metriclookback" + "github.com/DataDog/datadog-agent/pkg/metriclookback/monitor" + "github.com/DataDog/datadog-agent/pkg/metriclookback/ringbuffer" + "github.com/DataDog/datadog-agent/pkg/metrics" + "github.com/DataDog/datadog-agent/pkg/tagset" +) + +func TestDogStatsDAdapterDoesNotAppendUntilFirstScalarMatch(t *testing.T) { + retention := metriclookback.NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + adapter := NewDogStatsDAdapter(retention, DogStatsDOptions{MetricNames: []string{"target.metric"}}) + require.NotNil(t, adapter) + + adapter.AppendDogStatsDNoAggSerie(noAggSerie("other.metric", 1, 10, nil)) + require.Zero(t, retention.Stats().Records) + + adapter.AppendDogStatsDNoAggSerie(noAggSerie("target.metric", 2, 11, nil)) + require.Equal(t, 1, retention.Stats().Records) +} + +func TestDogStatsDAdapterAdmitsOnlySelectedNames(t *testing.T) { + retention := metriclookback.NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + adapter := NewDogStatsDAdapter(retention, DogStatsDOptions{MetricNames: []string{"target.metric"}}) + require.NotNil(t, adapter) + + adapter.AppendDogStatsDNoAggSerie(noAggSerie("other.metric", 1, 10, nil)) + adapter.AppendDogStatsDNoAggSerie(noAggSerie("target.metric", 2, 11, []string{"client:tag", "origin:tag"})) + + series := retention.Series() + require.Len(t, series, 1) + require.Equal(t, "target.metric", series[0].Name) + require.Equal(t, float64(2), series[0].Points[0].Value) + require.Equal(t, int64(10), series[0].Interval) + require.Equal(t, metrics.APIGaugeType, series[0].MType) + require.Equal(t, []string{"client:tag", "origin:tag"}, series[0].Tags.UnsafeToReadOnlySliceString()) + + stats := retention.Stats() + require.Equal(t, 1, stats.Records) + require.Equal(t, uint64(1), stats.AppendedSamples) +} + +func TestDogStatsDAdapterUsesMonitorMetricAsAdmissionName(t *testing.T) { + retention := metriclookback.NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + watcher := newNoopWatcher(t, "monitor.metric") + adapter := NewDogStatsDAdapter(retention, DogStatsDOptions{Monitor: watcher}) + require.NotNil(t, adapter) + + adapter.AppendDogStatsDNoAggSerie(noAggSerie("monitor.metric", 11, 11, nil)) + + series := retention.Series() + require.Len(t, series, 1) + require.Equal(t, "monitor.metric", series[0].Name) + require.Equal(t, uint64(0), watcher.Breaches()) +} + +func TestDogStatsDAdapterDoesNotMonitorNonAdmittedSamples(t *testing.T) { + retention := metriclookback.NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + watcher := newNoopWatcher(t, "monitor.metric") + adapter := NewDogStatsDAdapter(retention, DogStatsDOptions{MetricNames: []string{"stored.metric"}, Monitor: watcher}) + require.NotNil(t, adapter) + + adapter.AppendDogStatsDNoAggSerie(noAggSerie("other.metric", 100, 11, nil)) + + require.Empty(t, retention.Series()) + require.Equal(t, uint64(0), watcher.Breaches()) +} + +func TestDogStatsDAdapterIgnoresNilAndEmptySeries(t *testing.T) { + retention := metriclookback.NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + adapter := NewDogStatsDAdapter(retention, DogStatsDOptions{MetricNames: []string{"target.metric"}}) + require.NotNil(t, adapter) + + adapter.AppendDogStatsDNoAggSerie(nil) + adapter.AppendDogStatsDNoAggSerie(&metrics.Serie{Name: "target.metric"}) + + require.Empty(t, retention.Series()) +} + +func TestDogStatsDAdapterRoutesSelectedNormalSamplesToBucketMaterializer(t *testing.T) { + retention := metriclookback.NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + materializer := NewDogStatsDBucketMaterializer(retention, DogStatsDBucketMaterializerOptions{SealDelay: -1, ShardCount: 1}) + adapter := NewDogStatsDAdapter(retention, DogStatsDOptions{ + MetricNames: []string{"target.metric"}, + BucketMaterializer: materializer, + }) + require.NotNil(t, adapter) + require.True(t, adapter.WantsDogStatsDMetric("target.metric")) + require.False(t, adapter.WantsDogStatsDMetric("other.metric")) + + adapter.ObserveDogStatsDSample(&metrics.MetricSample{Name: "other.metric", Value: 10, Mtype: metrics.GaugeType, SampleRate: 1}, 10.1, aggregator.DogStatsDLookbackContext{ContextKey: ckey.ContextKey(1), Name: "other.metric"}) + adapter.ObserveDogStatsDSample(&metrics.MetricSample{Name: "target.metric", Value: 2, Mtype: metrics.GaugeType, SampleRate: 1}, 10.1, aggregator.DogStatsDLookbackContext{ + ContextKey: ckey.ContextKey(2), + Name: "target.metric", + Host: "host", + Tags: []string{"env:test"}, + }) + adapter.FlushDogStatsDBuckets(11, false) + + series := retention.Series() + require.Len(t, series, 1) + require.Equal(t, "target.metric", series[0].Name) + require.Equal(t, []metrics.Point{{Ts: 10, Value: 2}}, series[0].Points) + require.Equal(t, []string{"env:test"}, series[0].Tags.UnsafeToReadOnlySliceString()) +} + +func newNoopWatcher(t *testing.T, metricName string) *monitor.Watcher { + t.Helper() + watcher, err := monitor.New(monitor.Config{MetricName: metricName}, monitor.PointReaderFunc(func(_ string, _, _ time.Time) []monitor.Point { + return nil + }), monitor.DecisionSinkFunc(func(monitor.Decision) { + require.FailNow(t, "decision should not be emitted") + })) + require.NoError(t, err) + return watcher +} + +func noAggSerie(name string, value float64, ts float64, tags []string) *metrics.Serie { + return &metrics.Serie{ + Name: name, + Points: []metrics.Point{{Ts: ts, Value: value}}, + Tags: tagset.CompositeTagsFromSlice(tags), + Host: "host", + MType: metrics.APIGaugeType, + Interval: 10, + } +} diff --git a/pkg/metriclookback/dogstatsd/bucket.go b/pkg/metriclookback/dogstatsd/bucket.go new file mode 100644 index 000000000000..f047f6dd6802 --- /dev/null +++ b/pkg/metriclookback/dogstatsd/bucket.go @@ -0,0 +1,521 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package dogstatsd + +import ( + "context" + "math" + "sort" + "sync" + "time" + + telemetryimpl "github.com/DataDog/datadog-agent/comp/core/telemetry/impl" + "github.com/DataDog/datadog-agent/pkg/aggregator" + "github.com/DataDog/datadog-agent/pkg/aggregator/ckey" + pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" + "github.com/DataDog/datadog-agent/pkg/metriclookback" + "github.com/DataDog/datadog-agent/pkg/metriclookback/monitor" + "github.com/DataDog/datadog-agent/pkg/metriclookback/ringbuffer" + "github.com/DataDog/datadog-agent/pkg/metrics" + "github.com/DataDog/datadog-agent/pkg/tagset" + "github.com/DataDog/datadog-agent/pkg/util/quantile" +) + +const ( + // DefaultDogStatsDBucketWidth is the initial short-resolution bucket width for + // normal DogStatsD lookback materialization. It is an internal default so the + // code is not architecturally tied to exactly one second. + DefaultDogStatsDBucketWidth = time.Second + // DefaultDogStatsDSealDelay is the internal delay before open normal-DogStatsD + // lookback buckets are sealed into the retention ring. + DefaultDogStatsDSealDelay = 10 * time.Second + // DefaultDogStatsDMaterializerShardCount is the number of independent locks + // used by the selected DogStatsD bucket materializer. + DefaultDogStatsDMaterializerShardCount = 16 +) + +var ( + tlmDogStatsDBucketSamples = telemetryimpl.GetCompatComponent().NewCounter("metric_lookback", "dogstatsd_bucket_samples", []string{"state"}, "Count of selected normal DogStatsD samples handled by the metric lookback bucket materializer") + tlmDogStatsDBucketSealed = telemetryimpl.GetCompatComponent().NewSimpleCounter("metric_lookback", "dogstatsd_bucket_sealed", "Count of normal DogStatsD lookback buckets sealed into the retention ring") + tlmDogStatsDBucketPoints = telemetryimpl.GetCompatComponent().NewSimpleCounter("metric_lookback", "dogstatsd_bucket_points", "Count of normal DogStatsD lookback scalar points sealed into the retention ring") + tlmDogStatsDBucketSketches = telemetryimpl.GetCompatComponent().NewSimpleCounter("metric_lookback", "dogstatsd_bucket_sketches", "Count of normal DogStatsD lookback sketch points sealed into the retention ring") +) + +// DogStatsDBucketMaterializerOptions controls selected normal-DogStatsD bucket +// materialization. These are intentionally code-level options for now; the +// public config only enables DogStatsD lookback and selects metric names. +type DogStatsDBucketMaterializerOptions struct { + // BucketWidth controls the lookback materialization resolution. It defaults to + // DefaultDogStatsDBucketWidth and is normalized to a positive whole number of + // seconds because metrics.Serie.Interval is second-granularity. + BucketWidth time.Duration + // SealDelay delays sealing recent buckets so in-flight samples have time to + // arrive. It defaults to DefaultDogStatsDSealDelay. Set it to a negative value + // in tests to disable the delay. + SealDelay time.Duration + // ShardCount controls materializer lock sharding. It defaults to + // DefaultDogStatsDMaterializerShardCount. + ShardCount int + // ContextExpiry controls how long selected series descriptors stay available + // after their last real sample. It defaults to dogstatsd_context_expiry_seconds. + ContextExpiry time.Duration + // CounterExpiry controls how long selected counters keep emitting zero buckets + // after their last real sample. It defaults to dogstatsd_expiry_seconds. + CounterExpiry time.Duration + // Monitor is updated after sealed points have been appended to retention. + Monitor *monitor.Watcher +} + +// DogStatsDBucketMaterializer maintains a selected short-width materialized view +// of normal DogStatsD samples. Open buckets are mutable aggregation/sketch state; +// only sealed series and sketch series are appended to the shared retention ring. +type DogStatsDBucketMaterializer struct { + retention *metriclookback.Retention + monitor *monitor.Watcher + + bucketWidth time.Duration + bucketWidthSeconds int64 + sealDelay time.Duration + contextExpiry time.Duration + counterExpiry time.Duration + + shards []dogStatsDBucketShard +} + +type dogStatsDBucketShard struct { + mu sync.Mutex + + initialized bool + nextSealBucket int64 + buckets map[int64]metrics.ContextMetrics + sketchBuckets map[int64]map[ckey.ContextKey]*quantile.Agent + descriptors map[ckey.ContextKey]dogStatsDDescriptor +} + +type dogStatsDDescriptor struct { + name string + host string + tags []string + mtype metrics.MetricType + noIndex bool + source metrics.MetricSource + lastSeen int64 +} + +// NewDogStatsDBucketMaterializer creates a selected normal-DogStatsD bucket +// materializer. It returns nil when retention is nil. +func NewDogStatsDBucketMaterializer(retention *metriclookback.Retention, opts DogStatsDBucketMaterializerOptions) *DogStatsDBucketMaterializer { + if retention == nil { + return nil + } + + bucketWidth := normalizeDogStatsDBucketWidth(opts.BucketWidth) + shardCount := opts.ShardCount + if shardCount <= 0 { + shardCount = DefaultDogStatsDMaterializerShardCount + } + contextExpiry := opts.ContextExpiry + if contextExpiry <= 0 { + contextExpiry = time.Duration(pkgconfigsetup.Datadog().GetInt64("dogstatsd_context_expiry_seconds")) * time.Second + } + counterExpiry := opts.CounterExpiry + if counterExpiry <= 0 { + counterExpiry = time.Duration(pkgconfigsetup.Datadog().GetInt64("dogstatsd_expiry_seconds")) * time.Second + } + sealDelay := opts.SealDelay + if sealDelay == 0 { + sealDelay = DefaultDogStatsDSealDelay + } + if sealDelay < 0 { + sealDelay = 0 + } + + m := &DogStatsDBucketMaterializer{ + retention: retention, + monitor: opts.Monitor, + bucketWidth: bucketWidth, + bucketWidthSeconds: int64(bucketWidth / time.Second), + sealDelay: sealDelay, + contextExpiry: contextExpiry, + counterExpiry: counterExpiry, + shards: make([]dogStatsDBucketShard, shardCount), + } + for i := range m.shards { + m.shards[i].buckets = make(map[int64]metrics.ContextMetrics) + m.shards[i].sketchBuckets = make(map[int64]map[ckey.ContextKey]*quantile.Agent) + m.shards[i].descriptors = make(map[ckey.ContextKey]dogStatsDDescriptor) + } + return m +} + +func normalizeDogStatsDBucketWidth(width time.Duration) time.Duration { + if width <= 0 { + return DefaultDogStatsDBucketWidth + } + if width < time.Second { + return time.Second + } + if remainder := width % time.Second; remainder != 0 { + width += time.Second - remainder + } + return width +} + +// Observe updates the open lookback bucket for a selected normal DogStatsD +// sample using the effective series context resolved by the TimeSampler. +func (m *DogStatsDBucketMaterializer) Observe(sample *metrics.MetricSample, timestamp float64, dogctx aggregator.DogStatsDLookbackContext) { + if m == nil || sample == nil { + return + } + if sample.Mtype != metrics.GaugeType && sample.Mtype != metrics.CounterType && sample.Mtype != metrics.DistributionType { + tlmDogStatsDBucketSamples.Inc("unsupported_type") + return + } + + bucketStart := m.bucketStart(timestamp) + shard := &m.shards[m.shardIndex(dogctx.ContextKey)] + shard.mu.Lock() + defer shard.mu.Unlock() + + if shard.initialized && bucketStart < shard.nextSealBucket { + tlmDogStatsDBucketSamples.Inc("late_drop") + return + } + if !shard.initialized { + shard.initialized = true + shard.nextSealBucket = bucketStart + } + + desc, found := shard.descriptors[dogctx.ContextKey] + if !found { + desc = dogStatsDDescriptor{ + name: dogctx.Name, + host: dogctx.Host, + tags: append([]string(nil), dogctx.Tags...), + mtype: sample.Mtype, + noIndex: dogctx.NoIndex, + source: dogctx.Source, + } + } + desc.lastSeen = int64(timestamp) + shard.descriptors[dogctx.ContextKey] = desc + + if sample.Mtype == metrics.DistributionType { + if math.IsInf(sample.Value, 0) || math.IsNaN(sample.Value) { + tlmDogStatsDBucketSamples.Inc("error") + return + } + bucketSketches := shard.sketchBuckets[bucketStart] + if bucketSketches == nil { + bucketSketches = make(map[ckey.ContextKey]*quantile.Agent) + shard.sketchBuckets[bucketStart] = bucketSketches + } + agent := bucketSketches[dogctx.ContextKey] + if agent == nil { + agent = &quantile.Agent{} + bucketSketches[dogctx.ContextKey] = agent + } + agent.Insert(sample.Value, sample.SampleRate) + tlmDogStatsDBucketSamples.Inc("accepted") + return + } + + bucketMetrics := shard.buckets[bucketStart] + if bucketMetrics == nil { + bucketMetrics = metrics.MakeContextMetrics() + shard.buckets[bucketStart] = bucketMetrics + } + if err := bucketMetrics.AddSample(dogctx.ContextKey, sample, timestamp, m.bucketWidthSeconds, nil, pkgconfigsetup.Datadog()); err != nil { + tlmDogStatsDBucketSamples.Inc("error") + return + } + tlmDogStatsDBucketSamples.Inc("accepted") +} + +// Flush seals all buckets that are old enough relative to timestamp. +func (m *DogStatsDBucketMaterializer) Flush(timestamp float64) { + if m == nil { + return + } + last, ok := m.lastSealableBucketStart(timestamp) + if !ok { + return + } + + var observations []monitorObservation + for i := range m.shards { + result := m.shards[i].flushThrough(last, m) + observations = m.appendSealedSeries(result.series, observations) + observations = m.appendSealedSketchSeries(result.sketches, observations) + } + m.observeSealedPoints(observations) +} + +// FlushAll seals all currently open buckets, ignoring the normal seal delay. +// This mirrors the TimeSampler force-flush path used during shutdown when the +// Agent is configured to flush incomplete DogStatsD buckets. +func (m *DogStatsDBucketMaterializer) FlushAll(_ float64) { + if m == nil { + return + } + var observations []monitorObservation + for i := range m.shards { + last, ok := m.shards[i].lastOpenBucketStart() + if !ok { + continue + } + result := m.shards[i].flushThrough(last, m) + observations = m.appendSealedSeries(result.series, observations) + observations = m.appendSealedSketchSeries(result.sketches, observations) + } + m.observeSealedPoints(observations) +} + +type dogStatsDFlushResult struct { + series []*metrics.Serie + sketches []*metrics.SketchSeries +} + +type monitorObservation struct { + name string + at time.Time +} + +func (m *DogStatsDBucketMaterializer) appendSealedSeries(series []*metrics.Serie, observations []monitorObservation) []monitorObservation { + for _, serie := range series { + if serie == nil || len(serie.Points) == 0 { + continue + } + _ = m.retention.AppendSerie(context.Background(), ringbuffer.Source{Kind: ringbuffer.SourceDogStatsDBucketed}, serie) + tlmDogStatsDBucketPoints.Add(float64(len(serie.Points))) + if !m.shouldObserveMetric(serie.Name) { + continue + } + for _, point := range serie.Points { + observations = append(observations, monitorObservation{name: serie.Name, at: pointObservedAt(point)}) + } + } + return observations +} + +func (m *DogStatsDBucketMaterializer) appendSealedSketchSeries(series []*metrics.SketchSeries, observations []monitorObservation) []monitorObservation { + for _, serie := range series { + if serie == nil || len(serie.Points) == 0 { + continue + } + _ = m.retention.AppendSketchSeries(context.Background(), ringbuffer.Source{Kind: ringbuffer.SourceDogStatsDBucketed}, serie) + tlmDogStatsDBucketSketches.Add(float64(len(serie.Points))) + if !m.shouldObserveMetric(serie.Name) { + continue + } + for _, point := range serie.Points { + observations = append(observations, monitorObservation{name: serie.Name, at: sketchPointObservedAt(point)}) + } + } + return observations +} + +func (m *DogStatsDBucketMaterializer) shouldObserveMetric(name string) bool { + return m.monitor != nil && name == m.monitor.MetricName() +} + +func (m *DogStatsDBucketMaterializer) observeSealedPoints(observations []monitorObservation) { + if m.monitor == nil || len(observations) == 0 { + return + } + sort.SliceStable(observations, func(i, j int) bool { + if observations[i].at.Equal(observations[j].at) { + return observations[i].name < observations[j].name + } + return observations[i].at.Before(observations[j].at) + }) + for _, observation := range observations { + m.monitor.Observe(observation.name, observation.at) + } +} + +func sketchPointObservedAt(point metrics.SketchPoint) time.Time { + if point.Ts > 0 { + return time.Unix(point.Ts, 0) + } + return time.Now() +} + +func (s *dogStatsDBucketShard) lastOpenBucketStart() (int64, bool) { + s.mu.Lock() + defer s.mu.Unlock() + + if !s.initialized { + return 0, false + } + last := int64(0) + found := false + for bucketStart := range s.buckets { + if bucketStart < s.nextSealBucket { + continue + } + if !found || bucketStart > last { + last = bucketStart + found = true + } + } + for bucketStart := range s.sketchBuckets { + if bucketStart < s.nextSealBucket { + continue + } + if !found || bucketStart > last { + last = bucketStart + found = true + } + } + return last, found +} + +func (s *dogStatsDBucketShard) flushThrough(lastBucketStart int64, m *DogStatsDBucketMaterializer) dogStatsDFlushResult { + s.mu.Lock() + defer s.mu.Unlock() + + if !s.initialized || lastBucketStart < s.nextSealBucket { + return dogStatsDFlushResult{} + } + + result := dogStatsDFlushResult{} + for bucketStart := s.nextSealBucket; bucketStart <= lastBucketStart; bucketStart += m.bucketWidthSeconds { + sealedBucket := false + + bucketMetrics := s.buckets[bucketStart] + delete(s.buckets, bucketStart) + if bucketMetrics == nil && s.hasActiveCounter(bucketStart, m.counterExpiry) { + bucketMetrics = metrics.MakeContextMetrics() + } + if bucketMetrics != nil { + s.sampleCounterZeroValues(bucketStart, bucketMetrics, m) + rawSeries, _ := bucketMetrics.Flush(float64(bucketStart)) + for _, serie := range rawSeries { + desc, found := s.descriptors[serie.ContextKey] + if !found { + continue + } + serie.Name = desc.name + serie.NameSuffix + serie.Tags = tagset.CompositeTagsFromSlice(append([]string(nil), desc.tags...)) + serie.Host = desc.host + serie.NoIndex = desc.noIndex + serie.Source = desc.source + serie.Interval = m.bucketWidthSeconds + result.series = append(result.series, serie) + sealedBucket = true + } + } + + bucketSketches := s.sketchBuckets[bucketStart] + delete(s.sketchBuckets, bucketStart) + for contextKey, agent := range bucketSketches { + if agent == nil { + continue + } + sketch := agent.Finish() + if sketch == nil { + continue + } + desc, found := s.descriptors[contextKey] + if !found { + continue + } + result.sketches = append(result.sketches, &metrics.SketchSeries{ + DistributionMetadata: metrics.DistributionMetadata{ + Name: desc.name, + Tags: tagset.CompositeTagsFromSlice(append([]string(nil), desc.tags...)), + Host: desc.host, + Interval: m.bucketWidthSeconds, + NoIndex: desc.noIndex, + Source: desc.source, + }, + Points: []metrics.SketchPoint{{ + Ts: bucketStart, + Sketch: sketch, + }}, + }) + sealedBucket = true + } + + if sealedBucket { + tlmDogStatsDBucketSealed.Add(1) + } + } + s.expireDescriptors(lastBucketStart, m) + s.nextSealBucket = lastBucketStart + m.bucketWidthSeconds + return result +} + +func (s *dogStatsDBucketShard) hasActiveCounter(bucketStart int64, counterExpiry time.Duration) bool { + for _, desc := range s.descriptors { + if desc.mtype == metrics.CounterType && counterActiveAt(desc, bucketStart, counterExpiry) { + return true + } + } + return false +} + +func (s *dogStatsDBucketShard) sampleCounterZeroValues(bucketStart int64, bucketMetrics metrics.ContextMetrics, m *DogStatsDBucketMaterializer) { + for contextKey, desc := range s.descriptors { + if desc.mtype != metrics.CounterType || !counterActiveAt(desc, bucketStart, m.counterExpiry) { + continue + } + sample := &metrics.MetricSample{ + Name: desc.name, + Value: 0, + RawValue: "0.0", + Mtype: metrics.CounterType, + Tags: nil, + Host: desc.host, + SampleRate: 1, + Timestamp: float64(bucketStart), + NoIndex: desc.noIndex, + Source: desc.source, + } + _ = bucketMetrics.AddSample(contextKey, sample, float64(bucketStart), m.bucketWidthSeconds, nil, pkgconfigsetup.Datadog()) + } +} + +func (s *dogStatsDBucketShard) expireDescriptors(cutoffBucketStart int64, m *DogStatsDBucketMaterializer) { + for contextKey, desc := range s.descriptors { + expiry := m.contextExpiry + if desc.mtype == metrics.CounterType { + expiry += m.counterExpiry + } + if desc.lastSeen+int64(expiry/time.Second) < cutoffBucketStart { + delete(s.descriptors, contextKey) + } + } +} + +func counterActiveAt(desc dogStatsDDescriptor, bucketStart int64, counterExpiry time.Duration) bool { + return desc.lastSeen+int64(counterExpiry/time.Second) > bucketStart +} + +func (m *DogStatsDBucketMaterializer) bucketStart(timestamp float64) int64 { + unixSeconds := int64(timestamp) + return unixSeconds - unixSeconds%m.bucketWidthSeconds +} + +func (m *DogStatsDBucketMaterializer) lastSealableBucketStart(timestamp float64) (int64, bool) { + watermark := timestamp - m.sealDelay.Seconds() + if watermark < float64(m.bucketWidthSeconds) { + return 0, false + } + lastCandidate := int64(math.Floor(watermark)) - m.bucketWidthSeconds + if lastCandidate < 0 { + return 0, false + } + return lastCandidate - lastCandidate%m.bucketWidthSeconds, true +} + +func (m *DogStatsDBucketMaterializer) shardIndex(contextKey ckey.ContextKey) int { + if len(m.shards) <= 1 { + return 0 + } + return int(uint64(contextKey) % uint64(len(m.shards))) +} diff --git a/pkg/metriclookback/dogstatsd/bucket_test.go b/pkg/metriclookback/dogstatsd/bucket_test.go new file mode 100644 index 000000000000..74f0571eff57 --- /dev/null +++ b/pkg/metriclookback/dogstatsd/bucket_test.go @@ -0,0 +1,343 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package dogstatsd + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/pkg/aggregator" + "github.com/DataDog/datadog-agent/pkg/aggregator/ckey" + pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" + "github.com/DataDog/datadog-agent/pkg/metriclookback" + "github.com/DataDog/datadog-agent/pkg/metriclookback/monitor" + "github.com/DataDog/datadog-agent/pkg/metriclookback/ringbuffer" + "github.com/DataDog/datadog-agent/pkg/metrics" +) + +func TestDogStatsDBucketMaterializerSealsGaugeWithResolvedContext(t *testing.T) { + retention := metriclookback.NewRetention(ringbuffer.Options{Capacity: 16, ShardCount: 1}) + materializer := NewDogStatsDBucketMaterializer(retention, DogStatsDBucketMaterializerOptions{ + BucketWidth: time.Second, + SealDelay: -1, + ShardCount: 1, + }) + + ctx := aggregator.DogStatsDLookbackContext{ + ContextKey: ckey.ContextKey(42), + Name: "mapped.metric", + Host: "host-a", + Tags: []string{"tagger:tag", "metric:tag"}, + NoIndex: true, + Source: metrics.MetricSource(7), + } + materializer.Observe(&metrics.MetricSample{Name: "mapped.metric", Value: 1, Mtype: metrics.GaugeType, SampleRate: 1}, 10.1, ctx) + materializer.Observe(&metrics.MetricSample{Name: "mapped.metric", Value: 2, Mtype: metrics.GaugeType, SampleRate: 1}, 10.9, ctx) + materializer.Flush(11) + + series := retention.Series() + require.Len(t, series, 1) + require.Equal(t, "mapped.metric", series[0].Name) + require.Equal(t, "host-a", series[0].Host) + require.Equal(t, []string{"tagger:tag", "metric:tag"}, series[0].Tags.UnsafeToReadOnlySliceString()) + require.Equal(t, metrics.APIGaugeType, series[0].MType) + require.Equal(t, int64(1), series[0].Interval) + require.True(t, series[0].NoIndex) + require.Equal(t, metrics.MetricSource(7), series[0].Source) + require.Equal(t, []metrics.Point{{Ts: 10, Value: 2}}, series[0].Points) +} + +func TestDogStatsDBucketMaterializerSealsCounterUsingDogStatsDSemantics(t *testing.T) { + retention := metriclookback.NewRetention(ringbuffer.Options{Capacity: 16, ShardCount: 1}) + materializer := NewDogStatsDBucketMaterializer(retention, DogStatsDBucketMaterializerOptions{ + BucketWidth: time.Second, + SealDelay: -1, + ShardCount: 1, + CounterExpiry: time.Second, + }) + ctx := dogstatsdBucketTestContext("counter.metric", 1) + + materializer.Observe(&metrics.MetricSample{Name: "counter.metric", Value: 2, Mtype: metrics.CounterType, SampleRate: 0.5}, 10.1, ctx) + materializer.Observe(&metrics.MetricSample{Name: "counter.metric", Value: 3, Mtype: metrics.CounterType, SampleRate: 1}, 10.8, ctx) + materializer.Flush(11) + + series := retention.Series() + require.Len(t, series, 1) + require.Equal(t, "counter.metric", series[0].Name) + require.Equal(t, metrics.APIRateType, series[0].MType) + require.Equal(t, int64(1), series[0].Interval) + require.Equal(t, []metrics.Point{{Ts: 10, Value: 7}}, series[0].Points) +} + +func TestDogStatsDBucketMaterializerSealsDistributionSketch(t *testing.T) { + retention := metriclookback.NewRetention(ringbuffer.Options{Capacity: 16, ShardCount: 1}) + materializer := NewDogStatsDBucketMaterializer(retention, DogStatsDBucketMaterializerOptions{ + BucketWidth: time.Second, + SealDelay: -1, + ShardCount: 1, + }) + ctx := aggregator.DogStatsDLookbackContext{ + ContextKey: ckey.ContextKey(2), + Name: "dist.metric", + Host: "host-a", + Tags: []string{"env:test", "role:web"}, + NoIndex: true, + Source: metrics.MetricSourceDogstatsd, + } + + materializer.Observe(&metrics.MetricSample{Name: "dist.metric", Value: 1, Mtype: metrics.DistributionType, SampleRate: 1}, 10.1, ctx) + materializer.Observe(&metrics.MetricSample{Name: "dist.metric", Value: 3, Mtype: metrics.DistributionType, SampleRate: 1}, 10.2, ctx) + materializer.Observe(&metrics.MetricSample{Name: "dist.metric", Value: 5, Mtype: metrics.DistributionType, SampleRate: 0.5}, 10.3, ctx) + materializer.Flush(11) + + require.Empty(t, retention.Series()) + sketches := retention.SketchSeries() + require.Len(t, sketches, 1) + require.Equal(t, "dist.metric", sketches[0].Name) + require.Equal(t, "host-a", sketches[0].Host) + require.Equal(t, []string{"env:test", "role:web"}, sketches[0].Tags.UnsafeToReadOnlySliceString()) + require.Equal(t, int64(1), sketches[0].Interval) + require.True(t, sketches[0].NoIndex) + require.Equal(t, metrics.MetricSourceDogstatsd, sketches[0].Source) + require.Len(t, sketches[0].Points, 1) + require.Equal(t, int64(10), sketches[0].Points[0].Ts) + cnt, min, max, sum, avg := sketches[0].Points[0].Sketch.BasicStats() + require.Equal(t, int64(4), cnt) + require.Equal(t, float64(1), min) + require.Equal(t, float64(5), max) + require.Equal(t, float64(14), sum) + require.Equal(t, 3.5, avg) + + points := retention.ProjectedSketchPointsBetweenSources( + []ringbuffer.Source{{Kind: ringbuffer.SourceDogStatsDBucketed}}, + "dist.metric", + time.Unix(9, 0), + time.Unix(11, 0), + metriclookback.PlaceholderAverageSketchProjection{}, + ) + require.Equal(t, []ringbuffer.Point{{Ts: time.Unix(10, 0), Value: 3.5, Tags: []string{"env:test", "role:web"}}}, points) +} + +func TestDogStatsDBucketMaterializerFlushAllFlushesDistributionOnlyBuckets(t *testing.T) { + retention := metriclookback.NewRetention(ringbuffer.Options{Capacity: 16, ShardCount: 1}) + materializer := NewDogStatsDBucketMaterializer(retention, DogStatsDBucketMaterializerOptions{ + BucketWidth: time.Second, + SealDelay: 30 * time.Second, + ShardCount: 1, + }) + ctx := dogstatsdBucketTestContext("dist.metric", 1) + + materializer.Observe(&metrics.MetricSample{Name: "dist.metric", Value: 1, Mtype: metrics.DistributionType, SampleRate: 1}, 10.1, ctx) + materializer.Flush(11) + require.Empty(t, retention.SketchSeries()) + + materializer.FlushAll(11) + sketches := retention.SketchSeries() + require.Len(t, sketches, 1) + require.Equal(t, int64(10), sketches[0].Points[0].Ts) +} + +func TestDogStatsDBucketMaterializerZeroFillsActiveCounters(t *testing.T) { + retention := metriclookback.NewRetention(ringbuffer.Options{Capacity: 16, ShardCount: 1}) + materializer := NewDogStatsDBucketMaterializer(retention, DogStatsDBucketMaterializerOptions{ + BucketWidth: time.Second, + SealDelay: -1, + ShardCount: 1, + CounterExpiry: 2 * time.Second, + }) + ctx := dogstatsdBucketTestContext("counter.metric", 1) + + materializer.Observe(&metrics.MetricSample{Name: "counter.metric", Value: 5, Mtype: metrics.CounterType, SampleRate: 1}, 10.1, ctx) + materializer.Flush(14) + + series := retention.Series() + require.Len(t, series, 2) + require.Equal(t, []metrics.Point{{Ts: 10, Value: 5}}, series[0].Points) + require.Equal(t, []metrics.Point{{Ts: 11, Value: 0}}, series[1].Points) +} + +func TestDogStatsDBucketMaterializerDropsUnsupportedAndLateSamples(t *testing.T) { + retention := metriclookback.NewRetention(ringbuffer.Options{Capacity: 16, ShardCount: 1}) + materializer := NewDogStatsDBucketMaterializer(retention, DogStatsDBucketMaterializerOptions{ + BucketWidth: time.Second, + SealDelay: -1, + ShardCount: 1, + }) + ctx := dogstatsdBucketTestContext("metric", 1) + + materializer.Observe(&metrics.MetricSample{Name: "metric", Value: 100, Mtype: metrics.HistogramType, SampleRate: 1}, 10.1, ctx) + materializer.Observe(&metrics.MetricSample{Name: "metric", Value: 1, Mtype: metrics.GaugeType, SampleRate: 1}, 10.1, ctx) + materializer.Flush(11) + materializer.Observe(&metrics.MetricSample{Name: "metric", Value: 2, Mtype: metrics.GaugeType, SampleRate: 1}, 10.5, ctx) + materializer.Flush(12) + + series := retention.Series() + require.Len(t, series, 1) + require.Equal(t, []metrics.Point{{Ts: 10, Value: 1}}, series[0].Points) +} + +func TestDogStatsDBucketMaterializerHonorsWiderBucketWidth(t *testing.T) { + retention := metriclookback.NewRetention(ringbuffer.Options{Capacity: 16, ShardCount: 1}) + materializer := NewDogStatsDBucketMaterializer(retention, DogStatsDBucketMaterializerOptions{ + BucketWidth: 5 * time.Second, + SealDelay: -1, + ShardCount: 1, + }) + ctx := dogstatsdBucketTestContext("gauge.metric", 1) + + materializer.Observe(&metrics.MetricSample{Name: "gauge.metric", Value: 1, Mtype: metrics.GaugeType, SampleRate: 1}, 10.1, ctx) + materializer.Observe(&metrics.MetricSample{Name: "gauge.metric", Value: 2, Mtype: metrics.GaugeType, SampleRate: 1}, 14.9, ctx) + materializer.Flush(15) + + series := retention.Series() + require.Len(t, series, 1) + require.Equal(t, int64(5), series[0].Interval) + require.Equal(t, []metrics.Point{{Ts: 10, Value: 2}}, series[0].Points) +} + +func TestDogStatsDBucketMaterializerFlushAllIgnoresSealDelay(t *testing.T) { + retention := metriclookback.NewRetention(ringbuffer.Options{Capacity: 16, ShardCount: 1}) + materializer := NewDogStatsDBucketMaterializer(retention, DogStatsDBucketMaterializerOptions{ + BucketWidth: time.Second, + SealDelay: 30 * time.Second, + ShardCount: 1, + }) + ctx := dogstatsdBucketTestContext("gauge.metric", 1) + + materializer.Observe(&metrics.MetricSample{Name: "gauge.metric", Value: 1, Mtype: metrics.GaugeType, SampleRate: 1}, 10.1, ctx) + materializer.Flush(11) + require.Empty(t, retention.Series()) + + materializer.FlushAll(11) + series := retention.Series() + require.Len(t, series, 1) + require.Equal(t, []metrics.Point{{Ts: 10, Value: 1}}, series[0].Points) +} + +func TestDogStatsDBucketMaterializerExpiresDescriptors(t *testing.T) { + retention := metriclookback.NewRetention(ringbuffer.Options{Capacity: 16, ShardCount: 1}) + materializer := NewDogStatsDBucketMaterializer(retention, DogStatsDBucketMaterializerOptions{ + BucketWidth: time.Second, + SealDelay: -1, + ShardCount: 1, + ContextExpiry: time.Second, + CounterExpiry: time.Second, + }) + ctx := dogstatsdBucketTestContext("gauge.metric", 1) + + materializer.Observe(&metrics.MetricSample{Name: "gauge.metric", Value: 1, Mtype: metrics.GaugeType, SampleRate: 1}, 10.1, ctx) + materializer.Flush(14) + + require.Len(t, materializer.shards[0].descriptors, 0) +} + +func TestDogStatsDBucketMaterializerMatchesContextMetricsForSupportedTypes(t *testing.T) { + tests := []struct { + name string + samples []metrics.MetricSample + }{ + { + name: "gauge last value", + samples: []metrics.MetricSample{ + {Name: "metric", Value: 1, Mtype: metrics.GaugeType, SampleRate: 1}, + {Name: "metric", Value: 5, Mtype: metrics.GaugeType, SampleRate: 1}, + {Name: "metric", Value: 3, Mtype: metrics.GaugeType, SampleRate: 1}, + }, + }, + { + name: "counter sample rate adjusted rate", + samples: []metrics.MetricSample{ + {Name: "metric", Value: 2, Mtype: metrics.CounterType, SampleRate: 0.5}, + {Name: "metric", Value: 3, Mtype: metrics.CounterType, SampleRate: 1}, + {Name: "metric", Value: 4, Mtype: metrics.CounterType, SampleRate: 0.25}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + contextKey := ckey.ContextKey(99) + expectedMetrics := metrics.MakeContextMetrics() + retention := metriclookback.NewRetention(ringbuffer.Options{Capacity: 16, ShardCount: 1}) + materializer := NewDogStatsDBucketMaterializer(retention, DogStatsDBucketMaterializerOptions{ + BucketWidth: time.Second, + SealDelay: -1, + ShardCount: 1, + CounterExpiry: time.Second, + }) + ctx := dogstatsdBucketTestContext("metric", contextKey) + + for i := range tt.samples { + sample := tt.samples[i] + timestamp := 10.1 + float64(i)/10 + require.NoError(t, expectedMetrics.AddSample(contextKey, &sample, timestamp, 1, nil, pkgconfigsetup.Datadog())) + materializer.Observe(&sample, timestamp, ctx) + } + materializer.Flush(11) + + expected, errs := expectedMetrics.Flush(10) + require.Empty(t, errs) + require.Len(t, expected, 1) + actual := retention.Series() + require.Len(t, actual, 1) + require.Equal(t, expected[0].MType, actual[0].MType) + require.Equal(t, expected[0].Points, actual[0].Points) + }) + } +} + +func TestDogStatsDBucketMaterializerObservesMonitorAfterAppendingAllShards(t *testing.T) { + retention := metriclookback.NewRetention(ringbuffer.Options{Capacity: 16, ShardCount: 2}) + reader := monitor.PointReaderFunc(func(metricName string, from, to time.Time) []monitor.Point { + points := retention.PointsBetweenSources([]ringbuffer.Source{{Kind: ringbuffer.SourceDogStatsDBucketed}}, metricName, from, to) + out := make([]monitor.Point, 0, len(points)) + for _, point := range points { + out = append(out, monitor.Point{Ts: point.Ts, Value: point.Value, Tags: point.Tags}) + } + return out + }) + var decisions []monitor.Decision + watcher, err := monitor.New(monitor.Config{ + MetricName: "target.metric", + RangeEpsilon: 5, + EvaluationInterval: 2 * time.Second, + }, reader, monitor.DecisionSinkFunc(func(decision monitor.Decision) { + decisions = append(decisions, decision) + })) + require.NoError(t, err) + + materializer := NewDogStatsDBucketMaterializer(retention, DogStatsDBucketMaterializerOptions{ + BucketWidth: time.Second, + SealDelay: -1, + ShardCount: 2, + Monitor: watcher, + }) + + ctxShard0 := dogstatsdBucketTestContext("target.metric", 2) + ctxShard1 := dogstatsdBucketTestContext("target.metric", 3) + materializer.Observe(&metrics.MetricSample{Name: "target.metric", Value: 1, Mtype: metrics.GaugeType, SampleRate: 1}, 10.1, ctxShard0) + materializer.Observe(&metrics.MetricSample{Name: "target.metric", Value: 10, Mtype: metrics.GaugeType, SampleRate: 1}, 10.1, ctxShard1) + materializer.Observe(&metrics.MetricSample{Name: "target.metric", Value: 1, Mtype: metrics.GaugeType, SampleRate: 1}, 12.1, ctxShard0) + materializer.Observe(&metrics.MetricSample{Name: "target.metric", Value: 10, Mtype: metrics.GaugeType, SampleRate: 1}, 12.1, ctxShard1) + + materializer.Flush(13) + + require.NotEmpty(t, decisions) + require.Equal(t, monitor.Breach, decisions[0].State) + require.Equal(t, 4, decisions[0].PointCount) + require.Equal(t, float64(9), decisions[0].Range) +} + +func dogstatsdBucketTestContext(name string, key ckey.ContextKey) aggregator.DogStatsDLookbackContext { + return aggregator.DogStatsDLookbackContext{ + ContextKey: key, + Name: name, + Host: "host", + Tags: []string{"env:test"}, + } +} diff --git a/pkg/metriclookback/egress_controller.go b/pkg/metriclookback/egress_controller.go new file mode 100644 index 000000000000..e23c4f9d56f8 --- /dev/null +++ b/pkg/metriclookback/egress_controller.go @@ -0,0 +1,394 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package metriclookback + +import ( + "sync" + "sync/atomic" + "time" + + telemetryimpl "github.com/DataDog/datadog-agent/comp/core/telemetry/impl" + "github.com/DataDog/datadog-agent/pkg/metriclookback/monitor" + "github.com/DataDog/datadog-agent/pkg/metriclookback/ringbuffer" + "github.com/DataDog/datadog-agent/pkg/serializer" +) + +const ( + // DefaultEgressInterval is how often continuous egress wakes up to check for + // retained ranges that have become eligible. + DefaultEgressInterval = 10 * time.Second +) + +var ( + tlmEgressMode = telemetryimpl.GetCompatComponent().NewGauge("metric_lookback", "egress_mode", []string{"mode"}, "Current metric lookback egress mode") + tlmEgressTransitions = telemetryimpl.GetCompatComponent().NewCounter("metric_lookback", "egress_transitions", []string{"from", "to", "reason"}, "Count of metric lookback egress mode transitions") + tlmEgressRanges = telemetryimpl.GetCompatComponent().NewCounter("metric_lookback", "egress_ranges", []string{"state"}, "Count of metric lookback egress ranges") + tlmEgressRuns = telemetryimpl.GetCompatComponent().NewCounter("metric_lookback", "egress_runs", []string{"state"}, "Count of metric lookback egress forwarding attempts") + tlmEgressPayloadSeries = telemetryimpl.GetCompatComponent().NewGauge("metric_lookback", "egress_payload_series", []string{"payload"}, "Number of series or sketch series sent by the last metric lookback egress range") + tlmEgressRangeSeconds = telemetryimpl.GetCompatComponent().NewGauge("metric_lookback", "egress_range_seconds", nil, "Width of the last metric lookback egress range in seconds") +) + +// MonitorStateTransition describes an observed change in the monitor's own +// health state. It is emitted independently of whether the monitor is allowed to +// affect egress mode. +type MonitorStateTransition struct { + MetricName string + From monitor.State + To monitor.State + Initial bool + Decision monitor.Decision + DryRun bool + EgressMode EgressMode +} + +// MonitorStateTransitionLogger receives monitor state transition diagnostics. +type MonitorStateTransitionLogger func(MonitorStateTransition) + +// EgressControllerOptions controls how retained ranges are forwarded. Most +// options are code-level defaults; public config currently selects monitor +// enablement, the watched metric/range epsilon, dry-run mode, and trigger/recovery +// egress context windows. +type EgressControllerOptions struct { + // PreTriggerWindow extends forwarding before a monitor breach/unknown window. + PreTriggerWindow time.Duration + // PostRecoveryWindow extends forwarding after the first healthy window that + // suppresses egress. + PostRecoveryWindow time.Duration + // SendDelay is the minimum age of a metric timestamp before it is forwarded. + SendDelay time.Duration + // EgressInterval is how often the background loop checks for eligible ranges. + EgressInterval time.Duration + // MonitorStaleTimeout controls when suppressed egress returns to forwarding if + // no fresh monitor decision is observed. Zero disables stale reopening. + MonitorStaleTimeout time.Duration + // DryRun keeps egress forwarding from startup and prevents monitor decisions + // from changing egress mode. Monitor state transitions are still logged. + DryRun bool + // MonitorStateTransitionLogger is called whenever the monitor's own state + // changes, including the first observed state. + MonitorStateTransitionLogger MonitorStateTransitionLogger + + // Now is injectable for deterministic tests. It defaults to time.Now. + Now func() time.Time +} + +// EgressController applies monitor decisions to an EgressPolicy and forwards the +// policy-planned retained ranges through the serializer. It does not own monitor +// scoring or retention admission. +type EgressController struct { + retention *Retention + metricSerializer serializer.MetricSerializer + policy *EgressPolicy + egressInterval time.Duration + now func() time.Time + dryRun bool + logTransition MonitorStateTransitionLogger + + runMu sync.Mutex + policyMu sync.Mutex + + hasMonitorState bool + lastMonitorState monitor.State + + asyncRunMu sync.Mutex + asyncRunWG sync.WaitGroup + + startOnce sync.Once + stopOnce sync.Once + started atomic.Bool + stopped atomic.Bool + stopCh chan struct{} + doneCh chan struct{} +} + +// NewEgressController creates a monitor decision sink backed by the retention +// ring and serializer. It returns nil when forwarding is not possible. +func NewEgressController(retention *Retention, metricSerializer serializer.MetricSerializer, opts EgressControllerOptions) *EgressController { + if retention == nil || metricSerializer == nil { + return nil + } + if opts.EgressInterval <= 0 { + opts.EgressInterval = DefaultEgressInterval + } + if opts.Now == nil { + opts.Now = time.Now + } + policy := NewEgressPolicy(EgressPolicyOptions{ + PreTriggerWindow: opts.PreTriggerWindow, + PostRecoveryWindow: opts.PostRecoveryWindow, + SendDelay: opts.SendDelay, + MonitorStaleTimeout: opts.MonitorStaleTimeout, + StartForwarding: opts.DryRun, + }) + return &EgressController{ + retention: retention, + metricSerializer: metricSerializer, + policy: policy, + egressInterval: opts.EgressInterval, + now: opts.Now, + dryRun: opts.DryRun, + logTransition: opts.MonitorStateTransitionLogger, + stopCh: make(chan struct{}), + doneCh: make(chan struct{}), + } +} + +// Start begins the background egress loop. It is safe to call more than once. +func (c *EgressController) Start() { + if c == nil { + return + } + c.startOnce.Do(func() { + if c.stopped.Load() { + return + } + c.started.Store(true) + go c.loop() + }) +} + +// Stop asks the background egress loop to stop and waits for it to exit. +func (c *EgressController) Stop() { + if c == nil { + return + } + c.stopOnce.Do(func() { + c.asyncRunMu.Lock() + c.stopped.Store(true) + c.asyncRunMu.Unlock() + if c.started.Load() { + close(c.stopCh) + <-c.doneCh + } + c.asyncRunWG.Wait() + }) +} + +func (c *EgressController) loop() { + defer close(c.doneCh) + for { + select { + case <-c.stopCh: + return + default: + } + c.RunOnce() + + timer := time.NewTimer(c.egressInterval) + select { + case <-c.stopCh: + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + return + case <-timer.C: + } + } +} + +func (c *EgressController) runOnceAsync() { + c.asyncRunMu.Lock() + defer c.asyncRunMu.Unlock() + if c.stopped.Load() { + return + } + c.asyncRunWG.Add(1) + go func() { + defer c.asyncRunWG.Done() + if c.stopped.Load() { + return + } + c.RunOnce() + }() +} + +// OnDecision applies a monitor decision and asynchronously wakes egress so the +// DogStatsD append path does not block on serializer latency. In dry-run mode, +// the monitor decision is recorded for transition logging but does not affect +// egress policy, which remains forwarding. +func (c *EgressController) OnDecision(decision monitor.Decision) { + if c == nil || c.policy == nil { + return + } + var transition MonitorStateTransition + var logTransition bool + + c.policyMu.Lock() + before := c.policy.Mode() + if !c.dryRun { + c.policy.OnDecisionAt(decision, c.now()) + } + after := c.policy.Mode() + if !c.dryRun { + c.recordModeTransition(before, after, decision.State.String()) + } + transition, logTransition = c.recordMonitorStateLocked(decision, after) + c.policyMu.Unlock() + + if logTransition && c.logTransition != nil { + c.logTransition(transition) + } + c.runOnceAsync() +} + +// Mode returns the current egress mode. +func (c *EgressController) Mode() EgressMode { + if c == nil || c.policy == nil { + return EgressSuppressed + } + c.policyMu.Lock() + defer c.policyMu.Unlock() + return c.policy.Mode() +} + +// RunOnce forwards every policy-planned range currently eligible. It is exposed +// for deterministic tests; the background loop calls it periodically. +func (c *EgressController) RunOnce() { + if c == nil || c.policy == nil { + return + } + c.runMu.Lock() + defer c.runMu.Unlock() + + now := c.now() + seriesAvailable, hasSeries := c.availableSeriesRange() + sketchAvailable, hasSketch := c.availableSketchRange() + if !hasSeries && !hasSketch { + tlmEgressRuns.Inc("empty") + return + } + + c.policyMu.Lock() + before := c.policy.Mode() + if c.policy.MarkStaleIfNeeded(now) { + c.recordModeTransition(before, c.policy.Mode(), "stale") + } + mode := c.policy.Mode() + seriesRanges := c.policy.SeriesRangesToForward(now) + sketchRanges := c.policy.SketchRangesToForward(now) + c.policyMu.Unlock() + c.recordMode(mode) + + if len(seriesRanges) == 0 && len(sketchRanges) == 0 { + tlmEgressRuns.Inc("empty") + return + } + + forwardedAny := false + hadError := false + if hasSeries { + forwarded, failed := c.forwardRanges("series", seriesRanges, seriesAvailable, c.retention.ForwardSeriesRange, func(policy *EgressPolicy, r TimeRange) { + policy.MarkSeriesForwarded(r) + }) + forwardedAny = forwardedAny || forwarded + hadError = hadError || failed + } + if hasSketch { + forwarded, failed := c.forwardRanges("sketch", sketchRanges, sketchAvailable, c.retention.ForwardSketchRange, func(policy *EgressPolicy, r TimeRange) { + policy.MarkSketchForwarded(r) + }) + forwardedAny = forwardedAny || forwarded + hadError = hadError || failed + } + if hadError { + tlmEgressRuns.Inc("error") + return + } + if !forwardedAny { + tlmEgressRuns.Inc("empty") + } +} + +func (c *EgressController) forwardRanges(payload string, ranges []TimeRange, available TimeRange, forward func(serializer.MetricSerializer, time.Time, time.Time) (int, error), markForwarded func(*EgressPolicy, TimeRange)) (bool, bool) { + forwardedAny := false + for _, planned := range ranges { + r, ok := intersectRanges(planned, available) + if !ok { + continue + } + tlmEgressRanges.Inc("planned") + tlmEgressRangeSeconds.Set(r.To.Sub(r.From).Seconds()) + count, err := forward(c.metricSerializer, r.From, r.To) + tlmEgressPayloadSeries.Set(float64(count), payload) + if err != nil { + tlmEgressRanges.Inc("retry") + return forwardedAny, true + } + if count == 0 { + // Do not mark empty ranges as forwarded. A later-arriving retained point with + // an older timestamp should still be eligible if it falls in a forwarding + // interval. + tlmEgressRuns.Inc("empty") + continue + } + c.policyMu.Lock() + markForwarded(c.policy, r) + c.policyMu.Unlock() + tlmEgressRanges.Inc("forwarded") + tlmEgressRuns.Inc("success") + forwardedAny = true + } + return forwardedAny, false +} + +func (c *EgressController) availableSeriesRange() (TimeRange, bool) { + return availableRangeFromStats(c.retention.Stats()) +} + +func (c *EgressController) availableSketchRange() (TimeRange, bool) { + return availableRangeFromStats(c.retention.SketchStats()) +} + +func availableRangeFromStats(stats ringbuffer.Stats) (TimeRange, bool) { + if stats.Records == 0 || stats.OldestUnixMicro == 0 || stats.NewestUnixMicro == 0 { + return TimeRange{}, false + } + return TimeRange{ + From: time.UnixMicro(stats.OldestUnixMicro), + To: time.UnixMicro(stats.NewestUnixMicro).Add(time.Microsecond), + }, true +} + +func (c *EgressController) recordMode(mode EgressMode) { + tlmEgressMode.Set(0, "forwarding") + tlmEgressMode.Set(0, "suppressed") + tlmEgressMode.Set(1, mode.String()) +} + +func (c *EgressController) recordModeTransition(from, to EgressMode, reason string) { + if from == to { + return + } + tlmEgressTransitions.Inc(from.String(), to.String(), reason) +} + +func (c *EgressController) recordMonitorStateLocked(decision monitor.Decision, egressMode EgressMode) (MonitorStateTransition, bool) { + transition := MonitorStateTransition{ + MetricName: decision.MetricName, + From: c.lastMonitorState, + To: decision.State, + Initial: !c.hasMonitorState, + Decision: decision, + DryRun: c.dryRun, + EgressMode: egressMode, + } + if c.hasMonitorState && c.lastMonitorState == decision.State { + return transition, false + } + c.lastMonitorState = decision.State + c.hasMonitorState = true + return transition, true +} + +func intersectRanges(a, b TimeRange) (TimeRange, bool) { + from := maxTime(a.From, b.From) + to := minTime(a.To, b.To) + r := TimeRange{From: from, To: to} + return r, validHalfOpenRange(r) +} diff --git a/pkg/metriclookback/egress_controller_test.go b/pkg/metriclookback/egress_controller_test.go new file mode 100644 index 000000000000..b3a84b4f935f --- /dev/null +++ b/pkg/metriclookback/egress_controller_test.go @@ -0,0 +1,262 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package metriclookback + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/pkg/metriclookback/monitor" + "github.com/DataDog/datadog-agent/pkg/metriclookback/ringbuffer" + "github.com/DataDog/datadog-agent/pkg/metrics" + serializermocks "github.com/DataDog/datadog-agent/pkg/serializer/mocks" +) + +func TestEgressControllerForwardsEligibleRetainedRange(t *testing.T) { + retention := NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + from := time.Unix(100, 0) + require.NoError(t, retention.AppendSamples(context.Background(), ringbuffer.Source{Kind: ringbuffer.SourceDogStatsDNoAggregation}, []metrics.MetricSample{{ + Name: "target", + Value: 1, + Mtype: metrics.GaugeType, + Timestamp: float64(from.Unix()), + }})) + + serializer := serializermocks.NewMetricSerializer(t) + serializer.On("SendIterableSeries", mock.Anything).Run(func(args mock.Arguments) { + source := args.Get(0).(metrics.SerieSource) + require.Equal(t, uint64(1), source.Count()) + }).Return(nil).Once() + + controller := NewEgressController(retention, serializer, EgressControllerOptions{ + SendDelay: time.Nanosecond, + Now: func() time.Time { return from.Add(time.Minute) }, + }) + controller.policy.OnDecision(monitor.Decision{State: monitor.Breach, WindowFrom: from, WindowTo: from.Add(time.Second)}) + + controller.RunOnce() + require.Len(t, controller.policy.ForwardedRanges(), 1) +} + +func TestEgressControllerRespectsSendDelay(t *testing.T) { + retention := NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + pointTime := time.Unix(100, 0) + require.NoError(t, retention.AppendSamples(context.Background(), ringbuffer.Source{Kind: ringbuffer.SourceDogStatsDNoAggregation}, []metrics.MetricSample{{ + Name: "target", + Value: 1, + Mtype: metrics.GaugeType, + Timestamp: float64(pointTime.Unix()), + }})) + + serializer := serializermocks.NewMetricSerializer(t) + serializer.On("SendIterableSeries", mock.Anything).Return(nil).Once() + + now := pointTime.Add(5 * time.Second) + controller := NewEgressController(retention, serializer, EgressControllerOptions{ + SendDelay: 30 * time.Second, + Now: func() time.Time { return now }, + }) + controller.policy.OnDecision(monitor.Decision{State: monitor.Breach, WindowFrom: pointTime, WindowTo: pointTime.Add(time.Second)}) + + controller.RunOnce() + require.Empty(t, controller.policy.ForwardedRanges()) + + now = pointTime.Add(31 * time.Second) + controller.RunOnce() + require.Len(t, controller.policy.ForwardedRanges(), 1) +} + +func TestEgressControllerRetriesFailedRange(t *testing.T) { + retention := NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + from := time.Unix(100, 0) + require.NoError(t, retention.AppendSamples(context.Background(), ringbuffer.Source{Kind: ringbuffer.SourceDogStatsDNoAggregation}, []metrics.MetricSample{{ + Name: "target", + Value: 1, + Mtype: metrics.GaugeType, + Timestamp: float64(from.Unix()), + }})) + + serializer := serializermocks.NewMetricSerializer(t) + serializer.On("SendIterableSeries", mock.Anything).Return(errors.New("boom")).Once() + serializer.On("SendIterableSeries", mock.Anything).Return(nil).Once() + + controller := NewEgressController(retention, serializer, EgressControllerOptions{ + SendDelay: time.Nanosecond, + Now: func() time.Time { return from.Add(time.Minute) }, + }) + controller.policy.OnDecision(monitor.Decision{State: monitor.Breach, WindowFrom: from, WindowTo: from.Add(time.Second)}) + + controller.RunOnce() + require.Empty(t, controller.policy.ForwardedRanges()) + + controller.RunOnce() + require.Len(t, controller.policy.ForwardedRanges(), 1) +} + +func TestEgressControllerDoesNotRetrySeriesAfterSketchFailure(t *testing.T) { + retention := NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + from := time.Unix(100, 0) + require.NoError(t, retention.AppendSamples(context.Background(), ringbuffer.Source{Kind: ringbuffer.SourceDogStatsDNoAggregation}, []metrics.MetricSample{{ + Name: "target.series", + Value: 1, + Mtype: metrics.GaugeType, + Timestamp: float64(from.Unix()), + }})) + require.NoError(t, retention.AppendSketchSeries(context.Background(), ringbuffer.Source{Kind: ringbuffer.SourceDogStatsDBucketed}, &metrics.SketchSeries{ + DistributionMetadata: metrics.DistributionMetadata{Name: "target.sketch"}, + Points: []metrics.SketchPoint{{ + Ts: from.Unix(), + Sketch: testSketchData(1, 2), + }}, + })) + + serializer := serializermocks.NewMetricSerializer(t) + serializer.On("SendIterableSeries", mock.Anything).Run(func(args mock.Arguments) { + source := args.Get(0).(metrics.SerieSource) + require.Equal(t, uint64(1), source.Count()) + }).Return(nil).Once() + serializer.On("SendSketch", mock.Anything).Return(errors.New("boom")).Once() + serializer.On("SendSketch", mock.Anything).Run(func(args mock.Arguments) { + source := args.Get(0).(metrics.SketchesSource) + require.Equal(t, uint64(1), source.Count()) + }).Return(nil).Once() + + controller := NewEgressController(retention, serializer, EgressControllerOptions{ + SendDelay: time.Nanosecond, + Now: func() time.Time { return from.Add(time.Minute) }, + }) + controller.policy.OnDecision(monitor.Decision{State: monitor.Breach, WindowFrom: from, WindowTo: from.Add(time.Second)}) + + controller.RunOnce() + require.Len(t, controller.policy.ForwardedSeriesRanges(), 1) + require.Empty(t, controller.policy.ForwardedSketchRanges()) + + controller.RunOnce() + require.Len(t, controller.policy.ForwardedSeriesRanges(), 1) + require.Len(t, controller.policy.ForwardedSketchRanges(), 1) +} + +func TestEgressControllerAppliesMonitorDecisions(t *testing.T) { + retention := NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + serializer := serializermocks.NewMetricSerializer(t) + controller := NewEgressController(retention, serializer, EgressControllerOptions{ + Now: func() time.Time { return time.Unix(100, 0) }, + }) + + require.Equal(t, EgressSuppressed, controller.Mode()) + + controller.OnDecision(monitor.Decision{ + State: monitor.Breach, + WindowFrom: time.Unix(60, 0), + WindowTo: time.Unix(90, 0), + }) + require.Eventually(t, func() bool { + return controller.Mode() == EgressForwarding + }, time.Second, time.Millisecond) + + controller.OnDecision(monitor.Decision{ + State: monitor.Healthy, + WindowFrom: time.Unix(90, 0), + WindowTo: time.Unix(120, 0), + }) + require.Eventually(t, func() bool { + return controller.Mode() == EgressSuppressed + }, time.Second, time.Millisecond) +} + +func TestEgressControllerStopInterruptsEgressInterval(t *testing.T) { + retention := NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + serializer := serializermocks.NewMetricSerializer(t) + controller := NewEgressController(retention, serializer, EgressControllerOptions{ + EgressInterval: time.Hour, + }) + controller.Start() + + stopped := make(chan struct{}) + go func() { + controller.Stop() + close(stopped) + }() + + select { + case <-stopped: + case <-time.After(time.Second): + require.FailNow(t, "egress controller Stop waited for the egress interval") + } +} + +func TestEgressControllerDryRunStartsForwardingAndIgnoresMonitorDecisions(t *testing.T) { + retention := NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + serializer := serializermocks.NewMetricSerializer(t) + controller := NewEgressController(retention, serializer, EgressControllerOptions{ + DryRun: true, + Now: func() time.Time { return time.Unix(100, 0) }, + }) + + require.Equal(t, EgressForwarding, controller.Mode()) + require.Equal(t, []TimeRange{{}}, controller.policy.ForwardingRanges()) + + controller.OnDecision(monitor.Decision{ + State: monitor.Healthy, + WindowFrom: time.Unix(60, 0), + WindowTo: time.Unix(90, 0), + }) + controller.OnDecision(monitor.Decision{ + State: monitor.Healthy, + WindowFrom: time.Unix(90, 0), + WindowTo: time.Unix(120, 0), + }) + controller.OnDecision(monitor.Decision{ + State: monitor.Breach, + WindowFrom: time.Unix(120, 0), + WindowTo: time.Unix(150, 0), + }) + + require.Eventually(t, func() bool { + return controller.Mode() == EgressForwarding + }, time.Second, time.Millisecond) + require.Equal(t, []TimeRange{{}}, controller.policy.ForwardingRanges()) +} + +func TestEgressControllerLogsMonitorStateTransitionsRegardlessOfDryRun(t *testing.T) { + for _, dryRun := range []bool{false, true} { + t.Run(fmt.Sprintf("dry_run_%t", dryRun), func(t *testing.T) { + retention := NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + serializer := serializermocks.NewMetricSerializer(t) + var transitions []MonitorStateTransition + controller := NewEgressController(retention, serializer, EgressControllerOptions{ + DryRun: dryRun, + MonitorStateTransitionLogger: func(transition MonitorStateTransition) { + transitions = append(transitions, transition) + }, + Now: func() time.Time { return time.Unix(100, 0) }, + }) + + controller.OnDecision(monitor.Decision{MetricName: "target", State: monitor.Unknown, WindowFrom: time.Unix(0, 0), WindowTo: time.Unix(30, 0)}) + controller.OnDecision(monitor.Decision{MetricName: "target", State: monitor.Healthy, WindowFrom: time.Unix(30, 0), WindowTo: time.Unix(60, 0)}) + controller.OnDecision(monitor.Decision{MetricName: "target", State: monitor.Healthy, WindowFrom: time.Unix(60, 0), WindowTo: time.Unix(90, 0)}) + controller.OnDecision(monitor.Decision{MetricName: "target", State: monitor.Breach, WindowFrom: time.Unix(90, 0), WindowTo: time.Unix(120, 0)}) + + require.Len(t, transitions, 3) + require.True(t, transitions[0].Initial) + require.Equal(t, monitor.Unknown, transitions[0].To) + require.False(t, transitions[1].Initial) + require.Equal(t, monitor.Unknown, transitions[1].From) + require.Equal(t, monitor.Healthy, transitions[1].To) + require.Equal(t, monitor.Healthy, transitions[2].From) + require.Equal(t, monitor.Breach, transitions[2].To) + for _, transition := range transitions { + require.Equal(t, dryRun, transition.DryRun) + } + }) + } +} diff --git a/pkg/metriclookback/egress_policy.go b/pkg/metriclookback/egress_policy.go new file mode 100644 index 000000000000..58760064f111 --- /dev/null +++ b/pkg/metriclookback/egress_policy.go @@ -0,0 +1,484 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package metriclookback + +import ( + "sort" + "time" + + "github.com/DataDog/datadog-agent/pkg/metriclookback/monitor" +) + +const ( + // DefaultEgressSendDelay waits until retained metric timestamps are old enough + // that the regular coarser metric pipeline should have had time to send first. + DefaultEgressSendDelay = 30 * time.Second + // DefaultMonitorStaleTimeout keeps stale-monitor reopening disabled by default. + // Empty periods without the trigger metric should not enable egress. + DefaultMonitorStaleTimeout = 0 +) + +// TimeRange is a half-open timestamp range: [From, To). A zero From is an +// unbounded lower edge. Policy methods only return ranges with a non-zero To. +type TimeRange struct { + From time.Time + To time.Time +} + +// EgressMode is whether retained lookback ranges should currently be forwarded. +type EgressMode int + +const ( + // EgressForwarding means selected retained ranges should be forwarded once they + // are eligible under the send-delay policy. + EgressForwarding EgressMode = iota + // EgressSuppressed means retention continues but ranges are not forwarded until + // the monitor breaches, returns unknown, or becomes stale when stale reopening + // is explicitly configured. + EgressSuppressed +) + +// String returns a stable label for telemetry and diagnostics. +func (m EgressMode) String() string { + switch m { + case EgressSuppressed: + return "suppressed" + default: + return "forwarding" + } +} + +// EgressPolicyOptions controls the pure range-planning policy. +type EgressPolicyOptions struct { + // PreTriggerWindow extends a forwarding range before a monitor breach or + // unknown window. + PreTriggerWindow time.Duration + // PostRecoveryWindow extends a forwarding range after the first healthy window + // that suppresses egress. + PostRecoveryWindow time.Duration + // SendDelay is the minimum age of a metric timestamp before it is forwarded. + SendDelay time.Duration + // MonitorStaleTimeout controls when suppressed egress returns to forwarding if + // no fresh monitor decision is observed. Zero disables stale reopening. + MonitorStaleTimeout time.Duration + // StartForwarding makes the policy start in forwarding mode instead of the + // default suppressed mode. This is used for monitor dry-runs where monitor + // decisions are observed and logged but must not gate intake egress. + StartForwarding bool +} + +// EgressPolicy converts monitor decisions and wall-clock eligibility into +// half-open retained timestamp ranges that should be forwarded. It owns no +// retention or serializer resources so it can be tested exhaustively. +type EgressPolicy struct { + preTriggerWindow time.Duration + postRecoveryWindow time.Duration + sendDelay time.Duration + monitorStaleTimeout time.Duration + + mode EgressMode + + forwardingRanges []TimeRange + forwardedSeriesRanges []TimeRange + forwardedSketchRanges []TimeRange + + lastMonitorAt time.Time + lastDecisionAt time.Time +} + +// NewEgressPolicy creates an egress policy. By default it starts suppressed: +// retention is active immediately, but forwarding only opens after the watched +// metric breaches or produces an unknown monitor decision. StartForwarding +// overrides this initial mode for monitor dry-runs. +func NewEgressPolicy(opts EgressPolicyOptions) *EgressPolicy { + if opts.SendDelay < 0 { + opts.SendDelay = 0 + } + if opts.SendDelay == 0 { + opts.SendDelay = DefaultEgressSendDelay + } + if opts.MonitorStaleTimeout < 0 { + opts.MonitorStaleTimeout = 0 + } + policy := &EgressPolicy{ + preTriggerWindow: nonNegativeDuration(opts.PreTriggerWindow), + postRecoveryWindow: nonNegativeDuration(opts.PostRecoveryWindow), + sendDelay: opts.SendDelay, + monitorStaleTimeout: opts.MonitorStaleTimeout, + mode: EgressSuppressed, + } + if opts.StartForwarding { + policy.openForwardingAt(time.Time{}) + } + return policy +} + +func nonNegativeDuration(d time.Duration) time.Duration { + if d < 0 { + return 0 + } + return d +} + +// Mode returns the current egress mode. +func (p *EgressPolicy) Mode() EgressMode { + if p == nil { + return EgressSuppressed + } + return p.mode +} + +// OnDecision applies a monitor decision to the egress mode and forwarding +// ranges. The decision window timestamp is also used as the decision freshness +// time; callers with a wall-clock receipt time should use OnDecisionAt. +func (p *EgressPolicy) OnDecision(decision monitor.Decision) { + p.OnDecisionAt(decision, decision.WindowTo) +} + +// OnDecisionAt applies a monitor decision to the egress mode and forwarding +// ranges, recording observedAt as the wall-clock freshness time for stale-monitor +// fallback. +func (p *EgressPolicy) OnDecisionAt(decision monitor.Decision, observedAt time.Time) { + if p == nil { + return + } + if !decision.WindowTo.IsZero() { + p.lastMonitorAt = decision.WindowTo + } + if !observedAt.IsZero() { + p.lastDecisionAt = observedAt + } + + switch decision.State { + case monitor.Healthy: + p.onHealthy(decision) + case monitor.Breach: + p.onBreach(decision) + default: + p.onUnknown(decision) + } +} + +func (p *EgressPolicy) onHealthy(decision monitor.Decision) { + if p.mode == EgressSuppressed { + return + } + closeAt := decision.WindowTo.Add(p.postRecoveryWindow) + p.closeForwardingAt(closeAt) + p.mode = EgressSuppressed +} + +func (p *EgressPolicy) onBreach(decision monitor.Decision) { + if p.mode == EgressForwarding { + return + } + p.openForwardingAt(decision.WindowFrom.Add(-p.preTriggerWindow)) +} + +func (p *EgressPolicy) onUnknown(decision monitor.Decision) { + if p.mode == EgressForwarding { + return + } + p.openForwardingAt(decision.WindowFrom.Add(-p.preTriggerWindow)) +} + +// MarkStaleIfNeeded returns suppressed egress to forwarding when no fresh +// monitor decision has arrived within the stale timeout. +func (p *EgressPolicy) MarkStaleIfNeeded(now time.Time) bool { + if p == nil || p.mode == EgressForwarding || p.monitorStaleTimeout <= 0 || now.IsZero() || p.lastDecisionAt.IsZero() || p.lastMonitorAt.IsZero() { + return false + } + if now.Sub(p.lastDecisionAt) <= p.monitorStaleTimeout { + return false + } + p.openForwardingAt(p.lastMonitorAt) + return true +} + +// RangesToForward returns half-open series ranges that are in a forwarding +// interval, old enough under send delay, and not already marked as forwarded. +func (p *EgressPolicy) RangesToForward(now time.Time) []TimeRange { + return p.SeriesRangesToForward(now) +} + +// SeriesRangesToForward returns half-open series ranges that are in a forwarding +// interval, old enough under send delay, and not already marked as forwarded. +func (p *EgressPolicy) SeriesRangesToForward(now time.Time) []TimeRange { + if p == nil { + return nil + } + return p.rangesToForward(now, p.forwardedSeriesRanges) +} + +// SketchRangesToForward returns half-open sketch ranges that are in a forwarding +// interval, old enough under send delay, and not already marked as forwarded. +func (p *EgressPolicy) SketchRangesToForward(now time.Time) []TimeRange { + if p == nil { + return nil + } + return p.rangesToForward(now, p.forwardedSketchRanges) +} + +func (p *EgressPolicy) rangesToForward(now time.Time, forwardedRanges []TimeRange) []TimeRange { + if now.IsZero() { + return nil + } + eligibleThrough := now.Add(-p.sendDelay) + if eligibleThrough.IsZero() || eligibleThrough.After(now) { + return nil + } + + var candidates []TimeRange + for _, r := range p.forwardingRanges { + candidate := r + if candidate.To.IsZero() || eligibleThrough.Before(candidate.To) { + candidate.To = eligibleThrough + } + if validHalfOpenRange(candidate) { + candidates = append(candidates, candidate) + } + } + if len(candidates) == 0 { + return nil + } + return subtractRanges(candidates, forwardedRanges) +} + +// MarkForwarded records that a half-open range was successfully forwarded for +// both series and sketch payloads. +func (p *EgressPolicy) MarkForwarded(r TimeRange) { + p.MarkSeriesForwarded(r) + p.MarkSketchForwarded(r) +} + +// MarkSeriesForwarded records that a half-open range was successfully forwarded +// for series payloads. +func (p *EgressPolicy) MarkSeriesForwarded(r TimeRange) { + if p == nil || !validHalfOpenRange(r) { + return + } + p.forwardedSeriesRanges = appendAndMergeRanges(p.forwardedSeriesRanges, r) +} + +// MarkSketchForwarded records that a half-open range was successfully forwarded +// for sketch payloads. +func (p *EgressPolicy) MarkSketchForwarded(r TimeRange) { + if p == nil || !validHalfOpenRange(r) { + return + } + p.forwardedSketchRanges = appendAndMergeRanges(p.forwardedSketchRanges, r) +} + +// ForwardingRanges returns a copy of the policy's forwarding ranges for tests +// and diagnostics. +func (p *EgressPolicy) ForwardingRanges() []TimeRange { + if p == nil { + return nil + } + return append([]TimeRange(nil), p.forwardingRanges...) +} + +// ForwardedRanges returns a copy of successfully forwarded series ranges for +// tests and diagnostics. +func (p *EgressPolicy) ForwardedRanges() []TimeRange { + return p.ForwardedSeriesRanges() +} + +// ForwardedSeriesRanges returns a copy of successfully forwarded series ranges +// for tests and diagnostics. +func (p *EgressPolicy) ForwardedSeriesRanges() []TimeRange { + if p == nil { + return nil + } + return append([]TimeRange(nil), p.forwardedSeriesRanges...) +} + +// ForwardedSketchRanges returns a copy of successfully forwarded sketch ranges +// for tests and diagnostics. +func (p *EgressPolicy) ForwardedSketchRanges() []TimeRange { + if p == nil { + return nil + } + return append([]TimeRange(nil), p.forwardedSketchRanges...) +} + +func (p *EgressPolicy) openForwardingAt(from time.Time) { + p.mode = EgressForwarding + p.forwardingRanges = appendAndMergeRanges(p.forwardingRanges, TimeRange{From: from}) +} + +func (p *EgressPolicy) closeForwardingAt(to time.Time) { + if to.IsZero() { + return + } + for i := range p.forwardingRanges { + if !p.forwardingRanges[i].To.IsZero() { + continue + } + if !p.forwardingRanges[i].From.IsZero() && !p.forwardingRanges[i].From.Before(to) { + p.forwardingRanges = append(p.forwardingRanges[:i], p.forwardingRanges[i+1:]...) + return + } + p.forwardingRanges[i].To = to + return + } +} + +func appendAndMergeRanges(ranges []TimeRange, next TimeRange) []TimeRange { + // Open ranges are valid when they represent a forwarding interval. They are + // not returned from RangesToForward until clipped by send-delay eligibility. + if !next.To.IsZero() && !validHalfOpenRange(next) { + return ranges + } + + out := append([]TimeRange(nil), ranges...) + out = append(out, next) + sort.Slice(out, func(i, j int) bool { + if out[i].From.Equal(out[j].From) { + return rangeToAfter(out[i].To, out[j].To) + } + if out[i].From.IsZero() { + return true + } + if out[j].From.IsZero() { + return false + } + return out[i].From.Before(out[j].From) + }) + + merged := out[:0] + for _, r := range out { + if len(merged) == 0 { + merged = append(merged, r) + continue + } + last := &merged[len(merged)-1] + if rangesOverlapOrTouch(*last, r) { + last.To = maxRangeTo(last.To, r.To) + continue + } + merged = append(merged, r) + } + return append([]TimeRange(nil), merged...) +} + +func subtractRanges(candidates, forwarded []TimeRange) []TimeRange { + if len(forwarded) == 0 { + return append([]TimeRange(nil), candidates...) + } + forwarded = append([]TimeRange(nil), forwarded...) + sort.Slice(forwarded, func(i, j int) bool { + if forwarded[i].From.Equal(forwarded[j].From) { + return forwarded[i].To.Before(forwarded[j].To) + } + if forwarded[i].From.IsZero() { + return true + } + if forwarded[j].From.IsZero() { + return false + } + return forwarded[i].From.Before(forwarded[j].From) + }) + + var out []TimeRange + for _, candidate := range candidates { + pieces := []TimeRange{candidate} + for _, done := range forwarded { + var nextPieces []TimeRange + for _, piece := range pieces { + nextPieces = append(nextPieces, subtractRange(piece, done)...) + } + pieces = nextPieces + if len(pieces) == 0 { + break + } + } + out = append(out, pieces...) + } + return out +} + +func subtractRange(r, done TimeRange) []TimeRange { + if !validHalfOpenRange(r) || !validHalfOpenRange(done) || !rangesOverlap(r, done) { + return []TimeRange{r} + } + var out []TimeRange + if rangeFromBefore(r.From, done.From) { + left := TimeRange{From: r.From, To: minTime(r.To, done.From)} + if validHalfOpenRange(left) { + out = append(out, left) + } + } + if done.To.Before(r.To) { + right := TimeRange{From: maxTime(r.From, done.To), To: r.To} + if validHalfOpenRange(right) { + out = append(out, right) + } + } + return out +} + +func validHalfOpenRange(r TimeRange) bool { + return !r.To.IsZero() && (r.From.IsZero() || r.From.Before(r.To)) +} + +func rangesOverlap(a, b TimeRange) bool { + if !validHalfOpenRange(a) || !validHalfOpenRange(b) { + return false + } + return rangeFromBefore(a.From, b.To) && rangeFromBefore(b.From, a.To) +} + +func rangesOverlapOrTouch(a, b TimeRange) bool { + if a.To.IsZero() { + return true + } + return !a.To.Before(b.From) +} + +func rangeFromBefore(from, t time.Time) bool { + return from.IsZero() || from.Before(t) +} + +func rangeToAfter(a, b time.Time) bool { + if a.IsZero() { + return true + } + if b.IsZero() { + return false + } + return a.After(b) +} + +func maxRangeTo(a, b time.Time) time.Time { + if a.IsZero() || b.IsZero() { + return time.Time{} + } + if a.Before(b) { + return b + } + return a +} + +func minTime(a, b time.Time) time.Time { + if a.IsZero() { + return b + } + if b.IsZero() || a.Before(b) { + return a + } + return b +} + +func maxTime(a, b time.Time) time.Time { + if a.IsZero() { + return b + } + if b.IsZero() || a.After(b) { + return a + } + return b +} diff --git a/pkg/metriclookback/egress_policy_test.go b/pkg/metriclookback/egress_policy_test.go new file mode 100644 index 000000000000..d94d33d66ff8 --- /dev/null +++ b/pkg/metriclookback/egress_policy_test.go @@ -0,0 +1,194 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package metriclookback + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/pkg/metriclookback/monitor" +) + +func TestEgressPolicyStartsSuppressed(t *testing.T) { + start := time.Unix(100, 0) + policy := NewEgressPolicy(EgressPolicyOptions{SendDelay: 10 * time.Second}) + + require.Equal(t, EgressSuppressed, policy.Mode()) + require.Empty(t, policy.RangesToForward(start.Add(10*time.Second))) + require.Empty(t, policy.ForwardingRanges()) +} + +func TestEgressPolicyCanStartForwarding(t *testing.T) { + now := time.Unix(100, 0) + policy := NewEgressPolicy(EgressPolicyOptions{SendDelay: time.Nanosecond, StartForwarding: true}) + + require.Equal(t, EgressForwarding, policy.Mode()) + require.Equal(t, []TimeRange{{}}, policy.ForwardingRanges()) + require.Equal(t, []TimeRange{{To: now.Add(-time.Nanosecond)}}, policy.RangesToForward(now)) +} + +func TestEgressPolicyBreachOpensForwardingAndAppliesSendDelay(t *testing.T) { + start := time.Unix(100, 0) + policy := NewEgressPolicy(EgressPolicyOptions{ + SendDelay: 10 * time.Second, + PreTriggerWindow: 5 * time.Second, + }) + + policy.OnDecision(monitor.Decision{ + State: monitor.Breach, + WindowFrom: start.Add(20 * time.Second), + WindowTo: start.Add(30 * time.Second), + }) + + require.Equal(t, EgressForwarding, policy.Mode()) + require.Equal(t, []TimeRange{{From: start.Add(15 * time.Second), To: start.Add(25 * time.Second)}}, policy.RangesToForward(start.Add(35*time.Second))) +} + +func TestEgressPolicyMarksForwardedRangesWithoutDuplicates(t *testing.T) { + start := time.Unix(100, 0) + policy := NewEgressPolicy(EgressPolicyOptions{SendDelay: time.Nanosecond}) + policy.OnDecision(monitor.Decision{State: monitor.Breach, WindowFrom: start, WindowTo: start.Add(10 * time.Second)}) + + first := TimeRange{From: start, To: start.Add(10 * time.Second)} + policy.MarkForwarded(first) + + ranges := policy.RangesToForward(start.Add(20 * time.Second)) + require.NotContains(t, ranges, first) + for _, r := range ranges { + require.False(t, rangesOverlap(r, first)) + } +} + +func TestEgressPolicyTracksSeriesAndSketchForwardedRangesSeparately(t *testing.T) { + start := time.Unix(100, 0) + policy := NewEgressPolicy(EgressPolicyOptions{SendDelay: time.Nanosecond}) + policy.OnDecision(monitor.Decision{State: monitor.Breach, WindowFrom: start, WindowTo: start.Add(10 * time.Second)}) + + first := TimeRange{From: start, To: start.Add(10 * time.Second)} + policy.MarkSeriesForwarded(first) + + now := start.Add(10*time.Second + time.Nanosecond) + for _, r := range policy.SeriesRangesToForward(now) { + require.False(t, rangesOverlap(r, first)) + } + require.Contains(t, policy.SketchRangesToForward(now), first) + require.Equal(t, []TimeRange{first}, policy.ForwardedSeriesRanges()) + require.Empty(t, policy.ForwardedSketchRanges()) + + policy.MarkSketchForwarded(first) + for _, r := range policy.SketchRangesToForward(now) { + require.False(t, rangesOverlap(r, first)) + } + require.Equal(t, []TimeRange{first}, policy.ForwardedSketchRanges()) +} + +func TestEgressPolicySuppressesAfterHealthyWindowWithPostRecoveryWindow(t *testing.T) { + start := time.Unix(100, 0) + policy := NewEgressPolicy(EgressPolicyOptions{ + SendDelay: time.Nanosecond, + PostRecoveryWindow: 5 * time.Second, + }) + policy.OnDecision(monitor.Decision{State: monitor.Breach, WindowFrom: start, WindowTo: start.Add(15 * time.Second)}) + + policy.OnDecision(monitor.Decision{State: monitor.Healthy, WindowFrom: start.Add(15 * time.Second), WindowTo: start.Add(30 * time.Second)}) + require.Equal(t, EgressSuppressed, policy.Mode()) + require.Equal(t, []TimeRange{{From: start, To: start.Add(35 * time.Second)}}, policy.ForwardingRanges()) +} + +func TestEgressPolicyBreachReopensSuppressedEgressWithPreTriggerWindow(t *testing.T) { + start := time.Unix(100, 0) + policy := suppressedPolicy(start) + + policy.OnDecision(monitor.Decision{ + State: monitor.Breach, + WindowFrom: start.Add(60 * time.Second), + WindowTo: start.Add(75 * time.Second), + }) + + require.Equal(t, EgressForwarding, policy.Mode()) + require.Equal(t, []TimeRange{ + {From: start.Add(-5 * time.Second), To: start.Add(30 * time.Second)}, + {From: start.Add(55 * time.Second)}, + }, policy.ForwardingRanges()) +} + +func TestEgressPolicyUnknownReopensSuppressedEgress(t *testing.T) { + start := time.Unix(100, 0) + policy := suppressedPolicy(start) + + policy.OnDecision(monitor.Decision{ + State: monitor.Unknown, + WindowFrom: start.Add(60 * time.Second), + WindowTo: start.Add(75 * time.Second), + }) + + require.Equal(t, EgressForwarding, policy.Mode()) + require.Equal(t, []TimeRange{ + {From: start.Add(-5 * time.Second), To: start.Add(30 * time.Second)}, + {From: start.Add(55 * time.Second)}, + }, policy.ForwardingRanges()) +} + +func TestEgressPolicyStaleMonitorReopensSuppressedEgressWhenConfigured(t *testing.T) { + start := time.Unix(100, 0) + policy := suppressedPolicy(start) + + require.False(t, policy.MarkStaleIfNeeded(start.Add(40*time.Second))) + require.Equal(t, EgressSuppressed, policy.Mode()) + + require.True(t, policy.MarkStaleIfNeeded(start.Add(61*time.Second))) + require.Equal(t, EgressForwarding, policy.Mode()) + require.Equal(t, []TimeRange{{From: start.Add(-5 * time.Second)}}, policy.ForwardingRanges()) +} + +func TestEgressPolicyStaleMonitorReopenDisabledByDefault(t *testing.T) { + start := time.Unix(100, 0) + policy := NewEgressPolicy(EgressPolicyOptions{}) + policy.OnDecision(monitor.Decision{State: monitor.Breach, WindowFrom: start, WindowTo: start.Add(10 * time.Second)}) + policy.OnDecision(monitor.Decision{State: monitor.Healthy, WindowFrom: start.Add(10 * time.Second), WindowTo: start.Add(30 * time.Second)}) + + require.Equal(t, EgressSuppressed, policy.Mode()) + require.False(t, policy.MarkStaleIfNeeded(start.Add(time.Hour))) + require.Equal(t, EgressSuppressed, policy.Mode()) +} + +func TestEgressPolicyMergesOverlappingForwardingRanges(t *testing.T) { + start := time.Unix(100, 0) + policy := suppressedPolicy(start) + + policy.OnDecision(monitor.Decision{ + State: monitor.Breach, + WindowFrom: start.Add(25 * time.Second), + WindowTo: start.Add(40 * time.Second), + }) + + require.Equal(t, []TimeRange{{From: start.Add(-5 * time.Second)}}, policy.ForwardingRanges()) +} + +func TestEgressPolicyHalfOpenBoundariesDoNotOverlapForwardedRanges(t *testing.T) { + start := time.Unix(100, 0) + policy := NewEgressPolicy(EgressPolicyOptions{SendDelay: time.Nanosecond}) + policy.OnDecision(monitor.Decision{State: monitor.Breach, WindowFrom: start, WindowTo: start.Add(10 * time.Second)}) + policy.MarkForwarded(TimeRange{From: start, To: start.Add(10 * time.Second)}) + + ranges := policy.RangesToForward(start.Add(20 * time.Second)) + for _, r := range ranges { + require.False(t, rangesOverlap(r, TimeRange{From: start, To: start.Add(10 * time.Second)})) + } +} + +func suppressedPolicy(start time.Time) *EgressPolicy { + policy := NewEgressPolicy(EgressPolicyOptions{ + SendDelay: time.Nanosecond, + PreTriggerWindow: 5 * time.Second, + MonitorStaleTimeout: 30 * time.Second, + }) + policy.OnDecision(monitor.Decision{State: monitor.Breach, WindowFrom: start, WindowTo: start.Add(10 * time.Second)}) + policy.OnDecision(monitor.Decision{State: monitor.Healthy, WindowFrom: start.Add(10 * time.Second), WindowTo: start.Add(30 * time.Second)}) + return policy +} diff --git a/pkg/metriclookback/metric_lookback.allium b/pkg/metriclookback/metric_lookback.allium new file mode 100644 index 000000000000..83c79a5299c8 --- /dev/null +++ b/pkg/metriclookback/metric_lookback.allium @@ -0,0 +1,691 @@ +-- allium: 3 +-- metric_lookback.allium + +-- Scope: Metric lookback behaviour for retained high-resolution metric points, +-- monitor-triggered egress, DogStatsD admission, dry-run mode, and forwarded +-- time ranges. +-- +-- Includes: +-- - Configuration semantics for global enablement, monitor enablement, +-- DogStatsD additional-metric enablement, and monitor dry-run mode. +-- - DogStatsD admission semantics: the monitor metric is auto-admitted as a +-- trigger metric independently of DogStatsD additional-metric enablement. +-- - Retained-point source semantics: the monitor evaluates points by metric +-- name from the shared lookback ring, regardless of whether the producer is +-- DogStatsD or the check/shadow lookback sender path. +-- - Monitor window classification: unknown, healthy, breach. +-- - Monitor state-transition diagnostics, including dry-run visibility. +-- - Egress mode lifecycle in normal and dry-run operation. +-- - Forwarded time-range eligibility, send-delay clipping, retry, and +-- duplicate suppression. +-- +-- Excludes: +-- - The storage layout, sharding, eviction algorithm, and memory accounting of +-- the retention ring. +-- - Aggregator internals that transform DogStatsD packets into final series or +-- distribution sketches before they enter metric lookback. +-- - Serialization format and network transport to intake. +-- - Check scheduler wiring for shadow checks. This spec treats check/shadow +-- lookback sender points as an external producer of retained points. +-- - Exact telemetry metric names. This spec models observable domain effects, +-- not internal counters/gauges. + +------------------------------------------------------------ +-- Given +------------------------------------------------------------ + +-- Metric lookback has one effective runtime configuration and one shared +-- retention ring in the scoped Agent process. Rules that create Monitor and +-- EgressPolicy instances are conditional on this configuration. +given { + configuration: LookbackConfiguration + retention: RetentionRing +} + +------------------------------------------------------------ +-- Enumerations +------------------------------------------------------------ + +enum RetainedPointSource { + dogstatsd_bucketed | dogstatsd_no_aggregation | check_shadow +} + +enum DogStatsDAdmissionReason { + monitor_trigger | additional_metric +} + +enum MonitorState { + unknown | healthy | breach +} + +enum MonitorMode { + disabled | dry_run | enabled +} + +enum EgressMode { + forwarding | suppressed +} + +enum ForwardAttemptState { + planned | forwarded | retry | empty | error +} + +enum ForwardPayload { + series | sketch +} + +------------------------------------------------------------ +-- Value Types +------------------------------------------------------------ + +value MetricName { + value: String +} + +value TagName { + value: String +} + +value MetricTag { + name: TagName + value: String +} + +value MonitorPartitionKey { + values: List +} + +value TimeRange { + start: Timestamp? + end: Timestamp? + + -- Derived: a range can be forwarded only when it has a non-empty upper + -- bound. A nil start means unbounded lower edge. + is_forwardable: end != nil and (start = nil or start < end) +} + +value MonitorWindow { + metric_name: MetricName + start: Timestamp + end: Timestamp +} + +value MetricPoint { + name: MetricName + source: RetainedPointSource + timestamp: Timestamp + value: Decimal + valid_number: Boolean + tags: Set +} + +------------------------------------------------------------ +-- External Entities +------------------------------------------------------------ + +-- Effective runtime configuration after defaults and environment overrides are +-- applied. +external entity LookbackConfiguration { + enabled: Boolean + capacity: Integer + shard_count: Integer + + monitor_mode: MonitorMode + monitor_metric_name: MetricName? + monitor_range_epsilon: Decimal + monitor_partition_tags: List + monitor_evaluation_interval: Duration + + egress_pre_trigger_window: Duration + egress_post_recovery_window: Duration + + -- Additional DogStatsD metric names to retain. An empty set disables + -- additional DogStatsD retention. The monitor metric is separately + -- auto-admitted as a DogStatsD trigger source when the monitor is enabled. + dogstatsd_metric_names: Set +} + +-- DogStatsD packets and finalized no-aggregation series are external stimuli to +-- metric lookback. Their parsing, tag resolution, context resolution, and final +-- metric-type conversion are owned by the DogStatsD/aggregator pipeline. +external entity DogStatsDObservation { + name: MetricName + timestamp: Timestamp + value: Decimal + tags: Set + source: dogstatsd_bucketed | dogstatsd_no_aggregation + + -- A no-aggregation series is retention-ready when it reaches lookback. + -- A normal bucketed DogStatsD sample becomes retention-ready only after its + -- short-resolution bucket is sealed. The bucket sealing mechanics are owned + -- by the DogStatsD materialization path and are outside this spec. + retention_ready: Boolean +} + +-- A check/shadow sender write is an external producer of retained points. This +-- spec does not define how checks are scheduled or routed into the sender. +external entity CheckShadowObservation { + name: MetricName + timestamp: Timestamp + value: Decimal + tags: Set +} + +-- Intake is the externally observable destination for forwarded retained +-- ranges. Transport retry and HTTP details are outside this spec. +external entity Intake {} + +------------------------------------------------------------ +-- Entities +------------------------------------------------------------ + +entity RetentionRing { + points: List +} + +entity DogStatsDAdmission { + metric_name: MetricName + reason: DogStatsDAdmissionReason + active: Boolean +} + +entity Monitor { + metric_name: MetricName + range_epsilon: Decimal + partition_tags: List + evaluation_interval: Duration + state: MonitorState? + decisions: List + transition_logs: List +} + +entity MonitorDecision { + metric_name: MetricName + window: MonitorWindow + state: MonitorState + point_count: Integer + min: Decimal? + max: Decimal? + range: Decimal? + range_epsilon: Decimal + partition_tags: List + partition_key: MonitorPartitionKey? + partition_count: Integer +} + +entity MonitorStateTransitionLog { + metric_name: MetricName + from: MonitorState? + to: MonitorState + initial: Boolean + dry_run: Boolean + egress_mode: EgressMode + window: MonitorWindow + point_count: Integer + min: Decimal? + max: Decimal? + range: Decimal? + range_epsilon: Decimal + partition_tags: List + partition_key: MonitorPartitionKey? + partition_count: Integer +} + +entity EgressPolicy { + mode: EgressMode + dry_run: Boolean + forwarding_ranges: List + forwarded_series_ranges: List + forwarded_sketch_ranges: List +} + +entity ForwardAttempt { + range: TimeRange + payload: ForwardPayload + state: ForwardAttemptState + payload_count: Integer +} + +------------------------------------------------------------ +-- Config +------------------------------------------------------------ + +config { + -- Approximate monitor evaluation window size. + monitor_evaluation_interval: Duration = 30.seconds + + -- Minimum retained points needed to classify a monitor window as healthy or + -- breach. Fewer valid points are unknown. + monitor_min_points: Integer = 2 + + -- Minimum age of retained metric timestamps before they are eligible for + -- intake forwarding. + egress_send_delay: Duration = 30.seconds + + -- Domain-level context windows around monitor transitions. + egress_pre_trigger_window: Duration = 0.seconds + egress_post_recovery_window: Duration = 30.seconds +} + +------------------------------------------------------------ +-- Rules +------------------------------------------------------------ + +rule GlobalDisablePreventsLookbackEffects { + when: RuntimeConfigured(configuration) + requires: configuration.enabled = false + ensures: not exists Monitor + ensures: not exists EgressPolicy + ensures: not exists DogStatsDAdmission + @guidance + -- Producers outside this spec may still emit metrics through their + -- normal paths. The guarantee here is only that metric lookback does not + -- retain, monitor, or egress them. +} + +rule DisabledMonitorModeDisablesMonitorBehaviour { + when: RuntimeConfigured(configuration) + requires: configuration.enabled = true + requires: configuration.monitor_mode = disabled + ensures: not exists Monitor + ensures: not exists EgressPolicy + ensures: not exists DogStatsDAdmission where reason = monitor_trigger +} + +rule MonitorModeRequiresMetricName { + when: RuntimeConfigured(configuration) + requires: configuration.enabled = true + requires: configuration.monitor_mode in {dry_run, enabled} + requires: configuration.monitor_metric_name = nil + ensures: ConfigurationRejected(reason: "metric_lookback.monitor.metric_name is required when monitor.mode is dry_run or enabled") +} + +rule MonitorCreatesSharedRetentionReader { + when: RuntimeConfigured(configuration) + requires: configuration.enabled = true + requires: configuration.monitor_mode in {dry_run, enabled} + requires: configuration.monitor_metric_name != nil + ensures: Monitor.created( + metric_name: configuration.monitor_metric_name, + range_epsilon: configuration.monitor_range_epsilon, + partition_tags: configuration.monitor_partition_tags, + evaluation_interval: configuration.monitor_evaluation_interval, + state: nil, + decisions: [], + transition_logs: [] + ) +} + +rule MonitorMetricAutoAdmittedFromDogStatsD { + when: RuntimeConfigured(configuration) + requires: configuration.enabled = true + requires: configuration.monitor_mode in {dry_run, enabled} + requires: configuration.monitor_metric_name != nil + ensures: DogStatsDAdmission.created( + metric_name: configuration.monitor_metric_name, + reason: monitor_trigger, + active: true + ) + @guidance + -- The monitor metric is a trigger source, not an additional retained + -- metric, so it does not need to appear in dogstatsd_metric_names. +} + +rule DogStatsDAdditionalMetricAdmission { + when: RuntimeConfigured(configuration) + requires: configuration.enabled = true + for name in configuration.dogstatsd_metric_names: + ensures: DogStatsDAdmission.created( + metric_name: name, + reason: additional_metric, + active: true + ) +} + +rule EmptyDogStatsDAdditionalMetricsDoesNotDisableMonitorAdmission { + when: RuntimeConfigured(configuration) + requires: configuration.enabled = true + requires: configuration.monitor_mode in {dry_run, enabled} + requires: configuration.monitor_metric_name != nil + requires: configuration.dogstatsd_metric_names.count = 0 + ensures: DogStatsDAdmission.created( + metric_name: configuration.monitor_metric_name, + reason: monitor_trigger, + active: true + ) +} + +rule AdmittedDogStatsDObservationIsRetainedWhenReady { + when: observation: DogStatsDObservation.retention_ready becomes true + requires: exists DogStatsDAdmission where metric_name = observation.name and active = true + ensures: retention.points = retention.points + [MetricPoint{ + name: observation.name, + source: observation.source, + timestamp: observation.timestamp, + value: observation.value, + valid_number: true, + tags: observation.tags + }] +} + +rule UnadmittedDogStatsDObservationIsIgnoredByLookback { + when: observation: DogStatsDObservation.created + requires: not exists DogStatsDAdmission where metric_name = observation.name and active = true + ensures: retention.points = retention.points + @guidance + -- Unadmitted normal DogStatsD samples should not allocate lookback bucket + -- state. Unadmitted no-aggregation series should not append to retention. +} + +rule CheckShadowObservationIsRetained { + when: observation: CheckShadowObservation.created + requires: configuration.enabled = true + ensures: retention.points = retention.points + [MetricPoint{ + name: observation.name, + source: check_shadow, + timestamp: observation.timestamp, + value: observation.value, + valid_number: true, + tags: observation.tags + }] + @guidance + -- Check/shadow retention also depends on scheduler/sender wiring outside + -- this spec. Once a check writes through the lookback sender, the monitor + -- treats its points the same as DogStatsD points with the same name. +} + +rule MonitorWindowUsesAllRetainedSourcesForMetricName { + when: window: MonitorWindow.completed + requires: exists Monitor where metric_name = window.metric_name + let matching_points = retention.points where + name = window.metric_name and + timestamp >= window.start and + timestamp <= window.end and + source in {dogstatsd_bucketed, dogstatsd_no_aggregation, check_shadow} + let partitions = partition_by_tags(matching_points, Monitor.partition_tags) + ensures: MonitorEvaluationRequested(window: window, partitions: partitions) + @guidance + -- When Monitor.partition_tags is empty, partition_by_tags produces a + -- single partition containing all matching points. When a configured tag + -- is missing from a point, that point stays in evaluation under a stable + -- missing-value partition key rather than being dropped. +} + +rule MonitorWindowUnknownWhenNoPartitionHasEnoughValidPoints { + when: MonitorEvaluationRequested(window, partitions) + requires: + partitions.all(p => (p.points where valid_number = true).count < config.monitor_min_points) + ensures: MonitorDecision.created( + metric_name: window.metric_name, + window: window, + state: unknown, + point_count: sum(partitions, p => (p.points where valid_number = true).count), + min: nil, + max: nil, + range: nil, + range_epsilon: Monitor.range_epsilon, + partition_tags: Monitor.partition_tags, + partition_key: nil, + partition_count: partitions.count + ) +} + +rule MonitorWindowBreachesWhenAnySufficientPartitionRangeExceedsEpsilon { + when: MonitorEvaluationRequested(window, partitions) + let sufficient = partitions where (points where valid_number = true).count >= config.monitor_min_points + let selected = max_by(sufficient, p => range((p.points where valid_number = true).value)) + requires: sufficient.any(p => range((p.points where valid_number = true).value) > Monitor.range_epsilon) + requires: range((selected.points where valid_number = true).value) > Monitor.range_epsilon + ensures: MonitorDecision.created( + metric_name: window.metric_name, + window: window, + state: breach, + point_count: (selected.points where valid_number = true).count, + min: min((selected.points where valid_number = true).value), + max: max((selected.points where valid_number = true).value), + range: range((selected.points where valid_number = true).value), + range_epsilon: Monitor.range_epsilon, + partition_tags: Monitor.partition_tags, + partition_key: selected.key, + partition_count: sufficient.count + ) +} + +rule MonitorWindowHealthyWhenAllSufficientPartitionsAreWithinEpsilon { + when: MonitorEvaluationRequested(window, partitions) + let sufficient = partitions where (points where valid_number = true).count >= config.monitor_min_points + let selected = max_by(sufficient, p => range((p.points where valid_number = true).value)) + requires: sufficient.count > 0 + requires: sufficient.all(p => range((p.points where valid_number = true).value) <= Monitor.range_epsilon) + ensures: MonitorDecision.created( + metric_name: window.metric_name, + window: window, + state: healthy, + point_count: (selected.points where valid_number = true).count, + min: min((selected.points where valid_number = true).value), + max: max((selected.points where valid_number = true).value), + range: range((selected.points where valid_number = true).value), + range_epsilon: Monitor.range_epsilon, + partition_tags: Monitor.partition_tags, + partition_key: selected.key, + partition_count: sufficient.count + ) +} + +rule InitialMonitorStateIsLogged { + when: decision: MonitorDecision.created + requires: Monitor.state = nil + ensures: Monitor.state = decision.state + ensures: MonitorStateTransitionLog.created( + metric_name: decision.metric_name, + from: nil, + to: decision.state, + initial: true, + dry_run: EgressPolicy.dry_run, + egress_mode: EgressPolicy.mode, + window: decision.window, + point_count: decision.point_count, + min: decision.min, + max: decision.max, + range: decision.range, + range_epsilon: decision.range_epsilon, + partition_tags: decision.partition_tags, + partition_key: decision.partition_key, + partition_count: decision.partition_count + ) +} + +rule MonitorStateTransitionIsAlwaysLogged { + when: decision: MonitorDecision.created + requires: Monitor.state != nil + requires: Monitor.state != decision.state + ensures: MonitorStateTransitionLog.created( + metric_name: decision.metric_name, + from: Monitor.state, + to: decision.state, + initial: false, + dry_run: EgressPolicy.dry_run, + egress_mode: EgressPolicy.mode, + window: decision.window, + point_count: decision.point_count, + min: decision.min, + max: decision.max, + range: decision.range, + range_epsilon: decision.range_epsilon, + partition_tags: decision.partition_tags, + partition_key: decision.partition_key, + partition_count: decision.partition_count + ) + ensures: Monitor.state = decision.state + @guidance + -- This rule is independent of dry-run mode. Dry-run controls whether a + -- decision changes egress state; it never suppresses monitor transition + -- diagnostics. +} + +rule RepeatedMonitorStateDoesNotCreateTransitionLog { + when: decision: MonitorDecision.created + requires: Monitor.state = decision.state + ensures: Monitor.transition_logs = Monitor.transition_logs +} + +rule EnabledMonitorModeEgressStartsSuppressed { + when: RuntimeConfigured(configuration) + requires: configuration.enabled = true + requires: configuration.monitor_mode = enabled + ensures: EgressPolicy.created( + mode: suppressed, + dry_run: false, + forwarding_ranges: [], + forwarded_series_ranges: [], + forwarded_sketch_ranges: [] + ) +} + +rule DryRunMonitorModeEgressStartsForwarding { + when: RuntimeConfigured(configuration) + requires: configuration.enabled = true + requires: configuration.monitor_mode = dry_run + ensures: EgressPolicy.created( + mode: forwarding, + dry_run: true, + forwarding_ranges: [TimeRange{start: nil, end: nil}], + forwarded_series_ranges: [], + forwarded_sketch_ranges: [] + ) + @guidance + -- Dry-run overrides the enabled mode's suppressed startup mode. There is + -- no startup suppression window when monitor mode is dry_run. +} + +rule DryRunMonitorDecisionDoesNotChangeEgressMode { + when: decision: MonitorDecision.created + requires: EgressPolicy.dry_run = true + ensures: EgressPolicy.mode = forwarding + ensures: EgressPolicy.forwarding_ranges = EgressPolicy.forwarding_ranges +} + +rule BreachOrUnknownOpensForwardingInNormalMode { + when: decision: MonitorDecision.created + requires: EgressPolicy.dry_run = false + requires: decision.state in {breach, unknown} + requires: EgressPolicy.mode = suppressed + ensures: EgressPolicy.mode = forwarding + ensures: EgressPolicy.forwarding_ranges = EgressPolicy.forwarding_ranges + [TimeRange{ + start: decision.window.start - config.egress_pre_trigger_window, + end: nil + }] +} + +rule BreachOrUnknownKeepsForwardingOpenInNormalMode { + when: decision: MonitorDecision.created + requires: EgressPolicy.dry_run = false + requires: decision.state in {breach, unknown} + requires: EgressPolicy.mode = forwarding + ensures: EgressPolicy.mode = forwarding +} + +rule HealthyDecisionSuppressesEgressInNormalMode { + when: decision: MonitorDecision.created + requires: EgressPolicy.dry_run = false + requires: decision.state = healthy + requires: EgressPolicy.mode = forwarding + ensures: EgressPolicy.mode = suppressed + ensures: open_range(EgressPolicy.forwarding_ranges).end = decision.window.end + config.egress_post_recovery_window +} + +rule HealthyDecisionWhileSuppressedDoesNotOpenEgress { + when: decision: MonitorDecision.created + requires: EgressPolicy.dry_run = false + requires: decision.state = healthy + requires: EgressPolicy.mode = suppressed + ensures: EgressPolicy.mode = suppressed +} + +rule ForwardingRangesAreClippedBySendDelay { + when: EgressRun(now) + let eligible_through = now - config.egress_send_delay + let candidates = EgressPolicy.forwarding_ranges where start = nil or start < eligible_through + ensures: + for range in candidates: + if range.end = nil or eligible_through < range.end: + ForwardRangePlanned(range: TimeRange{start: range.start, end: eligible_through}) + else: + ForwardRangePlanned(range: range) +} + +rule ForwardingSeriesRangeCannotBeForwardedTwice { + when: ForwardRangePlanned(range) + requires: range.is_forwardable + requires: not overlaps(range, EgressPolicy.forwarded_series_ranges) + ensures: ForwardAttempt.created(range: range, payload: series, state: planned, payload_count: 0) +} + +rule ForwardingSketchRangeCannotBeForwardedTwice { + when: ForwardRangePlanned(range) + requires: range.is_forwardable + requires: not overlaps(range, EgressPolicy.forwarded_sketch_ranges) + ensures: ForwardAttempt.created(range: range, payload: sketch, state: planned, payload_count: 0) +} + +rule SuccessfulSeriesForwardMarksSeriesRangeForwarded { + when: attempt: ForwardAttempt.state transitions_to forwarded + requires: attempt.payload = series + requires: attempt.payload_count > 0 + ensures: EgressPolicy.forwarded_series_ranges = EgressPolicy.forwarded_series_ranges + [attempt.range] +} + +rule SuccessfulSketchForwardMarksSketchRangeForwarded { + when: attempt: ForwardAttempt.state transitions_to forwarded + requires: attempt.payload = sketch + requires: attempt.payload_count > 0 + ensures: EgressPolicy.forwarded_sketch_ranges = EgressPolicy.forwarded_sketch_ranges + [attempt.range] +} + +rule EmptyForwardDoesNotMarkRangeForwarded { + when: attempt: ForwardAttempt.state transitions_to empty + requires: attempt.payload_count = 0 + ensures: EgressPolicy.forwarded_series_ranges = EgressPolicy.forwarded_series_ranges + ensures: EgressPolicy.forwarded_sketch_ranges = EgressPolicy.forwarded_sketch_ranges + @guidance + -- A later-arriving retained point with an older timestamp can still be + -- forwarded if it falls in the same range. Empty sends are not treated as + -- a completed range. +} + +rule FailedForwardRemainsEligibleForRetry { + when: attempt: ForwardAttempt.state transitions_to retry + ensures: EgressPolicy.forwarded_series_ranges = EgressPolicy.forwarded_series_ranges + ensures: EgressPolicy.forwarded_sketch_ranges = EgressPolicy.forwarded_sketch_ranges +} + +------------------------------------------------------------ +-- Invariants +------------------------------------------------------------ + +invariant MonitorMetricHasSourceIndependentSemantics { + -- The monitor's metric_name is a query over retained points. DogStatsD is + -- one producer, and monitor configuration auto-admits the same name from + -- DogStatsD for convenience, but any retained point from a supported source + -- with the same metric name can affect monitor decisions. +} + +invariant DogStatsDAdditionalMetricNamesAreNotMonitorEnablement { + -- dogstatsd_metric_names controls only additional DogStatsD metrics to + -- retain. It must not be interpreted as a prerequisite for using a + -- DogStatsD-origin metric as the monitor trigger. +} + +invariant MonitorPartitionTagsPreventCrossPopulationRangeInflation { + -- When monitor_partition_tags is non-empty, the monitor evaluates range + -- independently per selected tag-value tuple. A breach in any sufficient + -- partition breaches the monitor; sparse partitions are ignored unless no + -- partition has enough valid points, in which case the decision is unknown. +} + +invariant DryRunChangesOnlyEgressControlNotMonitorObservation { + -- In dry-run mode, monitor windows are evaluated and monitor state + -- transitions are logged exactly as in normal mode. The only semantic + -- difference is that monitor decisions do not suppress or reopen egress; + -- egress starts and remains forwarding. +} diff --git a/pkg/metriclookback/monitor/BUILD.bazel b/pkg/metriclookback/monitor/BUILD.bazel new file mode 100644 index 000000000000..270068dce78e --- /dev/null +++ b/pkg/metriclookback/monitor/BUILD.bazel @@ -0,0 +1,17 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "monitor", + srcs = ["monitor.go"], + importpath = "github.com/DataDog/datadog-agent/pkg/metriclookback/monitor", + visibility = ["//visibility:public"], + deps = ["//comp/core/telemetry/impl"], +) + +go_test( + name = "monitor_test", + srcs = ["monitor_test.go"], + embed = [":monitor"], + gotags = ["test"], + deps = ["@com_github_stretchr_testify//require"], +) diff --git a/pkg/metriclookback/monitor/monitor.go b/pkg/metriclookback/monitor/monitor.go new file mode 100644 index 000000000000..d40c85588ff8 --- /dev/null +++ b/pkg/metriclookback/monitor/monitor.go @@ -0,0 +1,466 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package monitor provides a small materialized health view for metric +// lookback. The initial monitor watches one retained metric and reports whether +// recent high-resolution points vary by more than a configured range epsilon. +package monitor + +import ( + "errors" + "math" + "sort" + "strings" + "sync" + "time" + + telemetryimpl "github.com/DataDog/datadog-agent/comp/core/telemetry/impl" +) + +const ( + // DefaultEvaluationInterval is the approximate retained-point window size used + // by the monitor. + DefaultEvaluationInterval = 30 * time.Second + // DefaultMinPoints is the minimum number of retained points required to compute + // a range. A single valid point cannot show whether the window moved by more + // than epsilon. + DefaultMinPoints = 2 +) + +var ( + tlmMonitorEvaluations = telemetryimpl.GetCompatComponent().NewCounter("metric_lookback", "monitor_evaluations", []string{"result"}, "Count of metric lookback monitor window evaluations") + tlmMonitorWindowMin = telemetryimpl.GetCompatComponent().NewGauge("metric_lookback", "monitor_window_min", nil, "Minimum value in the last metric lookback monitor evaluation window") + tlmMonitorWindowMax = telemetryimpl.GetCompatComponent().NewGauge("metric_lookback", "monitor_window_max", nil, "Maximum value in the last metric lookback monitor evaluation window") + tlmMonitorWindowRange = telemetryimpl.GetCompatComponent().NewGauge("metric_lookback", "monitor_window_range", nil, "Maximum minus minimum value in the last metric lookback monitor evaluation window") + tlmMonitorRangeEpsilon = telemetryimpl.GetCompatComponent().NewGauge("metric_lookback", "monitor_range_epsilon", nil, "Configured metric lookback monitor range epsilon") + tlmMonitorWindowPts = telemetryimpl.GetCompatComponent().NewGauge("metric_lookback", "monitor_window_points", nil, "Number of points in the last metric lookback monitor evaluation window") +) + +// Point is a retained scalar point read by the monitor. +type Point struct { + Ts time.Time + Value float64 + Tags []string +} + +// PointReader returns retained points for a metric in an inclusive time range. +type PointReader interface { + PointsBetween(metricName string, from, to time.Time) []Point +} + +// PointReaderFunc adapts a function to PointReader. +type PointReaderFunc func(metricName string, from, to time.Time) []Point + +// PointsBetween implements PointReader. +func (f PointReaderFunc) PointsBetween(metricName string, from, to time.Time) []Point { + if f == nil { + return nil + } + return f(metricName, from, to) +} + +// State is the monitor's health classification for one evaluation window. +type State int + +const ( + // Unknown means the monitor did not have enough valid data to prove the + // watched metric was healthy. + Unknown State = iota + // Healthy means the valid point range in the window was at or below the + // configured range epsilon. + Healthy + // Breach means the valid point range in the window was greater than the + // configured range epsilon. + Breach +) + +// String returns a stable label for telemetry and diagnostics. +func (s State) String() string { + switch s { + case Healthy: + return "healthy" + case Breach: + return "breach" + default: + return "unknown" + } +} + +// Decision describes one evaluated monitor window. +type Decision struct { + MetricName string + WindowFrom time.Time + WindowTo time.Time + State State + Min float64 + Max float64 + Range float64 + RangeEpsilon float64 + PointCount int + PartitionTags []string + PartitionKey string + PartitionCount int +} + +// DecisionSink receives monitor decisions. Implementations should avoid +// blocking the caller; monitor evaluation runs synchronously on the selected +// metric's append path. +type DecisionSink interface { + OnDecision(Decision) +} + +// DecisionSinkFunc adapts a function to DecisionSink. +type DecisionSinkFunc func(Decision) + +// OnDecision implements DecisionSink. +func (f DecisionSinkFunc) OnDecision(decision Decision) { + if f != nil { + f(decision) + } +} + +// Config controls a Watcher. Zero values select conservative defaults so the +// public config surface can enable the monitor with just a metric name and range +// epsilon, and can optionally partition evaluation by tag keys. +type Config struct { + // MetricName is the exact metric name to watch. + MetricName string + // RangeEpsilon is the maximum healthy max-min range for a completed window. A + // window breaches when its valid point range is strictly greater than + // RangeEpsilon. + RangeEpsilon float64 + // PartitionTags are tag keys used to partition range evaluation. When empty, + // all retained points for MetricName are evaluated together. When set, each + // selected tag-value tuple is evaluated independently and any sufficient + // partition can breach the monitor. + PartitionTags []string + // EvaluationInterval is the approximate high-resolution window size. Defaults + // to DefaultEvaluationInterval. + EvaluationInterval time.Duration + // MinPoints is the minimum number of valid points required in a window. + // Defaults to DefaultMinPoints. Values below 2 are raised to 2 because a + // single point cannot establish a range. + MinPoints int +} + +// Watcher periodically evaluates a selected metric from a retained point store. +// It reports a decision for each completed evaluation window. It is safe for +// concurrent use. +type Watcher struct { + metricName string + rangeEpsilon float64 + evaluationInterval time.Duration + minPoints int + partitionTags []string + reader PointReader + sink DecisionSink + + // observeMu serializes window evaluation and decision delivery in the same + // order windows are claimed. + observeMu sync.Mutex + + mu sync.Mutex + lastEvaluation time.Time + decisions uint64 + breaches uint64 +} + +// New creates a Watcher. It returns nil when the configuration is inert or when +// reader/sink is nil, so callers can treat a nil Watcher as disabled. +func New(cfg Config, reader PointReader, sink DecisionSink) (*Watcher, error) { + if cfg.MetricName == "" || reader == nil || sink == nil { + return nil, nil + } + cfg, err := normalizeConfig(cfg) + if err != nil { + return nil, err + } + return &Watcher{ + metricName: cfg.MetricName, + rangeEpsilon: cfg.RangeEpsilon, + partitionTags: append([]string(nil), cfg.PartitionTags...), + evaluationInterval: cfg.EvaluationInterval, + minPoints: cfg.MinPoints, + reader: reader, + sink: sink, + }, nil +} + +func normalizeConfig(cfg Config) (Config, error) { + if cfg.RangeEpsilon < 0 { + return Config{}, errors.New("range epsilon must be non-negative") + } + cfg.PartitionTags = normalizePartitionTags(cfg.PartitionTags) + if cfg.EvaluationInterval <= 0 { + cfg.EvaluationInterval = DefaultEvaluationInterval + } + if cfg.MinPoints <= 0 { + cfg.MinPoints = DefaultMinPoints + } + if cfg.MinPoints < 2 { + cfg.MinPoints = 2 + } + return cfg, nil +} + +// MetricName returns the metric this watcher evaluates. +func (w *Watcher) MetricName() string { + if w == nil { + return "" + } + return w.metricName +} + +// Observe records that the watched metric has a newly admitted point at +// observedAt. When approximately one evaluation interval has elapsed, Observe +// reads the relevant metric points back from the retention store and evaluates +// the completed window. The sample value is intentionally not passed here: the +// monitor is a materialized view over retention, not a side tap with separate +// value storage. +func (w *Watcher) Observe(name string, observedAt time.Time) bool { + if w == nil || name != w.metricName || observedAt.IsZero() { + return false + } + + // Serialize window selection, evaluation, and decision delivery so concurrent + // producers cannot emit monitor decisions out of chronological order. + w.observeMu.Lock() + defer w.observeMu.Unlock() + + from, to, shouldEvaluate := w.nextWindow(observedAt) + if !shouldEvaluate { + return false + } + return w.evaluateWindow(from, to) == Breach +} + +func (w *Watcher) nextWindow(observedAt time.Time) (time.Time, time.Time, bool) { + w.mu.Lock() + defer w.mu.Unlock() + + if w.lastEvaluation.IsZero() { + w.lastEvaluation = observedAt + return time.Time{}, time.Time{}, false + } + if observedAt.Before(w.lastEvaluation) { + // Ignore out-of-order monitor clocks. The point is still retained and can be + // included in a later egress range if it falls inside one. + return time.Time{}, time.Time{}, false + } + if observedAt.Sub(w.lastEvaluation) < w.evaluationInterval { + return time.Time{}, time.Time{}, false + } + + from := w.lastEvaluation + to := observedAt + w.lastEvaluation = observedAt + return from, to, true +} + +func (w *Watcher) evaluateWindow(from, to time.Time) State { + points := w.reader.PointsBetween(w.metricName, from, to) + tlmMonitorWindowPts.Set(float64(len(points))) + tlmMonitorRangeEpsilon.Set(w.rangeEpsilon) + + partitionStats := windowPartitionStats(points, w.partitionTags) + selected, sufficientCount, ok := selectMonitorPartition(partitionStats, w.minPoints) + if !ok { + decision := Decision{ + MetricName: w.metricName, + WindowFrom: from, + WindowTo: to, + State: Unknown, + RangeEpsilon: w.rangeEpsilon, + PointCount: totalValidPointCount(partitionStats), + PartitionTags: append([]string(nil), w.partitionTags...), + PartitionCount: len(partitionStats), + } + w.recordDecision(decision) + tlmMonitorEvaluations.Inc("unknown") + w.sink.OnDecision(decision) + return Unknown + } + + state := Healthy + if selected.valueRange > w.rangeEpsilon { + state = Breach + } + tlmMonitorWindowMin.Set(selected.min) + tlmMonitorWindowMax.Set(selected.max) + tlmMonitorWindowRange.Set(selected.valueRange) + decision := Decision{ + MetricName: w.metricName, + WindowFrom: from, + WindowTo: to, + State: state, + Min: selected.min, + Max: selected.max, + Range: selected.valueRange, + RangeEpsilon: w.rangeEpsilon, + PointCount: selected.validCount, + PartitionTags: append([]string(nil), w.partitionTags...), + PartitionKey: selected.key, + PartitionCount: sufficientCount, + } + w.recordDecision(decision) + tlmMonitorEvaluations.Inc(state.String()) + w.sink.OnDecision(decision) + return state +} + +func windowMinMax(points []Point) (float64, float64, int, bool) { + minValue := math.Inf(1) + maxValue := math.Inf(-1) + validCount := 0 + for _, point := range points { + if math.IsNaN(point.Value) || math.IsInf(point.Value, 0) { + continue + } + validCount++ + if point.Value < minValue { + minValue = point.Value + } + if point.Value > maxValue { + maxValue = point.Value + } + } + if validCount == 0 { + return 0, 0, 0, false + } + return minValue, maxValue, validCount, true +} + +type partitionWindowStats struct { + key string + min float64 + max float64 + valueRange float64 + validCount int +} + +func windowPartitionStats(points []Point, partitionTags []string) []partitionWindowStats { + byKey := make(map[string][]Point) + for _, point := range points { + key := monitorPartitionKey(point.Tags, partitionTags) + byKey[key] = append(byKey[key], point) + } + keys := make([]string, 0, len(byKey)) + for key := range byKey { + keys = append(keys, key) + } + sort.Strings(keys) + + stats := make([]partitionWindowStats, 0, len(keys)) + for _, key := range keys { + minValue, maxValue, validCount, ok := windowMinMax(byKey[key]) + if !ok { + stats = append(stats, partitionWindowStats{key: key}) + continue + } + stats = append(stats, partitionWindowStats{ + key: key, + min: minValue, + max: maxValue, + valueRange: maxValue - minValue, + validCount: validCount, + }) + } + return stats +} + +func selectMonitorPartition(stats []partitionWindowStats, minPoints int) (partitionWindowStats, int, bool) { + var selected partitionWindowStats + sufficientCount := 0 + for _, stat := range stats { + if stat.validCount < minPoints { + continue + } + sufficientCount++ + if sufficientCount == 1 || stat.valueRange > selected.valueRange || (stat.valueRange == selected.valueRange && stat.key < selected.key) { + selected = stat + } + } + return selected, sufficientCount, sufficientCount > 0 +} + +func totalValidPointCount(stats []partitionWindowStats) int { + total := 0 + for _, stat := range stats { + total += stat.validCount + } + return total +} + +func monitorPartitionKey(tags []string, partitionTags []string) string { + if len(partitionTags) == 0 { + return "" + } + valuesByName := make(map[string]string, len(tags)) + for _, tag := range tags { + name, value, ok := strings.Cut(tag, ":") + if !ok || name == "" { + continue + } + if _, exists := valuesByName[name]; !exists { + valuesByName[name] = value + } + } + + parts := make([]string, 0, len(partitionTags)) + for _, name := range partitionTags { + value, ok := valuesByName[name] + if !ok { + value = "" + } + parts = append(parts, name+":"+value) + } + return strings.Join(parts, ",") +} + +func normalizePartitionTags(tags []string) []string { + seen := make(map[string]struct{}, len(tags)) + out := make([]string, 0, len(tags)) + for _, tag := range tags { + name := strings.TrimSpace(strings.TrimSuffix(tag, ":")) + if name == "" { + continue + } + if _, found := seen[name]; found { + continue + } + seen[name] = struct{}{} + out = append(out, name) + } + return out +} + +func (w *Watcher) recordDecision(decision Decision) { + w.mu.Lock() + defer w.mu.Unlock() + w.decisions++ + if decision.State == Breach { + w.breaches++ + } +} + +// Decisions returns how many monitor decisions this watcher has emitted. +func (w *Watcher) Decisions() uint64 { + if w == nil { + return 0 + } + w.mu.Lock() + defer w.mu.Unlock() + return w.decisions +} + +// Breaches returns how many breach decisions this watcher has emitted. +func (w *Watcher) Breaches() uint64 { + if w == nil { + return 0 + } + w.mu.Lock() + defer w.mu.Unlock() + return w.breaches +} diff --git a/pkg/metriclookback/monitor/monitor_test.go b/pkg/metriclookback/monitor/monitor_test.go new file mode 100644 index 000000000000..0488f8752f87 --- /dev/null +++ b/pkg/metriclookback/monitor/monitor_test.go @@ -0,0 +1,460 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package monitor + +import ( + "math" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type inMemoryReader struct { + points []Point +} + +func (r *inMemoryReader) PointsBetween(_ string, from, to time.Time) []Point { + var out []Point + for _, point := range r.points { + if !from.IsZero() && point.Ts.Before(from) { + continue + } + if !to.IsZero() && point.Ts.After(to) { + continue + } + out = append(out, point) + } + return out +} + +func TestWatcherEmitsHealthyWhenWindowRangeEqualsRangeEpsilon(t *testing.T) { + start := time.Unix(100, 0) + reader := &inMemoryReader{} + decisions := make(chan Decision, 1) + watcher := requireWatcher(t, Config{ + MetricName: "target", + RangeEpsilon: 0.25, + EvaluationInterval: 30 * time.Second, + MinPoints: 6, + }, reader, DecisionSinkFunc(func(decision Decision) { + decisions <- decision + })) + + appendWindow(reader, start, 0, 30, 1) + reader.points = append(reader.points, Point{Ts: start.Add(15 * time.Second), Value: 1.25}) + assertObserveSequence(t, watcher, start, false) + + decision := requireDecision(t, decisions) + require.Equal(t, Healthy, decision.State) + require.Equal(t, float64(1), decision.Min) + require.Equal(t, 1.25, decision.Max) + require.Equal(t, 0.25, decision.Range) + require.Equal(t, 0.25, decision.RangeEpsilon) + require.Equal(t, 32, decision.PointCount) + require.Equal(t, uint64(1), watcher.Decisions()) + require.Equal(t, uint64(0), watcher.Breaches()) +} + +func TestWatcherEmitsBreachWhenWindowRangeExceedsRangeEpsilon(t *testing.T) { + start := time.Unix(100, 0) + reader := &inMemoryReader{} + decisions := make(chan Decision, 1) + watcher := requireWatcher(t, Config{ + MetricName: "target", + RangeEpsilon: 0.05, + EvaluationInterval: 30 * time.Second, + MinPoints: 6, + }, reader, DecisionSinkFunc(func(decision Decision) { + decisions <- decision + })) + + appendWindow(reader, start, 0, 30, 1) + reader.points = append(reader.points, Point{Ts: start.Add(15 * time.Second), Value: 1.06}) + assertObserveSequence(t, watcher, start, true) + + decision := requireDecision(t, decisions) + require.Equal(t, Breach, decision.State) + require.Equal(t, float64(1), decision.Min) + require.Equal(t, 1.06, decision.Max) + require.InDelta(t, 0.06, decision.Range, 1e-12) + require.Equal(t, 0.05, decision.RangeEpsilon) + require.Equal(t, uint64(1), watcher.Decisions()) + require.Equal(t, uint64(1), watcher.Breaches()) +} + +func TestWatcherEmitsHealthyForStableLowAbsoluteValue(t *testing.T) { + start := time.Unix(100, 0) + reader := &inMemoryReader{} + decisions := make(chan Decision, 1) + watcher := requireWatcher(t, Config{ + MetricName: "target", + RangeEpsilon: 0.05, + EvaluationInterval: 30 * time.Second, + MinPoints: 6, + }, reader, DecisionSinkFunc(func(decision Decision) { + decisions <- decision + })) + + appendWindow(reader, start, 0, 30, -100) + assertObserveSequence(t, watcher, start, false) + + decision := requireDecision(t, decisions) + require.Equal(t, Healthy, decision.State) + require.Equal(t, float64(-100), decision.Min) + require.Equal(t, float64(-100), decision.Max) + require.Equal(t, float64(0), decision.Range) +} + +func TestWatcherPartitionsWindowRangeByConfiguredTags(t *testing.T) { + start := time.Unix(100, 0) + reader := &inMemoryReader{points: []Point{ + {Ts: start, Value: 1.00, Tags: []string{"az:a"}}, + {Ts: start.Add(time.Second), Value: 1.01, Tags: []string{"az:a"}}, + {Ts: start.Add(2 * time.Second), Value: 2.00, Tags: []string{"az:b"}}, + {Ts: start.Add(3 * time.Second), Value: 2.01, Tags: []string{"az:b"}}, + }} + decisions := make(chan Decision, 1) + watcher := requireWatcher(t, Config{ + MetricName: "target", + RangeEpsilon: 0.05, + PartitionTags: []string{"az"}, + EvaluationInterval: 30 * time.Second, + MinPoints: 2, + }, reader, DecisionSinkFunc(func(decision Decision) { + decisions <- decision + })) + + require.False(t, watcher.Observe("target", start)) + require.False(t, watcher.Observe("target", start.Add(30*time.Second))) + + decision := requireDecision(t, decisions) + require.Equal(t, Healthy, decision.State) + require.Equal(t, []string{"az"}, decision.PartitionTags) + require.Equal(t, 2, decision.PartitionCount) + require.Contains(t, []string{"az:a", "az:b"}, decision.PartitionKey) + require.InDelta(t, 0.01, decision.Range, 1e-12) +} + +func TestWatcherBreachesWhenAnyPartitionExceedsRangeEpsilon(t *testing.T) { + start := time.Unix(100, 0) + reader := &inMemoryReader{points: []Point{ + {Ts: start, Value: 1.00, Tags: []string{"az:a"}}, + {Ts: start.Add(time.Second), Value: 1.01, Tags: []string{"az:a"}}, + {Ts: start.Add(2 * time.Second), Value: 2.00, Tags: []string{"az:b"}}, + {Ts: start.Add(3 * time.Second), Value: 2.20, Tags: []string{"az:b"}}, + }} + decisions := make(chan Decision, 1) + watcher := requireWatcher(t, Config{ + MetricName: "target", + RangeEpsilon: 0.05, + PartitionTags: []string{"az"}, + EvaluationInterval: 30 * time.Second, + MinPoints: 2, + }, reader, DecisionSinkFunc(func(decision Decision) { + decisions <- decision + })) + + require.False(t, watcher.Observe("target", start)) + require.True(t, watcher.Observe("target", start.Add(30*time.Second))) + + decision := requireDecision(t, decisions) + require.Equal(t, Breach, decision.State) + require.Equal(t, "az:b", decision.PartitionKey) + require.Equal(t, 2, decision.PartitionCount) + require.Equal(t, 2.00, decision.Min) + require.Equal(t, 2.20, decision.Max) + require.InDelta(t, 0.20, decision.Range, 1e-12) +} + +func TestWatcherIgnoresSparsePartitionsWhenClassifyingHealthy(t *testing.T) { + start := time.Unix(100, 0) + reader := &inMemoryReader{points: []Point{ + {Ts: start, Value: 1.00, Tags: []string{"az:a"}}, + {Ts: start.Add(time.Second), Value: 1.01, Tags: []string{"az:a"}}, + {Ts: start.Add(2 * time.Second), Value: 100.00, Tags: []string{"az:b"}}, + }} + decisions := make(chan Decision, 1) + watcher := requireWatcher(t, Config{ + MetricName: "target", + RangeEpsilon: 0.05, + PartitionTags: []string{"az"}, + EvaluationInterval: 30 * time.Second, + MinPoints: 2, + }, reader, DecisionSinkFunc(func(decision Decision) { + decisions <- decision + })) + + require.False(t, watcher.Observe("target", start)) + require.False(t, watcher.Observe("target", start.Add(30*time.Second))) + + decision := requireDecision(t, decisions) + require.Equal(t, Healthy, decision.State) + require.Equal(t, "az:a", decision.PartitionKey) + require.Equal(t, 1, decision.PartitionCount) +} + +func TestWatcherUsesMissingTagPartition(t *testing.T) { + start := time.Unix(100, 0) + reader := &inMemoryReader{points: []Point{ + {Ts: start, Value: 1.00, Tags: []string{"env:prod"}}, + {Ts: start.Add(time.Second), Value: 1.20, Tags: []string{"env:prod"}}, + }} + decisions := make(chan Decision, 1) + watcher := requireWatcher(t, Config{ + MetricName: "target", + RangeEpsilon: 0.05, + PartitionTags: []string{"az"}, + EvaluationInterval: 30 * time.Second, + MinPoints: 2, + }, reader, DecisionSinkFunc(func(decision Decision) { + decisions <- decision + })) + + require.False(t, watcher.Observe("target", start)) + require.True(t, watcher.Observe("target", start.Add(30*time.Second))) + + decision := requireDecision(t, decisions) + require.Equal(t, Breach, decision.State) + require.Equal(t, "az:", decision.PartitionKey) +} + +func TestWatcherEmitsUnknownWhenNoPartitionHasEnoughPoints(t *testing.T) { + start := time.Unix(100, 0) + reader := &inMemoryReader{points: []Point{ + {Ts: start, Value: 1.00, Tags: []string{"az:a"}}, + {Ts: start.Add(time.Second), Value: 2.00, Tags: []string{"az:b"}}, + }} + decisions := make(chan Decision, 1) + watcher := requireWatcher(t, Config{ + MetricName: "target", + RangeEpsilon: 0.05, + PartitionTags: []string{"az"}, + EvaluationInterval: 30 * time.Second, + MinPoints: 2, + }, reader, DecisionSinkFunc(func(decision Decision) { + decisions <- decision + })) + + require.False(t, watcher.Observe("target", start)) + require.False(t, watcher.Observe("target", start.Add(30*time.Second))) + + decision := requireDecision(t, decisions) + require.Equal(t, Unknown, decision.State) + require.Equal(t, 2, decision.PointCount) + require.Equal(t, 2, decision.PartitionCount) +} + +func TestWatcherEmitsUnknownForSparseWindow(t *testing.T) { + start := time.Unix(100, 0) + reader := &inMemoryReader{points: []Point{{Ts: start, Value: 2}}} + decisions := make(chan Decision, 1) + watcher := requireWatcher(t, Config{ + MetricName: "target", + RangeEpsilon: 0.05, + EvaluationInterval: 30 * time.Second, + MinPoints: 6, + }, reader, DecisionSinkFunc(func(decision Decision) { + decisions <- decision + })) + + require.False(t, watcher.Observe("target", start)) + require.False(t, watcher.Observe("target", start.Add(30*time.Second))) + + decision := requireDecision(t, decisions) + require.Equal(t, Unknown, decision.State) + require.Equal(t, 1, decision.PointCount) + require.Equal(t, 0.05, decision.RangeEpsilon) + require.Equal(t, uint64(1), watcher.Decisions()) +} + +func TestWatcherIgnoresNaNAndInfinityInWindowRange(t *testing.T) { + start := time.Unix(100, 0) + reader := &inMemoryReader{points: []Point{ + {Ts: start, Value: math.NaN()}, + {Ts: start.Add(time.Second), Value: math.Inf(1)}, + {Ts: start.Add(2 * time.Second), Value: 10}, + {Ts: start.Add(3 * time.Second), Value: 10.04}, + }} + decisions := make(chan Decision, 1) + watcher := requireWatcher(t, Config{ + MetricName: "target", + RangeEpsilon: 0.05, + EvaluationInterval: 30 * time.Second, + MinPoints: 2, + }, reader, DecisionSinkFunc(func(decision Decision) { + decisions <- decision + })) + + require.False(t, watcher.Observe("target", start)) + require.False(t, watcher.Observe("target", start.Add(30*time.Second))) + + decision := requireDecision(t, decisions) + require.Equal(t, Healthy, decision.State) + require.Equal(t, 2, decision.PointCount) + require.Equal(t, float64(10), decision.Min) + require.Equal(t, 10.04, decision.Max) + require.InDelta(t, 0.04, decision.Range, 1e-12) +} + +func TestWatcherRejectsNegativeRangeEpsilon(t *testing.T) { + watcher, err := New(Config{ + MetricName: "target", + RangeEpsilon: -1, + EvaluationInterval: 30 * time.Second, + MinPoints: 2, + }, &inMemoryReader{}, DecisionSinkFunc(func(Decision) { + require.FailNow(t, "decision should not be emitted") + })) + + require.Nil(t, watcher) + require.ErrorContains(t, err, "range epsilon") +} + +func TestWatcherIgnoresNonMatchingMetric(t *testing.T) { + watcher := requireWatcher(t, Config{MetricName: "target"}, PointReaderFunc(func(_ string, _, _ time.Time) []Point { + require.FailNow(t, "reader should not be called") + return nil + }), DecisionSinkFunc(func(Decision) { + require.FailNow(t, "decision should not be emitted") + })) + + require.False(t, watcher.Observe("other", time.Now())) + require.Equal(t, uint64(0), watcher.Decisions()) +} + +func TestWatcherIgnoresOutOfOrderObservedTime(t *testing.T) { + start := time.Unix(100, 0) + reader := &inMemoryReader{} + watcher := requireWatcher(t, Config{MetricName: "target"}, reader, DecisionSinkFunc(func(Decision) { + require.FailNow(t, "decision should not be emitted") + })) + + require.False(t, watcher.Observe("target", start)) + require.False(t, watcher.Observe("target", start.Add(-time.Second))) + require.Equal(t, uint64(0), watcher.Decisions()) +} + +func TestWatcherSerializesConcurrentDecisionsInWindowOrder(t *testing.T) { + start := time.Unix(100, 0) + firstWindowTo := start.Add(30 * time.Second) + secondWindowTo := start.Add(60 * time.Second) + + firstReadStarted := make(chan struct{}) + releaseFirstRead := make(chan struct{}) + secondReadStarted := make(chan struct{}) + var firstOnce sync.Once + var secondOnce sync.Once + defer func() { + select { + case <-releaseFirstRead: + default: + close(releaseFirstRead) + } + }() + + reader := PointReaderFunc(func(_ string, from, to time.Time) []Point { + switch { + case from.Equal(start) && to.Equal(firstWindowTo): + firstOnce.Do(func() { close(firstReadStarted) }) + <-releaseFirstRead + return []Point{ + {Ts: start, Value: 1}, + {Ts: firstWindowTo, Value: 2}, + } + case from.Equal(firstWindowTo) && to.Equal(secondWindowTo): + secondOnce.Do(func() { close(secondReadStarted) }) + return []Point{ + {Ts: firstWindowTo, Value: 3}, + {Ts: secondWindowTo, Value: 3.1}, + } + default: + return nil + } + }) + decisions := make(chan Decision, 2) + watcher := requireWatcher(t, Config{ + MetricName: "target", + RangeEpsilon: 0.5, + EvaluationInterval: 30 * time.Second, + MinPoints: 2, + }, reader, DecisionSinkFunc(func(decision Decision) { + decisions <- decision + })) + + require.False(t, watcher.Observe("target", start)) + + firstResult := make(chan bool, 1) + go func() { + firstResult <- watcher.Observe("target", firstWindowTo) + }() + select { + case <-firstReadStarted: + case <-time.After(time.Second): + require.FailNow(t, "timed out waiting for first evaluation to start") + } + + secondResult := make(chan bool, 1) + go func() { + secondResult <- watcher.Observe("target", secondWindowTo) + }() + + select { + case <-secondReadStarted: + require.FailNow(t, "second evaluation started before the first decision was delivered") + case decision := <-decisions: + require.FailNow(t, "decision emitted before first evaluation was released", "decision=%+v", decision) + case <-time.After(50 * time.Millisecond): + } + + close(releaseFirstRead) + + firstDecision := requireDecision(t, decisions) + require.Equal(t, firstWindowTo, firstDecision.WindowTo) + require.Equal(t, Breach, firstDecision.State) + + secondDecision := requireDecision(t, decisions) + require.Equal(t, secondWindowTo, secondDecision.WindowTo) + require.Equal(t, Healthy, secondDecision.State) + + require.True(t, <-firstResult) + require.False(t, <-secondResult) +} + +func requireWatcher(t *testing.T, cfg Config, reader PointReader, sink DecisionSink) *Watcher { + t.Helper() + watcher, err := New(cfg, reader, sink) + require.NoError(t, err) + require.NotNil(t, watcher) + return watcher +} + +func appendWindow(reader *inMemoryReader, start time.Time, fromSecond, toSecond int, value float64) { + for second := fromSecond; second <= toSecond; second++ { + reader.points = append(reader.points, Point{ + Ts: start.Add(time.Duration(second) * time.Second), + Value: value, + }) + } +} + +func assertObserveSequence(t *testing.T, watcher *Watcher, start time.Time, wantBreach bool) { + t.Helper() + require.False(t, watcher.Observe("target", start)) + require.Equal(t, wantBreach, watcher.Observe("target", start.Add(30*time.Second))) +} + +func requireDecision(t *testing.T, decisions <-chan Decision) Decision { + t.Helper() + select { + case decision := <-decisions: + return decision + case <-time.After(time.Second): + require.FailNow(t, "timed out waiting for monitor decision") + return Decision{} + } +} diff --git a/pkg/metriclookback/retention.go b/pkg/metriclookback/retention.go new file mode 100644 index 000000000000..5c39d8b80883 --- /dev/null +++ b/pkg/metriclookback/retention.go @@ -0,0 +1,355 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package metriclookback owns the shared point-retention substrate used by +// metric lookback producers. The first producers are selected DogStatsD +// no-aggregation samples, selected normal DogStatsD buckets, and later +// check/shadow-check samples. +package metriclookback + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/DataDog/datadog-agent/pkg/metriclookback/monitor" + "github.com/DataDog/datadog-agent/pkg/metriclookback/ringbuffer" + "github.com/DataDog/datadog-agent/pkg/metrics" + "github.com/DataDog/datadog-agent/pkg/serializer" +) + +// Retention owns the metric lookback point rings. It is intentionally outside +// pkg/aggregator so binaries that do not enable lookback do not link the +// concrete buffer code. +type Retention struct { + bufferMu sync.Mutex + bufferOptions ringbuffer.Options + buffer *ringbuffer.Buffer + + sketchMu sync.Mutex + sketchOptions ringbuffer.Options + sketchBuffer *ringbuffer.SketchBuffer + + monitorMu sync.RWMutex + monitor *monitor.Watcher +} + +// NewRetention creates a metric lookback retention backend with the provided +// ring buffer options. The backing rings are allocated lazily on first admitted +// point so enabled-but-unmatched DogStatsD lookback does not reserve retention +// memory. +func NewRetention(opts ringbuffer.Options) *Retention { + return &Retention{bufferOptions: opts, sketchOptions: opts} +} + +func (r *Retention) getBuffer(create bool) *ringbuffer.Buffer { + if r == nil { + return nil + } + r.bufferMu.Lock() + defer r.bufferMu.Unlock() + if r.buffer == nil && create { + r.buffer = ringbuffer.New(r.bufferOptions) + } + return r.buffer +} + +func (r *Retention) getSketchBuffer(create bool) *ringbuffer.SketchBuffer { + if r == nil { + return nil + } + r.sketchMu.Lock() + defer r.sketchMu.Unlock() + if r.sketchBuffer == nil && create { + r.sketchBuffer = ringbuffer.NewSketchBuffer(r.sketchOptions) + } + return r.sketchBuffer +} + +// SetMonitor installs the optional monitor notified by shadow-check sender +// writes. Retention remains storage-only for ordinary AppendSamples callers; this +// hook lets the sender integration use the same materialized-read monitor path +// as DogStatsD without adding a second hot-path tap. +func (r *Retention) SetMonitor(watcher *monitor.Watcher) { + if r == nil { + return + } + r.monitorMu.Lock() + defer r.monitorMu.Unlock() + r.monitor = watcher +} + +func (r *Retention) getMonitor() *monitor.Watcher { + if r == nil { + return nil + } + r.monitorMu.RLock() + defer r.monitorMu.RUnlock() + return r.monitor +} + +// ObserveSamples notifies the optional monitor that retained samples were +// admitted. Source-specific integrations call this after successful AppendSamples +// writes so retention itself can stay independent of producer APIs. +func (r *Retention) ObserveSamples(samples []metrics.MetricSample) { + watcher := r.getMonitor() + if watcher == nil { + return + } + for i := range samples { + watcher.Observe(samples[i].Name, sampleObservedAt(samples[i])) + } +} + +func sampleObservedAt(sample metrics.MetricSample) time.Time { + if sample.Timestamp > 0 { + return time.UnixMicro(int64(sample.Timestamp * 1e6)) + } + return time.Now() +} + +// AppendSamples stores scalar samples from the provided source in the shared +// retention ring. +func (r *Retention) AppendSamples(ctx context.Context, source ringbuffer.Source, samples []metrics.MetricSample) error { + buffer := r.getBuffer(true) + if buffer == nil { + return nil + } + return buffer.AppendSamples(ctx, source, samples) +} + +// AppendSerie stores an already-normalized series from the provided source in +// the shared retention ring. It is used for DogStatsD no-aggregation after the +// worker has built the same series it sends to the serializer. +func (r *Retention) AppendSerie(ctx context.Context, source ringbuffer.Source, serie *metrics.Serie) error { + buffer := r.getBuffer(true) + if buffer == nil { + return nil + } + return buffer.AppendSerie(ctx, source, serie) +} + +// AppendSketchSeries stores an already-finalized sketch series from the provided +// source in the shared retention ring. +func (r *Retention) AppendSketchSeries(ctx context.Context, source ringbuffer.Source, series *metrics.SketchSeries) error { + if r == nil { + return nil + } + sketchBuffer := r.getSketchBuffer(true) + if sketchBuffer == nil { + return nil + } + return sketchBuffer.AppendSketchSeries(ctx, source, series) +} + +// Series returns every scalar sample currently retained in the metric lookback +// ring. It is primarily intended for tests and diagnostics; production egress +// should use ForwardRange. +func (r *Retention) Series() metrics.Series { + buffer := r.getBuffer(false) + if buffer == nil { + return nil + } + return buffer.Series() +} + +// SketchSeries returns every sketch point currently retained in the metric +// lookback ring. It is primarily intended for tests and diagnostics; production +// egress should use ForwardRange. +func (r *Retention) SketchSeries() metrics.SketchSeriesList { + if r == nil { + return nil + } + sketchBuffer := r.getSketchBuffer(false) + if sketchBuffer == nil { + return nil + } + return sketchBuffer.SketchSeriesBetween(time.Time{}, time.Time{}) +} + +// PointsBetween returns retained points for a metric/source in the inclusive +// [from, to] window. It is the local query surface used by monitors. +func (r *Retention) PointsBetween(source ringbuffer.Source, metricName string, from, to time.Time) []ringbuffer.Point { + buffer := r.getBuffer(false) + if buffer == nil { + return nil + } + return buffer.PointsBetween(source, metricName, from, to) +} + +// PointsBetweenSources returns retained points for a metric from any of the +// provided sources in the inclusive [from, to] window. It lets a monitor treat +// normal DogStatsD bucketed points and no-aggregation final points as one +// DogStatsD lookback view while retaining source provenance in the ring. +func (r *Retention) PointsBetweenSources(sources []ringbuffer.Source, metricName string, from, to time.Time) []ringbuffer.Point { + buffer := r.getBuffer(false) + if buffer == nil { + return nil + } + return buffer.PointsBetweenSources(sources, metricName, from, to) +} + +// ProjectedSketchPointsBetweenSources projects retained sketch points for a +// metric from any of the provided sources into scalar points in the inclusive +// [from, to] window. The projection is intentionally caller-provided so monitor +// semantics stay independent from retention and egress serialization. +func (r *Retention) ProjectedSketchPointsBetweenSources(sources []ringbuffer.Source, metricName string, from, to time.Time, projection SketchScalarProjection) []ringbuffer.Point { + if r == nil || projection == nil { + return nil + } + sketchBuffer := r.getSketchBuffer(false) + if sketchBuffer == nil { + return nil + } + return sketchBuffer.PointsBetweenSources(sources, metricName, from, to, projection.Project) +} + +// Stats returns a point-in-time summary of the scalar retention ring. +func (r *Retention) Stats() ringbuffer.Stats { + buffer := r.getBuffer(false) + if buffer == nil { + return ringbuffer.Stats{} + } + return buffer.Stats() +} + +// SketchStats returns a point-in-time summary of the sketch retention ring. +func (r *Retention) SketchStats() ringbuffer.Stats { + if r == nil { + return ringbuffer.Stats{} + } + sketchBuffer := r.getSketchBuffer(false) + if sketchBuffer == nil { + return ringbuffer.Stats{} + } + return sketchBuffer.Stats() +} + +// ForwardAll sends every sample currently retained in the metric lookback ring +// buffer through the provided serializer as iterable series/sketch payloads. It +// returns the number of series and sketch series sent. Forwarding is +// non-destructive. +func (r *Retention) ForwardAll(metricSerializer serializer.MetricSerializer) (int, error) { + return r.ForwardRange(metricSerializer, time.Time{}, time.Time{}) +} + +// ForwardRange sends samples whose original timestamps fall in the half-open +// [from, to) window through the provided serializer as iterable series and +// sketch payloads. A zero from or to leaves that side of the window unbounded. +// It returns the number of series and sketch series sent. Forwarding is +// non-destructive. Retrying callers that need to avoid duplicates after partial +// serializer failures should use ForwardSeriesRange and ForwardSketchRange so +// each pipeline can be marked forwarded independently. +func (r *Retention) ForwardRange(metricSerializer serializer.MetricSerializer, from, to time.Time) (int, error) { + seriesCount, err := r.ForwardSeriesRange(metricSerializer, from, to) + if err != nil { + return 0, err + } + sketchCount, err := r.ForwardSketchRange(metricSerializer, from, to) + if err != nil { + return 0, err + } + return seriesCount + sketchCount, nil +} + +// ForwardSeriesRange sends series samples whose original timestamps fall in the +// half-open [from, to) window. A zero from or to leaves that side of the window +// unbounded. Forwarding is non-destructive. +func (r *Retention) ForwardSeriesRange(metricSerializer serializer.MetricSerializer, from, to time.Time) (int, error) { + inclusiveFrom, inclusiveTo, ok, err := r.forwardRangeBounds(metricSerializer, from, to) + if err != nil || !ok { + return 0, err + } + + seriesCount := 0 + var seriesSource metrics.SerieSource + if buffer := r.getBuffer(false); buffer != nil { + seriesSource = buffer.SerieSourceBetween(inclusiveFrom, inclusiveTo) + seriesCount = int(seriesSource.Count()) + } + if seriesCount == 0 { + return 0, nil + } + if err := metricSerializer.SendIterableSeries(seriesSource); err != nil { + return 0, err + } + return seriesCount, nil +} + +// ForwardSketchRange sends sketch samples whose original timestamps fall in the +// half-open [from, to) window. A zero from or to leaves that side of the window +// unbounded. Forwarding is non-destructive. +func (r *Retention) ForwardSketchRange(metricSerializer serializer.MetricSerializer, from, to time.Time) (int, error) { + inclusiveFrom, inclusiveTo, ok, err := r.forwardRangeBounds(metricSerializer, from, to) + if err != nil || !ok { + return 0, err + } + + sketchCount := 0 + var sketchSource metrics.SketchesSource + if sketchBuffer := r.getSketchBuffer(false); sketchBuffer != nil { + sketchSource = sketchBuffer.SketchSourceBetween(inclusiveFrom, inclusiveTo) + sketchCount = int(sketchSource.Count()) + } + if sketchCount == 0 { + return 0, nil + } + if err := metricSerializer.SendSketch(sketchSource); err != nil { + return 0, err + } + return sketchCount, nil +} + +func (r *Retention) forwardRangeBounds(metricSerializer serializer.MetricSerializer, from, to time.Time) (time.Time, time.Time, bool, error) { + if r == nil { + return time.Time{}, time.Time{}, false, errors.New("metric lookback is disabled") + } + if metricSerializer == nil { + return time.Time{}, time.Time{}, false, errors.New("serializer is not available") + } + if !to.IsZero() && !from.IsZero() && !from.Before(to) { + return time.Time{}, time.Time{}, false, nil + } + + inclusiveFrom, inclusiveTo, ok := halfOpenRangeToInclusiveMicroRange(from, to) + if !ok { + return time.Time{}, time.Time{}, false, nil + } + return inclusiveFrom, inclusiveTo, true, nil +} + +// halfOpenRangeToInclusiveMicroRange converts the public [from, to) range to +// the inclusive microsecond bounds used by retention records. +func halfOpenRangeToInclusiveMicroRange(from, to time.Time) (time.Time, time.Time, bool) { + inclusiveFrom := inclusiveMicroLowerBound(from) + inclusiveTo := exclusiveMicroUpperBound(to) + if !inclusiveFrom.IsZero() && !inclusiveTo.IsZero() && inclusiveFrom.After(inclusiveTo) { + return time.Time{}, time.Time{}, false + } + return inclusiveFrom, inclusiveTo, true +} + +func inclusiveMicroLowerBound(from time.Time) time.Time { + if from.IsZero() { + return time.Time{} + } + truncated := time.UnixMicro(from.UnixMicro()) + if from.Equal(truncated) || from.Before(truncated) { + return truncated + } + return truncated.Add(time.Microsecond) +} + +func exclusiveMicroUpperBound(to time.Time) time.Time { + if to.IsZero() { + return time.Time{} + } + truncated := time.UnixMicro(to.UnixMicro()) + if to.Equal(truncated) || to.Before(truncated) { + return truncated.Add(-time.Microsecond) + } + return truncated +} diff --git a/pkg/metriclookback/retention_test.go b/pkg/metriclookback/retention_test.go new file mode 100644 index 000000000000..16cf9f589c85 --- /dev/null +++ b/pkg/metriclookback/retention_test.go @@ -0,0 +1,205 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package metriclookback + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/pkg/metriclookback/ringbuffer" + "github.com/DataDog/datadog-agent/pkg/metrics" + serializermocks "github.com/DataDog/datadog-agent/pkg/serializer/mocks" + "github.com/DataDog/datadog-agent/pkg/tagset" + "github.com/DataDog/datadog-agent/pkg/util/quantile" +) + +func TestRetentionAppendSamplesAndForwardRange(t *testing.T) { + retention := NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + err := retention.AppendSamples(context.Background(), ringbuffer.Source{Kind: ringbuffer.SourceCheckShadow, ID: "check:1"}, []metrics.MetricSample{ + {Name: "metric.1", Value: 1, Mtype: metrics.GaugeType, Timestamp: 10}, + {Name: "metric.2", Value: 2, Mtype: metrics.GaugeType, Timestamp: 20}, + }) + require.NoError(t, err) + + serializer := serializermocks.NewMetricSerializer(t) + serializer.On("SendIterableSeries", mock.Anything).Run(func(args mock.Arguments) { + source := args.Get(0).(metrics.SerieSource) + require.Equal(t, uint64(1), source.Count()) + require.True(t, source.MoveNext()) + require.Equal(t, "metric.2", source.Current().Name) + require.False(t, source.MoveNext()) + }).Return(nil).Once() + + count, err := retention.ForwardRange(serializer, time.Unix(15, 0), time.Unix(25, 0)) + require.NoError(t, err) + require.Equal(t, 1, count) +} + +func TestRetentionForwardRangeIsHalfOpen(t *testing.T) { + retention := NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + err := retention.AppendSamples(context.Background(), ringbuffer.Source{Kind: ringbuffer.SourceCheckShadow, ID: "check:1"}, []metrics.MetricSample{ + {Name: "metric.1", Value: 1, Mtype: metrics.GaugeType, Timestamp: 10}, + {Name: "metric.2", Value: 2, Mtype: metrics.GaugeType, Timestamp: 20}, + }) + require.NoError(t, err) + + serializer := serializermocks.NewMetricSerializer(t) + serializer.On("SendIterableSeries", mock.Anything).Run(func(args mock.Arguments) { + source := args.Get(0).(metrics.SerieSource) + require.Equal(t, uint64(1), source.Count()) + require.True(t, source.MoveNext()) + require.Equal(t, "metric.1", source.Current().Name) + }).Return(nil).Once() + + count, err := retention.ForwardRange(serializer, time.Unix(10, 0), time.Unix(20, 0)) + require.NoError(t, err) + require.Equal(t, 1, count) +} + +func TestRetentionForwardRangeIncludesFinalEligibleMicrosecond(t *testing.T) { + retention := NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + err := retention.AppendSamples(context.Background(), ringbuffer.Source{Kind: ringbuffer.SourceCheckShadow, ID: "check:1"}, []metrics.MetricSample{ + {Name: "metric.edge", Value: 1, Mtype: metrics.GaugeType, Timestamp: 20}, + {Name: "metric.after", Value: 2, Mtype: metrics.GaugeType, Timestamp: 21}, + }) + require.NoError(t, err) + + serializer := serializermocks.NewMetricSerializer(t) + serializer.On("SendIterableSeries", mock.Anything).Run(func(args mock.Arguments) { + source := args.Get(0).(metrics.SerieSource) + require.Equal(t, uint64(1), source.Count()) + require.True(t, source.MoveNext()) + require.Equal(t, "metric.edge", source.Current().Name) + require.False(t, source.MoveNext()) + }).Return(nil).Once() + + count, err := retention.ForwardRange(serializer, time.Unix(19, 0), time.Unix(20, 500)) + require.NoError(t, err) + require.Equal(t, 1, count) +} + +func TestRetentionForwardRangeExcludesMicrosecondBeforeSubMicrosecondLowerBound(t *testing.T) { + retention := NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + err := retention.AppendSamples(context.Background(), ringbuffer.Source{Kind: ringbuffer.SourceCheckShadow, ID: "check:1"}, []metrics.MetricSample{ + {Name: "metric.before", Value: 1, Mtype: metrics.GaugeType, Timestamp: 20}, + {Name: "metric.after", Value: 2, Mtype: metrics.GaugeType, Timestamp: 20.001}, + }) + require.NoError(t, err) + + serializer := serializermocks.NewMetricSerializer(t) + serializer.On("SendIterableSeries", mock.Anything).Run(func(args mock.Arguments) { + source := args.Get(0).(metrics.SerieSource) + require.Equal(t, uint64(1), source.Count()) + require.True(t, source.MoveNext()) + require.Equal(t, "metric.after", source.Current().Name) + require.False(t, source.MoveNext()) + }).Return(nil).Once() + + count, err := retention.ForwardRange(serializer, time.Unix(20, 500), time.Unix(21, 0)) + require.NoError(t, err) + require.Equal(t, 1, count) +} + +func TestRetentionAppendSketchSeriesAndForwardRange(t *testing.T) { + retention := NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + err := retention.AppendSketchSeries(context.Background(), ringbuffer.Source{Kind: ringbuffer.SourceDogStatsDBucketed}, &metrics.SketchSeries{ + DistributionMetadata: metrics.DistributionMetadata{ + Name: "dist.metric", + Tags: tagset.CompositeTagsFromSlice([]string{"env:test"}), + Host: "host", + NoIndex: true, + Source: metrics.MetricSourceDogstatsd, + }, + Points: []metrics.SketchPoint{ + {Ts: 10, Sketch: testSketchData(1, 3)}, + {Ts: 20, Sketch: testSketchData(2, 4)}, + }, + }) + require.NoError(t, err) + + serializer := serializermocks.NewMetricSerializer(t) + serializer.On("SendSketch", mock.Anything).Run(func(args mock.Arguments) { + source := args.Get(0).(metrics.SketchesSource) + require.Equal(t, uint64(1), source.Count()) + require.True(t, source.WaitForValue()) + require.True(t, source.MoveNext()) + series, ok := source.Current().(*metrics.SketchSeries) + require.True(t, ok) + require.Equal(t, "dist.metric", series.Name) + require.Equal(t, "host", series.Host) + require.Equal(t, []string{"env:test"}, series.Tags.UnsafeToReadOnlySliceString()) + require.True(t, series.NoIndex) + require.Equal(t, metrics.MetricSourceDogstatsd, series.Source) + require.Len(t, series.Points, 1) + require.Equal(t, int64(20), series.Points[0].Ts) + cnt, min, max, sum, avg := series.Points[0].Sketch.BasicStats() + require.Equal(t, int64(2), cnt) + require.Equal(t, float64(2), min) + require.Equal(t, float64(4), max) + require.Equal(t, float64(6), sum) + require.Equal(t, float64(3), avg) + require.False(t, source.MoveNext()) + }).Return(nil).Once() + + count, err := retention.ForwardRange(serializer, time.Unix(15, 0), time.Unix(25, 0)) + require.NoError(t, err) + require.Equal(t, 1, count) +} + +func TestRetentionForwardRangeSketchOnlyDoesNotRequireScalarRing(t *testing.T) { + retention := NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + require.Nil(t, retention.buffer) + + err := retention.AppendSketchSeries(context.Background(), ringbuffer.Source{Kind: ringbuffer.SourceDogStatsDBucketed}, &metrics.SketchSeries{ + DistributionMetadata: metrics.DistributionMetadata{Name: "dist.metric"}, + Points: []metrics.SketchPoint{ + {Ts: 20, Sketch: testSketchData(2, 4)}, + }, + }) + require.NoError(t, err) + require.Nil(t, retention.buffer) + require.Equal(t, 1, retention.SketchStats().Records) + + serializer := serializermocks.NewMetricSerializer(t) + serializer.On("SendSketch", mock.Anything).Return(nil).Once() + + count, err := retention.ForwardRange(serializer, time.Unix(15, 0), time.Unix(25, 0)) + require.NoError(t, err) + require.Equal(t, 1, count) +} + +func TestRetentionProjectsSketchPointsWithPlaceholderAverage(t *testing.T) { + retention := NewRetention(ringbuffer.Options{Capacity: 8, ShardCount: 1}) + err := retention.AppendSketchSeries(context.Background(), ringbuffer.Source{Kind: ringbuffer.SourceDogStatsDBucketed}, &metrics.SketchSeries{ + DistributionMetadata: metrics.DistributionMetadata{Name: "dist.metric"}, + Points: []metrics.SketchPoint{ + {Ts: 10, Sketch: testSketchData(1, 5)}, + {Ts: 20, Sketch: testSketchData(2, 6)}, + }, + }) + require.NoError(t, err) + + points := retention.ProjectedSketchPointsBetweenSources( + []ringbuffer.Source{{Kind: ringbuffer.SourceDogStatsDBucketed}}, + "dist.metric", + time.Unix(15, 0), + time.Unix(25, 0), + PlaceholderAverageSketchProjection{}, + ) + require.Equal(t, []ringbuffer.Point{{Ts: time.Unix(20, 0), Value: 4}}, points) +} + +func testSketchData(values ...float64) *quantile.Sketch { + var agent quantile.Agent + for _, value := range values { + agent.Insert(value, 1) + } + return agent.Finish() +} diff --git a/pkg/metriclookback/ringbuffer/BUILD.bazel b/pkg/metriclookback/ringbuffer/BUILD.bazel new file mode 100644 index 000000000000..65b941f89e49 --- /dev/null +++ b/pkg/metriclookback/ringbuffer/BUILD.bazel @@ -0,0 +1,33 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "ringbuffer", + srcs = [ + "buffer.go", + "sketch_buffer.go", + ], + importpath = "github.com/DataDog/datadog-agent/pkg/metriclookback/ringbuffer", + visibility = ["//visibility:public"], + deps = [ + "//comp/core/telemetry/impl", + "//pkg/metrics", + "//pkg/tagset", + "//pkg/util/quantile", + ], +) + +go_test( + name = "ringbuffer_test", + srcs = [ + "buffer_test.go", + "sketch_buffer_test.go", + ], + embed = [":ringbuffer"], + gotags = ["test"], + deps = [ + "//pkg/metrics", + "//pkg/tagset", + "//pkg/util/quantile", + "@com_github_stretchr_testify//require", + ], +) diff --git a/pkg/metriclookback/ringbuffer/buffer.go b/pkg/metriclookback/ringbuffer/buffer.go new file mode 100644 index 000000000000..560a124c11f9 --- /dev/null +++ b/pkg/metriclookback/ringbuffer/buffer.go @@ -0,0 +1,885 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package ringbuffer stores recent metric lookback observations in bounded +// in-memory rings. Scalar points and finalized sketch points use separate record +// layouts so the common scalar path stays compact while sharing the same source +// and context conventions. +package ringbuffer + +import ( + "context" + "hash/fnv" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + telemetryimpl "github.com/DataDog/datadog-agent/comp/core/telemetry/impl" + "github.com/DataDog/datadog-agent/pkg/metrics" + "github.com/DataDog/datadog-agent/pkg/tagset" +) + +var ( + tlmRingAppendedSamples = telemetryimpl.GetCompatComponent().NewCounter("metric_lookback", "ring_appended_samples", []string{"source"}, "Count of samples appended to the metric lookback ring") + tlmRingOverwrittenSamples = telemetryimpl.GetCompatComponent().NewSimpleCounter("metric_lookback", "ring_overwritten_samples", "Count of samples overwritten in the metric lookback ring") + tlmRingRecords = telemetryimpl.GetCompatComponent().NewGauge("metric_lookback", "ring_records", nil, "Current number of records retained in the metric lookback ring") + tlmRingCapacity = telemetryimpl.GetCompatComponent().NewGauge("metric_lookback", "ring_capacity", nil, "Total record capacity of the metric lookback ring") + tlmRingOldestTimestamp = telemetryimpl.GetCompatComponent().NewGauge("metric_lookback", "ring_oldest_timestamp", nil, "Oldest retained metric lookback timestamp, in Unix seconds") + tlmRingNewestTimestamp = telemetryimpl.GetCompatComponent().NewGauge("metric_lookback", "ring_newest_timestamp", nil, "Newest retained metric lookback timestamp, in Unix seconds") +) + +const ( + // DefaultCapacity is the total number of samples retained when Options.Capacity + // is not set. + DefaultCapacity = 262_144 + + // DefaultShardCount is the number of independent rings used when + // Options.ShardCount is not set. + DefaultShardCount = 16 + + // checkSeriesSourceTypeName matches the normal check sampler's v1 source type + // name for check-originated metric series. The numeric Source field still + // carries the more specific source/origin metadata for v2 payloads. + checkSeriesSourceTypeName = "System" + + // dogStatsDNoAggregationInterval matches the fixed aggregator bucket interval + // used by the no-aggregation serializer for rate-like DogStatsD samples. + // Keep this package independent from pkg/aggregator so the retention substrate + // can remain outside the hot aggregation package. + dogStatsDNoAggregationInterval = 10 +) + +// SourceKind identifies the retention producer. It is provenance for lookback +// accounting and source-specific egress semantics; it is not emitted as an extra +// backend metric tag. +type SourceKind string + +const ( + // SourceDogStatsDBucketed identifies selected normal DogStatsD samples after + // they have been materialized into short-width lookback buckets. + SourceDogStatsDBucketed SourceKind = "dogstatsd_bucketed" + // SourceDogStatsDNoAggregation identifies selected timestamped DogStatsD + // samples that bypass local aggregation. + SourceDogStatsDNoAggregation SourceKind = "dogstatsd_no_aggregation" + // SourceCheck identifies samples emitted by a normal check path. + SourceCheck SourceKind = "check" + // SourceCheckShadow identifies samples emitted by a future lookback shadow + // check path. + SourceCheckShadow SourceKind = "check_shadow" +) + +// Source describes the producer of retained points. +type Source struct { + Kind SourceKind + // ID is producer-specific: a check ID for check/shadow-check samples, or empty + // for the first-stage selected DogStatsD no-aggregation stream. + ID string +} + +type sampleFlags uint8 + +const ( + flagFlushFirstValue sampleFlags = 1 << iota +) + +// Options controls Buffer construction. +type Options struct { + // Capacity is the total number of sample slots allocated across all shards. + // Retention is shard-local: samples are assigned to shards by metric context, + // so a hot shard can overwrite its oldest samples while other shards still + // have unused slots. When zero or negative, DefaultCapacity is used. + Capacity int + + // ShardCount is the number of independent rings. When zero or negative, + // DefaultShardCount is used. Values larger than Capacity are clamped so every + // shard has at least one slot. + ShardCount int + + // Now supplies timestamps for samples without an explicit MetricSample + // timestamp. When nil, time.Now is used. + Now func() time.Time +} + +// Stats describes the current state of a Buffer. +type Stats struct { + Capacity int + ShardCount int + Records int + ActiveContexts int + TotalContextsCreated uint64 + AppendedSamples uint64 + OverwrittenSamples uint64 + OldestUnixMicro int64 + NewestUnixMicro int64 +} + +// Point is a retained scalar point returned to local readers such as monitors. +type Point struct { + Ts time.Time + Value float64 + Tags []string +} + +// Buffer is a bounded in-memory ring for recent scalar metric samples. +type Buffer struct { + now func() time.Time + + contexts *contextStore + shards []shard + + nextSequence atomic.Uint64 + appendedSamples atomic.Uint64 + overwrittenSamples atomic.Uint64 + records atomic.Int64 + newestUnixMicro atomic.Int64 +} + +// New creates a bounded in-memory ring buffer. +func New(options Options) *Buffer { + capacity := options.Capacity + if capacity <= 0 { + capacity = DefaultCapacity + } + + shardCount := options.ShardCount + if shardCount <= 0 { + shardCount = DefaultShardCount + } + if shardCount > capacity { + shardCount = capacity + } + + now := options.Now + if now == nil { + now = time.Now + } + + b := &Buffer{ + now: now, + contexts: newContextStore(shardCount), + shards: make([]shard, shardCount), + } + + baseCapacity := capacity / shardCount + extraSlots := capacity % shardCount + for i := range b.shards { + shardCapacity := baseCapacity + if i < extraSlots { + shardCapacity++ + } + b.shards[i].records = make([]record, shardCapacity) + } + tlmRingCapacity.Set(float64(capacity)) + tlmRingRecords.Set(0) + + return b +} + +// AppendSamples stores scalar metric samples from the given source. If the +// context is cancelled before or during the call, AppendSamples returns the +// context error after retaining any samples already appended. +func (b *Buffer) AppendSamples(ctx context.Context, source Source, samples []metrics.MetricSample) error { + if b == nil || len(samples) == 0 { + return nil + } + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } + + nowUnixMicro := int64(0) + for i := range samples { + if err := ctx.Err(); err != nil { + return err + } + + sample := samples[i] + timestampUnixMicro := sampleTimestampUnixMicro(sample.Timestamp) + if timestampUnixMicro == 0 { + if nowUnixMicro == 0 { + nowUnixMicro = b.now().UnixMicro() + } + timestampUnixMicro = nowUnixMicro + } + + contextID, shardID := b.contexts.retain(source, sample) + sequence := b.nextSequence.Add(1) + rec := record{ + contextID: contextID, + timestampUnixMicro: timestampUnixMicro, + sequence: sequence, + value: recordValue(source, sample), + sampleRate: sample.SampleRate, + flags: flagsForSample(sample), + } + + b.appendRecord(source, shardID, rec) + } + + return nil +} + +// AppendSerie stores every point in serie as already-normalized retained points. +// This is used by DogStatsD no-aggregation after the worker has constructed the +// same series it sends to the normal serializer, preserving tag enrichment and +// type/value mapping. +func (b *Buffer) AppendSerie(ctx context.Context, source Source, serie *metrics.Serie) error { + if b == nil || serie == nil || len(serie.Points) == 0 { + return nil + } + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } + + for i := range serie.Points { + if err := ctx.Err(); err != nil { + return err + } + contextID, shardID := b.contexts.retainSerie(source, serie) + point := serie.Points[i] + timestampUnixMicro := sampleTimestampUnixMicro(point.Ts) + if timestampUnixMicro == 0 { + timestampUnixMicro = b.now().UnixMicro() + } + rec := record{ + contextID: contextID, + timestampUnixMicro: timestampUnixMicro, + sequence: b.nextSequence.Add(1), + value: point.Value, + } + b.appendRecord(source, shardID, rec) + } + + return nil +} + +func (b *Buffer) appendRecord(source Source, shardID int, rec record) { + overwrittenContextID, overwritten := b.shards[shardID].append(rec) + b.appendedSamples.Add(1) + tlmRingAppendedSamples.Inc(string(source.Kind)) + if overwritten { + b.overwrittenSamples.Add(1) + tlmRingOverwrittenSamples.Inc() + b.contexts.release(overwrittenContextID) + } else { + b.records.Add(1) + } + b.setNewestUnixMicro(rec.timestampUnixMicro) + tlmRingRecords.Set(float64(b.records.Load())) + tlmRingNewestTimestamp.Set(float64(b.newestUnixMicro.Load()) / 1e6) +} + +func (b *Buffer) setNewestUnixMicro(timestampUnixMicro int64) { + for { + current := b.newestUnixMicro.Load() + if timestampUnixMicro <= current { + return + } + if b.newestUnixMicro.CompareAndSwap(current, timestampUnixMicro) { + return + } + } +} + +// Series reconstructs every retained sample into a metrics.Series, ordered by +// the global append sequence. +func (b *Buffer) Series() metrics.Series { + return b.SeriesBetween(time.Time{}, time.Time{}) +} + +// SeriesBetween reconstructs retained samples whose original timestamps fall in +// the inclusive [from, to] window, ordered by the global append sequence. A zero +// from or to leaves that side of the window unbounded. Each retained sample +// becomes a single-point serie carrying the canonicalized metric context (name, +// host, tags, source, unit, no-index) recorded at append time. This is a +// non-destructive snapshot: the buffer keeps its samples so forwarding can be +// retried or repeated. +func (b *Buffer) SeriesBetween(from, to time.Time) metrics.Series { + if b == nil || invalidRange(from, to) { + return nil + } + + records := b.snapshotSortedRecords() + if len(records) == 0 { + return nil + } + contexts := b.contexts.snapshot() + + series := make(metrics.Series, 0, len(records)) + for i := range records { + rec := &records[i] + if !recordInRange(rec, from, to) { + continue + } + ctx, found := contexts[rec.contextID] + if !found { + // The context was evicted between snapshots; skip the orphan record. + continue + } + serie := &metrics.Serie{ + Name: ctx.name, + Points: []metrics.Point{{ + Ts: float64(rec.timestampUnixMicro) / 1e6, + Value: rec.value, + }}, + Tags: tagset.CompositeTagsFromSlice(append([]string(nil), ctx.tags...)), + Host: ctx.host, + MType: ctx.apiType, + Interval: ctx.interval, + } + if ctx.source.Kind != SourceDogStatsDNoAggregation { + serie.NoIndex = ctx.noIndex + serie.Source = ctx.metricSource + serie.SourceTypeName = ctx.sourceTypeName + serie.Unit = ctx.unit + } + series = append(series, serie) + } + return series +} + +// PointsBetween returns retained points for a metric/source in the inclusive +// [from, to] window, ordered by point timestamp. A zero from or to leaves that +// side of the window unbounded. The returned points are a stable snapshot. +func (b *Buffer) PointsBetween(source Source, metricName string, from, to time.Time) []Point { + return b.PointsBetweenSources([]Source{source}, metricName, from, to) +} + +// PointsBetweenSources returns retained points for a metric from any of the +// provided sources in the inclusive [from, to] window, ordered by point +// timestamp. A zero from or to leaves that side of the window unbounded. The +// returned points are a stable snapshot. A source with an empty ID matches every +// source with the same kind, which lets monitors read all shadow-check instances. +func (b *Buffer) PointsBetweenSources(sources []Source, metricName string, from, to time.Time) []Point { + if b == nil || len(sources) == 0 || metricName == "" || invalidRange(from, to) { + return nil + } + + records := b.snapshotSortedRecords() + if len(records) == 0 { + return nil + } + contexts := b.contexts.snapshot() + + points := make([]Point, 0) + for i := range records { + rec := &records[i] + if !recordInRange(rec, from, to) { + continue + } + ctx, found := contexts[rec.contextID] + if !found || ctx.name != metricName { + continue + } + if !sourceMatchesAny(sources, ctx.source) { + continue + } + points = append(points, Point{ + Ts: time.UnixMicro(rec.timestampUnixMicro), + Value: rec.value, + Tags: append([]string(nil), ctx.tags...), + }) + } + sort.Slice(points, func(i, j int) bool { + if points[i].Ts.Equal(points[j].Ts) { + return points[i].Value < points[j].Value + } + return points[i].Ts.Before(points[j].Ts) + }) + return points +} + +// SerieSource returns a metrics.SerieSource over all current retained samples, +// suitable for passing directly to serializer.SendIterableSeries. The snapshot +// is taken eagerly when SerieSource is called. +func (b *Buffer) SerieSource() metrics.SerieSource { + return b.SerieSourceBetween(time.Time{}, time.Time{}) +} + +// SerieSourceBetween returns a metrics.SerieSource over retained samples whose +// original timestamps fall in the inclusive [from, to] window. A zero from or to +// leaves that side of the window unbounded. The snapshot is taken eagerly when +// SerieSourceBetween is called. +func (b *Buffer) SerieSourceBetween(from, to time.Time) metrics.SerieSource { + return &serieSliceSource{series: b.SeriesBetween(from, to), index: -1} +} + +func invalidRange(from, to time.Time) bool { + return !from.IsZero() && !to.IsZero() && from.After(to) +} + +func recordInRange(rec *record, from, to time.Time) bool { + if !from.IsZero() && rec.timestampUnixMicro < from.UnixMicro() { + return false + } + if !to.IsZero() && rec.timestampUnixMicro > to.UnixMicro() { + return false + } + return true +} + +// snapshotSortedRecords returns a copy of every retained record ordered by the +// global append sequence. +func (b *Buffer) snapshotSortedRecords() []record { + var out []record + for i := range b.shards { + out = b.shards[i].appendRecordsTo(out) + } + sort.Slice(out, func(i, j int) bool { + return out[i].sequence < out[j].sequence + }) + return out +} + +// serieSliceSource is a metrics.SerieSource backed by a fixed slice of series. +type serieSliceSource struct { + series metrics.Series + index int +} + +func (s *serieSliceSource) MoveNext() bool { + s.index++ + return s.index < len(s.series) +} + +func (s *serieSliceSource) Current() *metrics.Serie { + return s.series[s.index] +} + +func (s *serieSliceSource) Count() uint64 { + return uint64(len(s.series)) +} + +func sourceMatchesAny(queries []Source, actual Source) bool { + for _, query := range queries { + if query == actual || (query.ID == "" && query.Kind == actual.Kind) { + return true + } + } + return false +} + +func recordValue(source Source, sample metrics.MetricSample) float64 { + if source.Kind == SourceDogStatsDNoAggregation && isDogStatsDNoAggRate(sample.Mtype) { + return sample.Value / dogStatsDNoAggregationInterval + } + return sample.Value +} + +func apiMetricType(source Source, mtype metrics.MetricType) metrics.APIMetricType { + if source.Kind == SourceDogStatsDNoAggregation { + switch mtype { + case metrics.CounterType, metrics.RateType: + return metrics.APIRateType + case metrics.CountType, metrics.CountWithTimestampType: + return metrics.APICountType + default: + return metrics.APIGaugeType + } + } + + switch mtype { + case metrics.CountType, metrics.CounterType, metrics.CountWithTimestampType: + return metrics.APICountType + default: + // GaugeType, GaugeWithTimestampType, RateType, MonotonicCountType, + // HistogramType, HistorateType, and unsupported/non-scalar types. + return metrics.APIGaugeType + } +} + +func apiInterval(source Source, _ metrics.MetricType) int64 { + if source.Kind == SourceDogStatsDNoAggregation { + return dogStatsDNoAggregationInterval + } + return 0 +} + +func isDogStatsDNoAggRate(mtype metrics.MetricType) bool { + return mtype == metrics.CounterType || mtype == metrics.RateType +} + +func sourceTypeName(source Source) string { + switch source.Kind { + case SourceCheck, SourceCheckShadow: + return checkSeriesSourceTypeName + default: + return "" + } +} + +// Stats returns a point-in-time summary of the buffer. +func (b *Buffer) Stats() Stats { + if b == nil { + return Stats{} + } + + records := 0 + capacity := 0 + oldestUnixMicro := int64(0) + newestUnixMicro := int64(0) + for i := range b.shards { + recordCount, shardCapacity, shardOldest, shardNewest := b.shards[i].stats() + records += recordCount + capacity += shardCapacity + if shardOldest > 0 && (oldestUnixMicro == 0 || shardOldest < oldestUnixMicro) { + oldestUnixMicro = shardOldest + } + if shardNewest > newestUnixMicro { + newestUnixMicro = shardNewest + } + } + + activeContexts, totalContextsCreated := b.contexts.stats() + tlmRingCapacity.Set(float64(capacity)) + tlmRingRecords.Set(float64(records)) + tlmRingOldestTimestamp.Set(float64(oldestUnixMicro) / 1e6) + tlmRingNewestTimestamp.Set(float64(newestUnixMicro) / 1e6) + return Stats{ + Capacity: capacity, + ShardCount: len(b.shards), + Records: records, + ActiveContexts: activeContexts, + TotalContextsCreated: totalContextsCreated, + AppendedSamples: b.appendedSamples.Load(), + OverwrittenSamples: b.overwrittenSamples.Load(), + OldestUnixMicro: oldestUnixMicro, + NewestUnixMicro: newestUnixMicro, + } +} + +type record struct { + contextID uint64 + timestampUnixMicro int64 + sequence uint64 + value float64 + sampleRate float64 + flags sampleFlags +} + +type shard struct { + mu sync.Mutex + records []record + start int + length int + next int +} + +func (s *shard) append(rec record) (uint64, bool) { + s.mu.Lock() + defer s.mu.Unlock() + + if len(s.records) == 0 { + return 0, false + } + + overwrittenContextID := uint64(0) + overwritten := s.length == len(s.records) + if overwritten { + overwrittenContextID = s.records[s.next].contextID + } else { + s.length++ + } + + s.records[s.next] = rec + s.next = (s.next + 1) % len(s.records) + if overwritten { + s.start = s.next + } + + return overwrittenContextID, overwritten +} + +func (s *shard) stats() (count, capacity int, oldestUnixMicro, newestUnixMicro int64) { + s.mu.Lock() + defer s.mu.Unlock() + for i := 0; i < s.length; i++ { + idx := (s.start + i) % len(s.records) + timestamp := s.records[idx].timestampUnixMicro + if timestamp > 0 && (oldestUnixMicro == 0 || timestamp < oldestUnixMicro) { + oldestUnixMicro = timestamp + } + if timestamp > newestUnixMicro { + newestUnixMicro = timestamp + } + } + return s.length, len(s.records), oldestUnixMicro, newestUnixMicro +} + +// appendRecordsTo appends the shard's live records, oldest first, to out. +func (s *shard) appendRecordsTo(out []record) []record { + s.mu.Lock() + defer s.mu.Unlock() + for i := 0; i < s.length; i++ { + idx := (s.start + i) % len(s.records) + out = append(out, s.records[idx]) + } + return out +} + +type metricContext struct { + id uint64 + source Source + name string + host string + tags []string + mtype metrics.MetricType + apiType metrics.APIMetricType + interval int64 + noIndex bool + metricSource metrics.MetricSource + sourceTypeName string + unit string + shardID int +} + +type contextEntry struct { + key string + ctx metricContext + refs int +} + +type contextStore struct { + mu sync.Mutex + shardCount int + nextID uint64 + byKey map[string]*contextEntry + byID map[uint64]*contextEntry + totalContextsCreated uint64 +} + +func newContextStore(shardCount int) *contextStore { + return &contextStore{ + shardCount: shardCount, + nextID: 1, + byKey: make(map[string]*contextEntry), + byID: make(map[uint64]*contextEntry), + } +} + +func (s *contextStore) retain(source Source, sample metrics.MetricSample) (uint64, int) { + tags := canonicalTags(sample.Tags) + key := buildContextKey(source, sample, tags) + ctx := metricContext{ + source: source, + name: sample.Name, + host: sample.Host, + tags: tags, + mtype: sample.Mtype, + apiType: apiMetricType(source, sample.Mtype), + interval: apiInterval(source, sample.Mtype), + noIndex: sample.NoIndex, + metricSource: sample.Source, + sourceTypeName: sourceTypeName(source), + unit: sample.Unit, + } + return s.retainContext(key, ctx) +} + +func (s *contextStore) retainSerie(source Source, serie *metrics.Serie) (uint64, int) { + rawTags := append([]string(nil), serie.Tags.UnsafeToReadOnlySliceString()...) + keyTags := canonicalTags(rawTags) + key := buildSerieContextKey(source, serie, keyTags) + ctx := metricContext{ + source: source, + name: serie.Name, + host: serie.Host, + tags: rawTags, + apiType: serie.MType, + interval: serie.Interval, + noIndex: serie.NoIndex, + metricSource: serie.Source, + sourceTypeName: serie.SourceTypeName, + unit: serie.Unit, + } + return s.retainContext(key, ctx) +} + +func (s *contextStore) retainContext(key string, ctx metricContext) (uint64, int) { + s.mu.Lock() + defer s.mu.Unlock() + + if entry, found := s.byKey[key]; found { + entry.refs++ + return entry.ctx.id, entry.ctx.shardID + } + + contextID := s.nextID + s.nextID++ + shardID := shardIndex(hashString(key), s.shardCount) + ctx.id = contextID + ctx.shardID = shardID + entry := &contextEntry{ + key: key, + ctx: ctx, + refs: 1, + } + s.byKey[key] = entry + s.byID[contextID] = entry + s.totalContextsCreated++ + return contextID, shardID +} + +func (s *contextStore) release(contextID uint64) { + if contextID == 0 { + return + } + + s.mu.Lock() + defer s.mu.Unlock() + + entry, found := s.byID[contextID] + if !found { + return + } + entry.refs-- + if entry.refs > 0 { + return + } + delete(s.byID, contextID) + delete(s.byKey, entry.key) +} + +func (s *contextStore) stats() (int, uint64) { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.byID), s.totalContextsCreated +} + +// snapshot returns a copy of the active metric contexts keyed by context ID. +// Tag slices are cloned so callers can retain them safely. +func (s *contextStore) snapshot() map[uint64]metricContext { + s.mu.Lock() + defer s.mu.Unlock() + + out := make(map[uint64]metricContext, len(s.byID)) + for id, entry := range s.byID { + ctx := entry.ctx + ctx.tags = append([]string(nil), entry.ctx.tags...) + out[id] = ctx + } + return out +} + +func sampleTimestampUnixMicro(timestamp float64) int64 { + if timestamp <= 0 { + return 0 + } + return int64(timestamp * 1e6) +} + +func flagsForSample(sample metrics.MetricSample) sampleFlags { + var flags sampleFlags + if sample.FlushFirstValue { + flags |= flagFlushFirstValue + } + return flags +} + +func canonicalTags(tags []string) []string { + if len(tags) == 0 { + return nil + } + + out := append([]string(nil), tags...) + sort.Strings(out) + return dedupeSortedStrings(out) +} + +func dedupeSortedStrings(values []string) []string { + if len(values) < 2 { + return values + } + + writeIdx := 0 + for readIdx := 1; readIdx < len(values); readIdx++ { + if values[readIdx] == values[writeIdx] { + continue + } + writeIdx++ + values[writeIdx] = values[readIdx] + } + return values[:writeIdx+1] +} + +func buildContextKey(source Source, sample metrics.MetricSample, tags []string) string { + var builder strings.Builder + appendContextKeyPrefix(&builder, source, sample.Name, sample.Host) + appendIntField(&builder, int64(sample.Mtype)) + appendBoolField(&builder, sample.NoIndex) + appendIntField(&builder, int64(sample.Source)) + appendStringField(&builder, sample.Unit) + appendTagsField(&builder, tags) + return builder.String() +} + +func buildSerieContextKey(source Source, serie *metrics.Serie, tags []string) string { + var builder strings.Builder + appendContextKeyPrefix(&builder, source, serie.Name, serie.Host) + appendIntField(&builder, int64(serie.MType)) + appendIntField(&builder, serie.Interval) + appendBoolField(&builder, serie.NoIndex) + appendIntField(&builder, int64(serie.Source)) + appendStringField(&builder, serie.SourceTypeName) + appendStringField(&builder, serie.Unit) + appendTagsField(&builder, tags) + return builder.String() +} + +func appendContextKeyPrefix(builder *strings.Builder, source Source, name, host string) { + appendStringField(builder, string(source.Kind)) + appendStringField(builder, source.ID) + appendStringField(builder, name) + appendStringField(builder, host) +} + +func appendTagsField(builder *strings.Builder, tags []string) { + appendIntField(builder, int64(len(tags))) + for _, tag := range tags { + appendStringField(builder, tag) + } +} + +func appendStringField(builder *strings.Builder, value string) { + builder.WriteString(strconv.Itoa(len(value))) + builder.WriteByte(':') + builder.WriteString(value) + builder.WriteByte('|') +} + +func appendIntField(builder *strings.Builder, value int64) { + builder.WriteString(strconv.FormatInt(value, 10)) + builder.WriteByte('|') +} + +func appendBoolField(builder *strings.Builder, value bool) { + if value { + builder.WriteByte('1') + } else { + builder.WriteByte('0') + } + builder.WriteByte('|') +} + +func hashString(value string) uint64 { + h := fnv.New64a() + _, _ = h.Write([]byte(value)) + return h.Sum64() +} + +func shardIndex(hash uint64, shardCount int) int { + if shardCount <= 1 { + return 0 + } + return int(hash % uint64(shardCount)) +} diff --git a/pkg/metriclookback/ringbuffer/buffer_test.go b/pkg/metriclookback/ringbuffer/buffer_test.go new file mode 100644 index 000000000000..5ab1258188ba --- /dev/null +++ b/pkg/metriclookback/ringbuffer/buffer_test.go @@ -0,0 +1,285 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package ringbuffer + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/pkg/metrics" + "github.com/DataDog/datadog-agent/pkg/tagset" +) + +func TestAppendSamplesStoresCheckAndDogStatsDNoAggInOneRing(t *testing.T) { + buf := New(Options{Capacity: 8, ShardCount: 1}) + + err := buf.AppendSamples(context.Background(), Source{Kind: SourceCheckShadow, ID: "check:1"}, []metrics.MetricSample{{ + Name: "check.metric", + Value: 1, + Mtype: metrics.GaugeType, + Timestamp: 10, + Host: "host-a", + Tags: []string{"b:2", "a:1", "a:1"}, + NoIndex: true, + Unit: "unit", + }}) + require.NoError(t, err) + + err = buf.AppendSamples(context.Background(), Source{Kind: SourceDogStatsDNoAggregation}, []metrics.MetricSample{{ + Name: "dog.metric", + Value: 2, + Mtype: metrics.GaugeType, + Timestamp: 11, + Host: "host-b", + Tags: []string{"z:9"}, + }}) + require.NoError(t, err) + + series := buf.Series() + require.Len(t, series, 2) + + require.Equal(t, "check.metric", series[0].Name) + require.Equal(t, float64(10), series[0].Points[0].Ts) + require.Equal(t, float64(1), series[0].Points[0].Value) + require.Equal(t, []string{"a:1", "b:2"}, series[0].Tags.UnsafeToReadOnlySliceString()) + require.Equal(t, "host-a", series[0].Host) + require.Equal(t, metrics.APIGaugeType, series[0].MType) + require.True(t, series[0].NoIndex) + require.Equal(t, "System", series[0].SourceTypeName) + require.Equal(t, "unit", series[0].Unit) + + require.Equal(t, "dog.metric", series[1].Name) + require.Equal(t, float64(11), series[1].Points[0].Ts) + require.Equal(t, float64(2), series[1].Points[0].Value) + require.Equal(t, []string{"z:9"}, series[1].Tags.UnsafeToReadOnlySliceString()) + require.Equal(t, "host-b", series[1].Host) + require.Equal(t, metrics.APIGaugeType, series[1].MType) + require.Equal(t, int64(dogStatsDNoAggregationInterval), series[1].Interval) + require.Empty(t, series[1].SourceTypeName) + require.Empty(t, series[1].Unit) + require.Equal(t, metrics.MetricSource(0), series[1].Source) + + stats := buf.Stats() + require.Equal(t, 8, stats.Capacity) + require.Equal(t, 1, stats.ShardCount) + require.Equal(t, 2, stats.Records) + require.Equal(t, 2, stats.ActiveContexts) + require.Equal(t, uint64(2), stats.AppendedSamples) + require.Equal(t, uint64(0), stats.OverwrittenSamples) + require.Equal(t, time.Unix(10, 0).UnixMicro(), stats.OldestUnixMicro) + require.Equal(t, time.Unix(11, 0).UnixMicro(), stats.NewestUnixMicro) +} + +func TestDogStatsDNoAggRateLikeSamplesUseNoAggAPISemantics(t *testing.T) { + buf := New(Options{Capacity: 4, ShardCount: 1}) + + err := buf.AppendSamples(context.Background(), Source{Kind: SourceDogStatsDNoAggregation}, []metrics.MetricSample{{ + Name: "dog.counter", + Value: 20, + Mtype: metrics.CounterType, + Timestamp: 10, + }}) + require.NoError(t, err) + + series := buf.Series() + require.Len(t, series, 1) + require.Equal(t, metrics.APIRateType, series[0].MType) + require.Equal(t, int64(dogStatsDNoAggregationInterval), series[0].Interval) + require.Equal(t, float64(2), series[0].Points[0].Value) +} + +func TestAppendSeriePreservesNormalizedSeriesFields(t *testing.T) { + buf := New(Options{Capacity: 4, ShardCount: 1}) + serie := &metrics.Serie{ + Name: "dog.metric", + Points: []metrics.Point{{Ts: 10, Value: 2}}, + Tags: tagset.CompositeTagsFromSlice([]string{"client:tag", "origin:tag"}), + Host: "host", + MType: metrics.APIRateType, + Interval: 10, + } + + require.NoError(t, buf.AppendSerie(context.Background(), Source{Kind: SourceDogStatsDNoAggregation}, serie)) + + series := buf.Series() + require.Len(t, series, 1) + require.Equal(t, serie.Name, series[0].Name) + require.Equal(t, serie.Points, series[0].Points) + require.Equal(t, serie.Tags.UnsafeToReadOnlySliceString(), series[0].Tags.UnsafeToReadOnlySliceString()) + require.Equal(t, serie.Host, series[0].Host) + require.Equal(t, serie.MType, series[0].MType) + require.Equal(t, serie.Interval, series[0].Interval) +} + +func TestAppendSerieRetainsContextPerPoint(t *testing.T) { + buf := New(Options{Capacity: 2, ShardCount: 1}) + serie := &metrics.Serie{ + Name: "dog.metric", + Points: []metrics.Point{{Ts: 10, Value: 2}, {Ts: 11, Value: 3}}, + Tags: tagset.CompositeTagsFromSlice([]string{"client:tag"}), + Host: "host", + MType: metrics.APIGaugeType, + Interval: 10, + } + + require.NoError(t, buf.AppendSerie(context.Background(), Source{Kind: SourceDogStatsDNoAggregation}, serie)) + require.NoError(t, buf.AppendSamples(context.Background(), Source{Kind: SourceCheckShadow, ID: "check:1"}, []metrics.MetricSample{{ + Name: "check.metric", + Value: 4, + Mtype: metrics.GaugeType, + Timestamp: 12, + }})) + + series := buf.Series() + require.Len(t, series, 2) + require.Equal(t, "dog.metric", series[0].Name) + require.Equal(t, []metrics.Point{{Ts: 11, Value: 3}}, series[0].Points) + require.Equal(t, "check.metric", series[1].Name) + stats := buf.Stats() + require.Equal(t, 2, stats.Records) + require.Equal(t, 2, stats.ActiveContexts) +} + +func TestSeriesBetweenFiltersByTimestamp(t *testing.T) { + buf := New(Options{Capacity: 8, ShardCount: 1}) + err := buf.AppendSamples(context.Background(), Source{Kind: SourceCheckShadow, ID: "check:1"}, []metrics.MetricSample{ + {Name: "metric.1", Value: 1, Mtype: metrics.GaugeType, Timestamp: 10}, + {Name: "metric.2", Value: 2, Mtype: metrics.GaugeType, Timestamp: 20}, + {Name: "metric.3", Value: 3, Mtype: metrics.GaugeType, Timestamp: 30}, + }) + require.NoError(t, err) + + series := buf.SeriesBetween(time.Unix(15, 0), time.Unix(25, 0)) + require.Len(t, series, 1) + require.Equal(t, "metric.2", series[0].Name) +} + +func TestPointsBetweenFiltersBySourceNameAndTimestamp(t *testing.T) { + buf := New(Options{Capacity: 8, ShardCount: 1}) + dogstatsdSource := Source{Kind: SourceDogStatsDNoAggregation} + checkSource := Source{Kind: SourceCheckShadow, ID: "check:1"} + require.NoError(t, buf.AppendSamples(context.Background(), dogstatsdSource, []metrics.MetricSample{ + {Name: "metric", Value: 1, Mtype: metrics.GaugeType, Timestamp: 10}, + {Name: "metric", Value: 2, Mtype: metrics.GaugeType, Timestamp: 20}, + {Name: "other", Value: 3, Mtype: metrics.GaugeType, Timestamp: 20}, + })) + require.NoError(t, buf.AppendSamples(context.Background(), checkSource, []metrics.MetricSample{ + {Name: "metric", Value: 4, Mtype: metrics.GaugeType, Timestamp: 20}, + })) + + points := buf.PointsBetween(dogstatsdSource, "metric", time.Unix(15, 0), time.Unix(25, 0)) + require.Equal(t, []Point{{Ts: time.Unix(20, 0), Value: 2}}, points) +} + +func TestPointsBetweenReturnsRetainedPointTags(t *testing.T) { + buf := New(Options{Capacity: 8, ShardCount: 1}) + source := Source{Kind: SourceCheckShadow, ID: "check:1"} + require.NoError(t, buf.AppendSamples(context.Background(), source, []metrics.MetricSample{ + {Name: "metric", Value: 1, Mtype: metrics.GaugeType, Timestamp: 10, Tags: []string{"env:prod", "az:a"}}, + })) + + points := buf.PointsBetween(source, "metric", time.Time{}, time.Time{}) + require.Equal(t, []Point{{Ts: time.Unix(10, 0), Value: 1, Tags: []string{"az:a", "env:prod"}}}, points) +} + +func TestPointsBetweenSourcesReadsMultipleDogStatsDSources(t *testing.T) { + buf := New(Options{Capacity: 8, ShardCount: 1}) + bucketedSource := Source{Kind: SourceDogStatsDBucketed} + noAggSource := Source{Kind: SourceDogStatsDNoAggregation} + checkSource := Source{Kind: SourceCheckShadow, ID: "check:1"} + require.NoError(t, buf.AppendSamples(context.Background(), bucketedSource, []metrics.MetricSample{ + {Name: "metric", Value: 1, Mtype: metrics.GaugeType, Timestamp: 10}, + })) + require.NoError(t, buf.AppendSamples(context.Background(), noAggSource, []metrics.MetricSample{ + {Name: "metric", Value: 2, Mtype: metrics.GaugeType, Timestamp: 11}, + })) + require.NoError(t, buf.AppendSamples(context.Background(), checkSource, []metrics.MetricSample{ + {Name: "metric", Value: 3, Mtype: metrics.GaugeType, Timestamp: 12}, + })) + + points := buf.PointsBetweenSources([]Source{bucketedSource, noAggSource}, "metric", time.Time{}, time.Time{}) + require.Equal(t, []Point{ + {Ts: time.Unix(10, 0), Value: 1}, + {Ts: time.Unix(11, 0), Value: 2}, + }, points) +} + +func TestPointsBetweenSourcesEmptyIDMatchesAllSourcesWithKind(t *testing.T) { + buf := New(Options{Capacity: 8, ShardCount: 1}) + require.NoError(t, buf.AppendSamples(context.Background(), Source{Kind: SourceCheckShadow, ID: "check:1"}, []metrics.MetricSample{ + {Name: "metric", Value: 1, Mtype: metrics.GaugeType, Timestamp: 10}, + })) + require.NoError(t, buf.AppendSamples(context.Background(), Source{Kind: SourceCheckShadow, ID: "check:2"}, []metrics.MetricSample{ + {Name: "metric", Value: 2, Mtype: metrics.GaugeType, Timestamp: 11}, + })) + require.NoError(t, buf.AppendSamples(context.Background(), Source{Kind: SourceDogStatsDNoAggregation}, []metrics.MetricSample{ + {Name: "metric", Value: 3, Mtype: metrics.GaugeType, Timestamp: 12}, + })) + + points := buf.PointsBetweenSources([]Source{{Kind: SourceCheckShadow}}, "metric", time.Time{}, time.Time{}) + require.Equal(t, []Point{ + {Ts: time.Unix(10, 0), Value: 1}, + {Ts: time.Unix(11, 0), Value: 2}, + }, points) +} + +func TestAppendSamplesUsesNowForUntimestampedSamples(t *testing.T) { + now := time.Unix(123, 456000) + buf := New(Options{Capacity: 4, ShardCount: 1, Now: func() time.Time { return now }}) + + err := buf.AppendSamples(context.Background(), Source{Kind: SourceCheckShadow, ID: "check:1"}, []metrics.MetricSample{{ + Name: "metric", + Value: 1, + Mtype: metrics.GaugeType, + }}) + require.NoError(t, err) + + series := buf.Series() + require.Len(t, series, 1) + require.Equal(t, float64(now.UnixMicro())/1e6, series[0].Points[0].Ts) +} + +func TestOverwriteReleasesContexts(t *testing.T) { + buf := New(Options{Capacity: 2, ShardCount: 1}) + for i := 0; i < 3; i++ { + err := buf.AppendSamples(context.Background(), Source{Kind: SourceCheckShadow, ID: "check:1"}, []metrics.MetricSample{{ + Name: "metric", + Value: float64(i), + Mtype: metrics.GaugeType, + Timestamp: float64(i), + Tags: []string{string(rune('a' + i))}, + }}) + require.NoError(t, err) + } + + stats := buf.Stats() + require.Equal(t, 2, stats.Records) + require.Equal(t, 2, stats.ActiveContexts) + require.Equal(t, uint64(3), stats.AppendedSamples) + require.Equal(t, uint64(1), stats.OverwrittenSamples) + + series := buf.Series() + require.Len(t, series, 2) + require.Equal(t, float64(1), series[0].Points[0].Value) + require.Equal(t, float64(2), series[1].Points[0].Value) +} + +func TestAppendSamplesReturnsContextCancellation(t *testing.T) { + buf := New(Options{Capacity: 4, ShardCount: 1}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := buf.AppendSamples(ctx, Source{Kind: SourceCheckShadow, ID: "check:1"}, []metrics.MetricSample{{ + Name: "metric", + Value: 1, + Mtype: metrics.GaugeType, + }}) + require.ErrorIs(t, err, context.Canceled) + require.Equal(t, 0, buf.Stats().Records) +} diff --git a/pkg/metriclookback/ringbuffer/sketch_buffer.go b/pkg/metriclookback/ringbuffer/sketch_buffer.go new file mode 100644 index 000000000000..ddfb899cfce2 --- /dev/null +++ b/pkg/metriclookback/ringbuffer/sketch_buffer.go @@ -0,0 +1,431 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package ringbuffer + +import ( + "context" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + telemetryimpl "github.com/DataDog/datadog-agent/comp/core/telemetry/impl" + "github.com/DataDog/datadog-agent/pkg/metrics" + "github.com/DataDog/datadog-agent/pkg/tagset" + "github.com/DataDog/datadog-agent/pkg/util/quantile" +) + +var ( + tlmRingAppendedSketches = telemetryimpl.GetCompatComponent().NewCounter("metric_lookback", "ring_appended_sketches", []string{"source"}, "Count of sketch points appended to the metric lookback ring") + tlmRingOverwrittenSketches = telemetryimpl.GetCompatComponent().NewSimpleCounter("metric_lookback", "ring_overwritten_sketches", "Count of sketch points overwritten in the metric lookback ring") + tlmRingSketchRecords = telemetryimpl.GetCompatComponent().NewGauge("metric_lookback", "ring_sketch_records", nil, "Current number of sketch records retained in the metric lookback ring") + tlmRingSketchCapacity = telemetryimpl.GetCompatComponent().NewGauge("metric_lookback", "ring_sketch_capacity", nil, "Total sketch record capacity of the metric lookback ring") +) + +// SketchBuffer is a bounded in-memory ring for recent finalized sketch points. +// It is intentionally separate from Buffer so scalar records keep their compact +// representation when distribution lookback is not used. +type SketchBuffer struct { + now func() time.Time + + contexts *contextStore + shards []sketchShard + + nextSequence atomic.Uint64 + appendedSketches atomic.Uint64 + overwrittenSketches atomic.Uint64 + records atomic.Int64 +} + +// NewSketchBuffer creates a bounded in-memory sketch buffer. +func NewSketchBuffer(options Options) *SketchBuffer { + capacity := options.Capacity + if capacity <= 0 { + capacity = DefaultCapacity + } + + shardCount := options.ShardCount + if shardCount <= 0 { + shardCount = DefaultShardCount + } + if shardCount > capacity { + shardCount = capacity + } + + now := options.Now + if now == nil { + now = time.Now + } + + b := &SketchBuffer{ + now: now, + contexts: newContextStore(shardCount), + shards: make([]sketchShard, shardCount), + } + + baseCapacity := capacity / shardCount + extraSlots := capacity % shardCount + for i := range b.shards { + shardCapacity := baseCapacity + if i < extraSlots { + shardCapacity++ + } + b.shards[i].records = make([]sketchRecord, shardCapacity) + } + tlmRingSketchCapacity.Set(float64(capacity)) + tlmRingSketchRecords.Set(0) + + return b +} + +// AppendSketchSeries stores every sketch point in series as an already-finalized +// retained sketch point. +func (b *SketchBuffer) AppendSketchSeries(ctx context.Context, source Source, series *metrics.SketchSeries) error { + if b == nil || series == nil || len(series.Points) == 0 { + return nil + } + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } + + hasSketch := false + for i := range series.Points { + if series.Points[i].Sketch != nil { + hasSketch = true + break + } + } + if !hasSketch { + return nil + } + + for i := range series.Points { + if err := ctx.Err(); err != nil { + return err + } + point := series.Points[i] + if point.Sketch == nil { + continue + } + contextID, shardID := b.contexts.retainSketchSeries(source, series) + timestampUnixMicro := sketchTimestampUnixMicro(point.Ts) + if timestampUnixMicro == 0 { + timestampUnixMicro = b.now().UnixMicro() + } + rec := sketchRecord{ + contextID: contextID, + timestampUnixMicro: timestampUnixMicro, + sequence: b.nextSequence.Add(1), + sketch: retainSketchData(point.Sketch), + } + b.appendRecord(source, shardID, rec) + } + + return nil +} + +func (b *SketchBuffer) appendRecord(source Source, shardID int, rec sketchRecord) { + overwrittenContextID, overwritten := b.shards[shardID].append(rec) + b.appendedSketches.Add(1) + tlmRingAppendedSketches.Inc(string(source.Kind)) + if overwritten { + b.overwrittenSketches.Add(1) + tlmRingOverwrittenSketches.Inc() + b.contexts.release(overwrittenContextID) + } else { + b.records.Add(1) + } + tlmRingSketchRecords.Set(float64(b.records.Load())) +} + +// SketchSeriesBetween reconstructs retained sketch points whose original +// timestamps fall in the inclusive [from, to] window, ordered by append sequence. +func (b *SketchBuffer) SketchSeriesBetween(from, to time.Time) metrics.SketchSeriesList { + if b == nil || invalidRange(from, to) { + return nil + } + + records := b.snapshotSortedRecords() + if len(records) == 0 { + return nil + } + contexts := b.contexts.snapshot() + + series := make(metrics.SketchSeriesList, 0, len(records)) + for i := range records { + rec := &records[i] + if !sketchRecordInRange(rec, from, to) { + continue + } + ctx, found := contexts[rec.contextID] + if !found || rec.sketch == nil { + continue + } + series = append(series, &metrics.SketchSeries{ + DistributionMetadata: metrics.DistributionMetadata{ + Name: ctx.name, + Tags: tagset.CompositeTagsFromSlice(append([]string(nil), ctx.tags...)), + Host: ctx.host, + Interval: ctx.interval, + NoIndex: ctx.noIndex, + Source: ctx.metricSource, + }, + Points: []metrics.SketchPoint{{ + Ts: rec.timestampUnixMicro / 1e6, + Sketch: cloneSketchData(rec.sketch), + }}, + }) + } + return series +} + +// PointsBetweenSources projects retained sketch points for a metric from any of +// the provided sources into scalar points in the inclusive [from, to] window. +// A source with an empty ID matches every source with the same kind, which lets +// monitors read all shadow-check instances. The projection is deliberately +// supplied by the caller so the sketch retention layer does not own monitor +// semantics. +func (b *SketchBuffer) PointsBetweenSources(sources []Source, metricName string, from, to time.Time, project func(metrics.SketchPoint) (float64, bool)) []Point { + if b == nil || len(sources) == 0 || metricName == "" || project == nil || invalidRange(from, to) { + return nil + } + + records := b.snapshotSortedRecords() + if len(records) == 0 { + return nil + } + contexts := b.contexts.snapshot() + + points := make([]Point, 0) + for i := range records { + rec := &records[i] + if !sketchRecordInRange(rec, from, to) { + continue + } + ctx, found := contexts[rec.contextID] + if !found || ctx.name != metricName { + continue + } + if !sourceMatchesAny(sources, ctx.source) { + continue + } + value, ok := project(metrics.SketchPoint{Ts: rec.timestampUnixMicro / 1e6, Sketch: rec.sketch}) + if !ok { + continue + } + points = append(points, Point{ + Ts: time.UnixMicro(rec.timestampUnixMicro), + Value: value, + Tags: append([]string(nil), ctx.tags...), + }) + } + sort.Slice(points, func(i, j int) bool { + if points[i].Ts.Equal(points[j].Ts) { + return points[i].Value < points[j].Value + } + return points[i].Ts.Before(points[j].Ts) + }) + return points +} + +// SketchSourceBetween returns a metrics.SketchesSource over retained sketches. +func (b *SketchBuffer) SketchSourceBetween(from, to time.Time) metrics.SketchesSource { + return &sketchSliceSource{series: b.SketchSeriesBetween(from, to), index: -1} +} + +// Stats returns a point-in-time summary of the sketch buffer. +func (b *SketchBuffer) Stats() Stats { + if b == nil { + return Stats{} + } + + records := 0 + capacity := 0 + oldestUnixMicro := int64(0) + newestUnixMicro := int64(0) + for i := range b.shards { + recordCount, shardCapacity, shardOldest, shardNewest := b.shards[i].stats() + records += recordCount + capacity += shardCapacity + if shardOldest > 0 && (oldestUnixMicro == 0 || shardOldest < oldestUnixMicro) { + oldestUnixMicro = shardOldest + } + if shardNewest > newestUnixMicro { + newestUnixMicro = shardNewest + } + } + + activeContexts, totalContextsCreated := b.contexts.stats() + tlmRingSketchRecords.Set(float64(records)) + return Stats{ + Capacity: capacity, + ShardCount: len(b.shards), + Records: records, + ActiveContexts: activeContexts, + TotalContextsCreated: totalContextsCreated, + AppendedSamples: b.appendedSketches.Load(), + OverwrittenSamples: b.overwrittenSketches.Load(), + OldestUnixMicro: oldestUnixMicro, + NewestUnixMicro: newestUnixMicro, + } +} + +func (b *SketchBuffer) snapshotSortedRecords() []sketchRecord { + var out []sketchRecord + for i := range b.shards { + out = b.shards[i].appendRecordsTo(out) + } + sort.Slice(out, func(i, j int) bool { + return out[i].sequence < out[j].sequence + }) + return out +} + +type sketchSliceSource struct { + series metrics.SketchSeriesList + index int +} + +func (s *sketchSliceSource) MoveNext() bool { + s.index++ + return s.index < len(s.series) +} + +func (s *sketchSliceSource) Current() metrics.Distribution { + return s.series[s.index] +} + +func (s *sketchSliceSource) Count() uint64 { + return uint64(len(s.series)) +} + +func (s *sketchSliceSource) WaitForValue() bool { + return s.index+1 < len(s.series) +} + +type sketchRecord struct { + contextID uint64 + timestampUnixMicro int64 + sequence uint64 + sketch *quantile.Sketch +} + +type sketchShard struct { + mu sync.Mutex + records []sketchRecord + start int + length int + next int +} + +func (s *sketchShard) append(rec sketchRecord) (uint64, bool) { + s.mu.Lock() + defer s.mu.Unlock() + + if len(s.records) == 0 { + return 0, false + } + + overwrittenContextID := uint64(0) + overwritten := s.length == len(s.records) + if overwritten { + overwrittenContextID = s.records[s.next].contextID + } else { + s.length++ + } + + s.records[s.next] = rec + s.next = (s.next + 1) % len(s.records) + if overwritten { + s.start = s.next + } + + return overwrittenContextID, overwritten +} + +func (s *sketchShard) appendRecordsTo(out []sketchRecord) []sketchRecord { + s.mu.Lock() + defer s.mu.Unlock() + for i := 0; i < s.length; i++ { + idx := (s.start + i) % len(s.records) + out = append(out, s.records[idx]) + } + return out +} + +func (s *sketchShard) stats() (count, capacity int, oldestUnixMicro, newestUnixMicro int64) { + s.mu.Lock() + defer s.mu.Unlock() + for i := 0; i < s.length; i++ { + idx := (s.start + i) % len(s.records) + timestamp := s.records[idx].timestampUnixMicro + if timestamp > 0 && (oldestUnixMicro == 0 || timestamp < oldestUnixMicro) { + oldestUnixMicro = timestamp + } + if timestamp > newestUnixMicro { + newestUnixMicro = timestamp + } + } + return s.length, len(s.records), oldestUnixMicro, newestUnixMicro +} + +func (s *contextStore) retainSketchSeries(source Source, series *metrics.SketchSeries) (uint64, int) { + rawTags := append([]string(nil), series.Tags.UnsafeToReadOnlySliceString()...) + keyTags := canonicalTags(rawTags) + key := buildSketchContextKey(source, series, keyTags) + ctx := metricContext{ + source: source, + name: series.Name, + host: series.Host, + tags: rawTags, + interval: series.Interval, + noIndex: series.NoIndex, + metricSource: series.Source, + } + return s.retainContext(key, ctx) +} + +func buildSketchContextKey(source Source, series *metrics.SketchSeries, tags []string) string { + var builder strings.Builder + appendContextKeyPrefix(&builder, source, series.Name, series.Host) + appendIntField(&builder, series.Interval) + appendBoolField(&builder, series.NoIndex) + appendIntField(&builder, int64(series.Source)) + appendTagsField(&builder, tags) + return builder.String() +} + +func sketchTimestampUnixMicro(timestamp int64) int64 { + if timestamp <= 0 { + return 0 + } + return timestamp * 1e6 +} + +func sketchRecordInRange(rec *sketchRecord, from, to time.Time) bool { + if !from.IsZero() && rec.timestampUnixMicro < from.UnixMicro() { + return false + } + if !to.IsZero() && rec.timestampUnixMicro > to.UnixMicro() { + return false + } + return true +} + +func retainSketchData(data *quantile.Sketch) *quantile.Sketch { + if data == nil { + return nil + } + return data.Copy() +} + +func cloneSketchData(data *quantile.Sketch) *quantile.Sketch { + return retainSketchData(data) +} diff --git a/pkg/metriclookback/ringbuffer/sketch_buffer_test.go b/pkg/metriclookback/ringbuffer/sketch_buffer_test.go new file mode 100644 index 000000000000..aab1eab82c62 --- /dev/null +++ b/pkg/metriclookback/ringbuffer/sketch_buffer_test.go @@ -0,0 +1,47 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package ringbuffer + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/pkg/metrics" + "github.com/DataDog/datadog-agent/pkg/util/quantile" +) + +func TestSketchBufferRetainsContextPerPoint(t *testing.T) { + buffer := NewSketchBuffer(Options{Capacity: 2, ShardCount: 1}) + source := Source{Kind: SourceDogStatsDBucketed} + + require.NoError(t, buffer.AppendSketchSeries(context.Background(), source, &metrics.SketchSeries{ + DistributionMetadata: metrics.DistributionMetadata{Name: "dist.a"}, + Points: []metrics.SketchPoint{ + {Ts: 10, Sketch: testRetainedSketchData(1)}, + {Ts: 11, Sketch: testRetainedSketchData(2)}, + }, + })) + require.NoError(t, buffer.AppendSketchSeries(context.Background(), source, &metrics.SketchSeries{ + DistributionMetadata: metrics.DistributionMetadata{Name: "dist.b"}, + Points: []metrics.SketchPoint{{Ts: 12, Sketch: testRetainedSketchData(3)}}, + })) + + series := buffer.SketchSeriesBetween(time.Time{}, time.Time{}) + require.Len(t, series, 2) + require.Equal(t, "dist.a", series[0].Name) + require.Equal(t, int64(11), series[0].Points[0].Ts) + require.Equal(t, "dist.b", series[1].Name) + require.Equal(t, int64(12), series[1].Points[0].Ts) +} + +func testRetainedSketchData(value float64) *quantile.Sketch { + agent := quantile.Agent{} + agent.Insert(value, 1) + return agent.Finish() +} diff --git a/pkg/metriclookback/sketch_projection.go b/pkg/metriclookback/sketch_projection.go new file mode 100644 index 000000000000..f1c128a9e3f6 --- /dev/null +++ b/pkg/metriclookback/sketch_projection.go @@ -0,0 +1,38 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package metriclookback + +import ( + "math" + + "github.com/DataDog/datadog-agent/pkg/metrics" +) + +// SketchScalarProjection projects a retained sketch point into a scalar point +// for monitor evaluation. The projection is intentionally separate from sketch +// retention and egress serialization: lookback still retains and forwards the +// full sketch, while this interface only bridges the current scalar monitor. +type SketchScalarProjection interface { + Project(metrics.SketchPoint) (float64, bool) +} + +// PlaceholderAverageSketchProjection is a deliberately isolated placeholder for +// distribution-monitor evaluation. It projects each retained sketch to its +// average so DogStatsD distribution metrics can exercise the scalar monitor path +// while the product semantics for sketch-aware monitoring are deferred. +type PlaceholderAverageSketchProjection struct{} + +// Project returns the sketch average from BasicStats. +func (PlaceholderAverageSketchProjection) Project(point metrics.SketchPoint) (float64, bool) { + if point.Sketch == nil { + return 0, false + } + cnt, _, _, _, avg := point.Sketch.BasicStats() + if cnt <= 0 || math.IsInf(avg, 0) || math.IsNaN(avg) { + return 0, false + } + return avg, true +}