Skip to content

[Metrics] add support for datadog metric reporting via statsd #23

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
suez1224 wants to merge 6 commits into
base: main
Choose a base branch
from
Open

[Metrics] add support for datadog metric reporting via statsd #23

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
26de35c
make uForwarder emit metrics via Statsd
suez1224 Apr 19, 2024
817a1e3
support both statsd and existing M3 reporting via config
suez1224 Apr 22, 2024
41b0671
keep m3 as default
suez1224 Apr 22, 2024
6e69799
use host.docker.internal for statsd reporting host instead
suez1224 Apr 22, 2024
b16970d
update metricsReporter comment
suez1224 Apr 22, 2024
782eb94
only report metrics if environment variable UFORWARDER_REPORT_METRICS…
suez1224 Apr 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions buildSrc/src/main/groovy/uforwarder.java-common-conventions.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ dependencies {
implementation 'ch.qos.logback:logback-classic:1.2.3'
implementation 'ch.qos.logback:logback-core:1.2.3'
implementation 'com.101tec:zkclient:0.11'
implementation 'com.datadoghq:java-dogstatsd-client:4.3.0'
implementation 'com.facebook.infer.annotation:infer-annotation:0.17.0'
implementation 'com.github.stefanbirkner:system-rules:1.19.0'
implementation 'com.google.api:api-common:1.7.0'
Expand All @@ -30,6 +31,8 @@ dependencies {
implementation 'com.uber.concurrency-loadbalancer:concurrency-loadbalancer-core:0.1.5'
implementation 'com.uber.m3:tally-core:0.13.0'
implementation 'com.uber.m3:tally-m3:0.13.0'
implementation 'com.uber.m3:tally-statsd:0.13.0'
implementation 'commons-codec:commons-codec:1.15'
implementation 'io.grpc:grpc-core:1.49.2'
implementation 'io.grpc:grpc-netty-shaded:1.49.2'
implementation 'io.grpc:grpc-protobuf:1.49.2'
Expand Down
1 change: 1 addition & 0 deletions instrumentation/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ plugins {
dependencies {
implementation 'com.uber.m3:tally-core'
implementation 'com.uber.m3:tally-m3'
implementation 'com.uber.m3:tally-statsd'
implementation 'io.grpc:grpc-stub'
implementation 'io.opentracing:opentracing-api'
implementation 'net.logstash.logback:logstash-logback-encoder'
Expand Down
3 changes: 3 additions & 0 deletions uforwarder-core/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,18 @@ plugins {

dependencies {
implementation 'com.101tec:zkclient'
implementation 'com.datadoghq:java-dogstatsd-client'
implementation 'com.facebook.infer.annotation:infer-annotation'
implementation 'com.google.api:api-common'
implementation 'com.google.api.grpc:proto-google-common-protos'
implementation 'com.google.guava:guava'
implementation 'com.google.protobuf:protobuf-java'
implementation 'com.google.protobuf:protobuf-java-util'
implementation 'commons-codec:commons-codec'
implementation 'com.uber.concurrency-loadbalancer:concurrency-loadbalancer-core'
implementation 'com.uber.m3:tally-core'
implementation 'com.uber.m3:tally-m3'
implementation 'com.uber.m3:tally-statsd'
implementation 'io.grpc:grpc-netty-shaded'
implementation 'io.grpc:grpc-protobuf'
implementation 'io.grpc:grpc-services'
Expand Down
37 changes: 37 additions & 0 deletions ...rder-core/src/main/java/com/uber/data/kafka/datatransfer/common/MetricsConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
package com.uber.data.kafka.datatransfer.common;

import com.timgroup.statsd.NonBlockingStatsDClientBuilder;
import com.timgroup.statsd.StatsDClient;
import com.uber.m3.tally.RootScopeBuilder;
import com.uber.m3.tally.Scope;
import com.uber.m3.tally.StatsReporter;
import com.uber.m3.tally.m3.M3Reporter;
import com.uber.m3.tally.statsd.StatsdReporter;
import com.uber.m3.util.Duration;
import com.uber.m3.util.ImmutableMap;
import java.util.Objects;
Expand All @@ -18,6 +23,23 @@
public class MetricsConfiguration {
@Nullable static Scope INSTANCE;

private static final String ENV_UFORWARDER_REPORT_METRICS = "UFORWARDER_REPORT_METRICS";
private static final String DOCKER_HOST_INTERNAL_ADDRESS = "host.docker.internal";
private static final String METRICS_REPORTER_STATSD = "statsd";
private static final String METRICS_REPORTER_M3 = "m3";

// The metrics reporter to use, currently only M3 and statsd is supported now
// see https://github.com/uber-java/tally for more details
private String metricsReporter = METRICS_REPORTER_M3;

public String getMetricsReporter() {
return metricsReporter;
}

public void setMetricsReporter(String metricsReporter) {
this.metricsReporter = metricsReporter;
}

@Bean
@Singleton
@ConditionalOnProperty(
Expand All @@ -29,8 +51,23 @@ public class MetricsConfiguration {
@ConditionalOnMissingBean
public Scope rootScope(@Value("${tally.publish.interval.sec:5}") int tallyPublishIntervalSec) {
if (INSTANCE == null) {
StatsReporter statsReporter = null;
// If UFORWARDER_REPORT_METRICS is set, we will set metric reporter
// according to the metricsReporter property.
String reportMetrics = System.getenv().get(ENV_UFORWARDER_REPORT_METRICS);
if (reportMetrics != null) {
if (metricsReporter.equals(METRICS_REPORTER_STATSD)) {
StatsDClient statsd = new NonBlockingStatsDClientBuilder()
.prefix(METRICS_REPORTER_STATSD)
.hostname(DOCKER_HOST_INTERNAL_ADDRESS)
.port(8125)
Copy link
Contributor

@mcruzdev mcruzdev Mar 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It should be configurable too, no?

.build();
statsReporter = new StatsdReporter(statsd);
}
}
INSTANCE =
new RootScopeBuilder()
.reporter(statsReporter)
.tags(new ImmutableMap.Builder<String, String>().build())
.reportEvery(Duration.ofSeconds(tallyPublishIntervalSec));
}
Expand Down
1 change: 1 addition & 0 deletions uforwarder/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ dependencies {
implementation 'com.uber.concurrency-loadbalancer:concurrency-loadbalancer-core'
implementation 'com.uber.m3:tally-core'
implementation 'com.uber.m3:tally-m3'
implementation 'com.uber.m3:tally-statsd'
implementation 'io.grpc:grpc-core'
implementation 'io.grpc:grpc-netty-shaded'
implementation 'io.grpc:grpc-services'
Expand Down
2 changes: 2 additions & 0 deletions uforwarder/src/main/resources/application-uforwarder-controller.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ metrics:
# we add host tags manually to only metrics that require it.
# The majority of host level information should leverage the debug pages.
includeHostTag: false
# Currently, the supported metrics reporter is m3 and statsd
metricsReporter: m3

# Enable async-profiler debug pages.
jvm:
Expand Down
2 changes: 2 additions & 0 deletions uforwarder/src/main/resources/application-uforwarder-worker.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ metrics:
# we add host tags manually to only metrics that require it.
# The majority of host level information should leverage the debug pages.
includeHostTag: false
# Currently, the supported metrics reporter is m3 and statsd
metricsReporter: m3

# Enable async-profiler debug pages.
jvm:
Expand Down