diff --git a/buildSrc/src/main/groovy/uforwarder.java-common-conventions.gradle b/buildSrc/src/main/groovy/uforwarder.java-common-conventions.gradle index 93023a7..5b9bfcd 100644 --- a/buildSrc/src/main/groovy/uforwarder.java-common-conventions.gradle +++ b/buildSrc/src/main/groovy/uforwarder.java-common-conventions.gradle @@ -19,6 +19,7 @@ dependencies { implementation 'ch.qos.logback:logback-classic:1.2.3' implementation 'ch.qos.logback:logback-core:1.2.3' implementation 'com.101tec:zkclient:0.11' + implementation 'com.datadoghq:java-dogstatsd-client:4.3.0' implementation 'com.facebook.infer.annotation:infer-annotation:0.17.0' implementation 'com.github.stefanbirkner:system-rules:1.19.0' implementation 'com.google.api:api-common:1.7.0' @@ -30,6 +31,8 @@ dependencies { implementation 'com.uber.concurrency-loadbalancer:concurrency-loadbalancer-core:0.1.5' implementation 'com.uber.m3:tally-core:0.13.0' implementation 'com.uber.m3:tally-m3:0.13.0' + implementation 'com.uber.m3:tally-statsd:0.13.0' + implementation 'commons-codec:commons-codec:1.15' implementation 'io.grpc:grpc-core:1.49.2' implementation 'io.grpc:grpc-netty-shaded:1.49.2' implementation 'io.grpc:grpc-protobuf:1.49.2' diff --git a/instrumentation/build.gradle b/instrumentation/build.gradle index c427d72..5aff1bf 100644 --- a/instrumentation/build.gradle +++ b/instrumentation/build.gradle @@ -9,6 +9,7 @@ plugins { dependencies { implementation 'com.uber.m3:tally-core' implementation 'com.uber.m3:tally-m3' + implementation 'com.uber.m3:tally-statsd' implementation 'io.grpc:grpc-stub' implementation 'io.opentracing:opentracing-api' implementation 'net.logstash.logback:logstash-logback-encoder' diff --git a/uforwarder-core/build.gradle b/uforwarder-core/build.gradle index d09b785..82f11f6 100644 --- a/uforwarder-core/build.gradle +++ b/uforwarder-core/build.gradle @@ -8,15 +8,18 @@ plugins { dependencies { implementation 'com.101tec:zkclient' + implementation 'com.datadoghq:java-dogstatsd-client' implementation 'com.facebook.infer.annotation:infer-annotation' implementation 'com.google.api:api-common' implementation 'com.google.api.grpc:proto-google-common-protos' implementation 'com.google.guava:guava' implementation 'com.google.protobuf:protobuf-java' implementation 'com.google.protobuf:protobuf-java-util' + implementation 'commons-codec:commons-codec' implementation 'com.uber.concurrency-loadbalancer:concurrency-loadbalancer-core' implementation 'com.uber.m3:tally-core' implementation 'com.uber.m3:tally-m3' + implementation 'com.uber.m3:tally-statsd' implementation 'io.grpc:grpc-netty-shaded' implementation 'io.grpc:grpc-protobuf' implementation 'io.grpc:grpc-services' diff --git a/uforwarder-core/src/main/java/com/uber/data/kafka/datatransfer/common/MetricsConfiguration.java b/uforwarder-core/src/main/java/com/uber/data/kafka/datatransfer/common/MetricsConfiguration.java index 209ea95..f017a4b 100644 --- a/uforwarder-core/src/main/java/com/uber/data/kafka/datatransfer/common/MetricsConfiguration.java +++ b/uforwarder-core/src/main/java/com/uber/data/kafka/datatransfer/common/MetricsConfiguration.java @@ -1,7 +1,12 @@ package com.uber.data.kafka.datatransfer.common; +import com.timgroup.statsd.NonBlockingStatsDClientBuilder; +import com.timgroup.statsd.StatsDClient; import com.uber.m3.tally.RootScopeBuilder; import com.uber.m3.tally.Scope; +import com.uber.m3.tally.StatsReporter; +import com.uber.m3.tally.m3.M3Reporter; +import com.uber.m3.tally.statsd.StatsdReporter; import com.uber.m3.util.Duration; import com.uber.m3.util.ImmutableMap; import java.util.Objects; @@ -18,6 +23,23 @@ public class MetricsConfiguration { @Nullable static Scope INSTANCE; + private static final String ENV_UFORWARDER_REPORT_METRICS = "UFORWARDER_REPORT_METRICS"; + private static final String DOCKER_HOST_INTERNAL_ADDRESS = "host.docker.internal"; + private static final String METRICS_REPORTER_STATSD = "statsd"; + private static final String METRICS_REPORTER_M3 = "m3"; + + // The metrics reporter to use, currently only M3 and statsd is supported now + // see https://github.com/uber-java/tally for more details + private String metricsReporter = METRICS_REPORTER_M3; + + public String getMetricsReporter() { + return metricsReporter; + } + + public void setMetricsReporter(String metricsReporter) { + this.metricsReporter = metricsReporter; + } + @Bean @Singleton @ConditionalOnProperty( @@ -29,8 +51,23 @@ public class MetricsConfiguration { @ConditionalOnMissingBean public Scope rootScope(@Value("${tally.publish.interval.sec:5}") int tallyPublishIntervalSec) { if (INSTANCE == null) { + StatsReporter statsReporter = null; + // If UFORWARDER_REPORT_METRICS is set, we will set metric reporter + // according to the metricsReporter property. + String reportMetrics = System.getenv().get(ENV_UFORWARDER_REPORT_METRICS); + if (reportMetrics != null) { + if (metricsReporter.equals(METRICS_REPORTER_STATSD)) { + StatsDClient statsd = new NonBlockingStatsDClientBuilder() + .prefix(METRICS_REPORTER_STATSD) + .hostname(DOCKER_HOST_INTERNAL_ADDRESS) + .port(8125) + .build(); + statsReporter = new StatsdReporter(statsd); + } + } INSTANCE = new RootScopeBuilder() + .reporter(statsReporter) .tags(new ImmutableMap.Builder().build()) .reportEvery(Duration.ofSeconds(tallyPublishIntervalSec)); } diff --git a/uforwarder/build.gradle b/uforwarder/build.gradle index f3fb7ae..4914b04 100644 --- a/uforwarder/build.gradle +++ b/uforwarder/build.gradle @@ -17,6 +17,7 @@ dependencies { implementation 'com.uber.concurrency-loadbalancer:concurrency-loadbalancer-core' implementation 'com.uber.m3:tally-core' implementation 'com.uber.m3:tally-m3' + implementation 'com.uber.m3:tally-statsd' implementation 'io.grpc:grpc-core' implementation 'io.grpc:grpc-netty-shaded' implementation 'io.grpc:grpc-services' diff --git a/uforwarder/src/main/resources/application-uforwarder-controller.yaml b/uforwarder/src/main/resources/application-uforwarder-controller.yaml index 000b115..1e3e665 100644 --- a/uforwarder/src/main/resources/application-uforwarder-controller.yaml +++ b/uforwarder/src/main/resources/application-uforwarder-controller.yaml @@ -23,6 +23,8 @@ metrics: # we add host tags manually to only metrics that require it. # The majority of host level information should leverage the debug pages. includeHostTag: false + # Currently, the supported metrics reporter is m3 and statsd + metricsReporter: m3 # Enable async-profiler debug pages. jvm: diff --git a/uforwarder/src/main/resources/application-uforwarder-worker.yaml b/uforwarder/src/main/resources/application-uforwarder-worker.yaml index 94b03f4..8fd3fc3 100644 --- a/uforwarder/src/main/resources/application-uforwarder-worker.yaml +++ b/uforwarder/src/main/resources/application-uforwarder-worker.yaml @@ -21,6 +21,8 @@ metrics: # we add host tags manually to only metrics that require it. # The majority of host level information should leverage the debug pages. includeHostTag: false + # Currently, the supported metrics reporter is m3 and statsd + metricsReporter: m3 # Enable async-profiler debug pages. jvm: