From 26de35c0252a01ff7a42ef8f9e98faca555f43e6 Mon Sep 17 00:00:00 2001 From: Shuyi Chen Date: Fri, 19 Apr 2024 15:34:53 -0700 Subject: [PATCH 1/6] make uForwarder emit metrics via Statsd --- .../groovy/uforwarder.java-common-conventions.gradle | 2 +- instrumentation/build.gradle | 2 +- uforwarder-core/build.gradle | 4 +++- .../datatransfer/common/MetricsConfiguration.java | 10 +++++++++- uforwarder/build.gradle | 2 +- 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/buildSrc/src/main/groovy/uforwarder.java-common-conventions.gradle b/buildSrc/src/main/groovy/uforwarder.java-common-conventions.gradle index 93023a7..f7b073c 100644 --- a/buildSrc/src/main/groovy/uforwarder.java-common-conventions.gradle +++ b/buildSrc/src/main/groovy/uforwarder.java-common-conventions.gradle @@ -29,7 +29,7 @@ dependencies { implementation 'com.netflix.concurrency-limits:concurrency-limits-core:0.3.6' implementation 'com.uber.concurrency-loadbalancer:concurrency-loadbalancer-core:0.1.5' implementation 'com.uber.m3:tally-core:0.13.0' - implementation 'com.uber.m3:tally-m3:0.13.0' + implementation 'com.uber.m3:tally-statsd:0.13.0' implementation 'io.grpc:grpc-core:1.49.2' implementation 'io.grpc:grpc-netty-shaded:1.49.2' implementation 'io.grpc:grpc-protobuf:1.49.2' diff --git a/instrumentation/build.gradle b/instrumentation/build.gradle index c427d72..ec20e92 100644 --- a/instrumentation/build.gradle +++ b/instrumentation/build.gradle @@ -8,7 +8,7 @@ plugins { dependencies { implementation 'com.uber.m3:tally-core' - implementation 'com.uber.m3:tally-m3' + implementation 'com.uber.m3:tally-statsd' implementation 'io.grpc:grpc-stub' implementation 'io.opentracing:opentracing-api' implementation 'net.logstash.logback:logstash-logback-encoder' diff --git a/uforwarder-core/build.gradle b/uforwarder-core/build.gradle index d09b785..3e46795 100644 --- a/uforwarder-core/build.gradle +++ b/uforwarder-core/build.gradle @@ -8,15 +8,17 @@ plugins { dependencies { implementation 'com.101tec:zkclient' + implementation 'com.datadoghq:java-dogstatsd-client:4.3.0' implementation 'com.facebook.infer.annotation:infer-annotation' implementation 'com.google.api:api-common' implementation 'com.google.api.grpc:proto-google-common-protos' implementation 'com.google.guava:guava' implementation 'com.google.protobuf:protobuf-java' implementation 'com.google.protobuf:protobuf-java-util' + implementation 'commons-codec:commons-codec:1.15' implementation 'com.uber.concurrency-loadbalancer:concurrency-loadbalancer-core' implementation 'com.uber.m3:tally-core' - implementation 'com.uber.m3:tally-m3' + implementation 'com.uber.m3:tally-statsd' implementation 'io.grpc:grpc-netty-shaded' implementation 'io.grpc:grpc-protobuf' implementation 'io.grpc:grpc-services' diff --git a/uforwarder-core/src/main/java/com/uber/data/kafka/datatransfer/common/MetricsConfiguration.java b/uforwarder-core/src/main/java/com/uber/data/kafka/datatransfer/common/MetricsConfiguration.java index 209ea95..6a23a30 100644 --- a/uforwarder-core/src/main/java/com/uber/data/kafka/datatransfer/common/MetricsConfiguration.java +++ b/uforwarder-core/src/main/java/com/uber/data/kafka/datatransfer/common/MetricsConfiguration.java @@ -1,7 +1,10 @@ package com.uber.data.kafka.datatransfer.common; +import com.timgroup.statsd.NonBlockingStatsDClientBuilder; +import com.timgroup.statsd.StatsDClient; import com.uber.m3.tally.RootScopeBuilder; import com.uber.m3.tally.Scope; +import com.uber.m3.tally.statsd.StatsdReporter; import com.uber.m3.util.Duration; import com.uber.m3.util.ImmutableMap; import java.util.Objects; @@ -29,8 +32,13 @@ public class MetricsConfiguration { @ConditionalOnMissingBean public Scope rootScope(@Value("${tally.publish.interval.sec:5}") int tallyPublishIntervalSec) { if (INSTANCE == null) { + StatsDClient statsd = new NonBlockingStatsDClientBuilder() + .prefix("kforwarder") + .hostname("localhost") + .port(8125) + .build(); INSTANCE = - new RootScopeBuilder() + new RootScopeBuilder().reporter(new StatsdReporter(statsd)) .tags(new ImmutableMap.Builder().build()) .reportEvery(Duration.ofSeconds(tallyPublishIntervalSec)); } diff --git a/uforwarder/build.gradle b/uforwarder/build.gradle index f3fb7ae..1cdf554 100644 --- a/uforwarder/build.gradle +++ b/uforwarder/build.gradle @@ -16,7 +16,7 @@ dependencies { implementation 'com.netflix.concurrency-limits:concurrency-limits-core' implementation 'com.uber.concurrency-loadbalancer:concurrency-loadbalancer-core' implementation 'com.uber.m3:tally-core' - implementation 'com.uber.m3:tally-m3' + implementation 'com.uber.m3:tally-statsd' implementation 'io.grpc:grpc-core' implementation 'io.grpc:grpc-netty-shaded' implementation 'io.grpc:grpc-services' From 817a1e3b4ae0656b457aac5946f82e56615db3f4 Mon Sep 17 00:00:00 2001 From: suez1224 Date: Sun, 21 Apr 2024 23:38:52 -0700 Subject: [PATCH 2/6] support both statsd and existing M3 reporting via config --- .../uforwarder.java-common-conventions.gradle | 3 ++ instrumentation/build.gradle | 1 + uforwarder-core/build.gradle | 5 +-- .../common/MetricsConfiguration.java | 34 +++++++++++++++---- uforwarder/build.gradle | 1 + .../application-uforwarder-controller.yaml | 1 + .../application-uforwarder-worker.yaml | 1 + 7 files changed, 38 insertions(+), 8 deletions(-) diff --git a/buildSrc/src/main/groovy/uforwarder.java-common-conventions.gradle b/buildSrc/src/main/groovy/uforwarder.java-common-conventions.gradle index f7b073c..5b9bfcd 100644 --- a/buildSrc/src/main/groovy/uforwarder.java-common-conventions.gradle +++ b/buildSrc/src/main/groovy/uforwarder.java-common-conventions.gradle @@ -19,6 +19,7 @@ dependencies { implementation 'ch.qos.logback:logback-classic:1.2.3' implementation 'ch.qos.logback:logback-core:1.2.3' implementation 'com.101tec:zkclient:0.11' + implementation 'com.datadoghq:java-dogstatsd-client:4.3.0' implementation 'com.facebook.infer.annotation:infer-annotation:0.17.0' implementation 'com.github.stefanbirkner:system-rules:1.19.0' implementation 'com.google.api:api-common:1.7.0' @@ -29,7 +30,9 @@ dependencies { implementation 'com.netflix.concurrency-limits:concurrency-limits-core:0.3.6' implementation 'com.uber.concurrency-loadbalancer:concurrency-loadbalancer-core:0.1.5' implementation 'com.uber.m3:tally-core:0.13.0' + implementation 'com.uber.m3:tally-m3:0.13.0' implementation 'com.uber.m3:tally-statsd:0.13.0' + implementation 'commons-codec:commons-codec:1.15' implementation 'io.grpc:grpc-core:1.49.2' implementation 'io.grpc:grpc-netty-shaded:1.49.2' implementation 'io.grpc:grpc-protobuf:1.49.2' diff --git a/instrumentation/build.gradle b/instrumentation/build.gradle index ec20e92..5aff1bf 100644 --- a/instrumentation/build.gradle +++ b/instrumentation/build.gradle @@ -8,6 +8,7 @@ plugins { dependencies { implementation 'com.uber.m3:tally-core' + implementation 'com.uber.m3:tally-m3' implementation 'com.uber.m3:tally-statsd' implementation 'io.grpc:grpc-stub' implementation 'io.opentracing:opentracing-api' diff --git a/uforwarder-core/build.gradle b/uforwarder-core/build.gradle index 3e46795..82f11f6 100644 --- a/uforwarder-core/build.gradle +++ b/uforwarder-core/build.gradle @@ -8,16 +8,17 @@ plugins { dependencies { implementation 'com.101tec:zkclient' - implementation 'com.datadoghq:java-dogstatsd-client:4.3.0' + implementation 'com.datadoghq:java-dogstatsd-client' implementation 'com.facebook.infer.annotation:infer-annotation' implementation 'com.google.api:api-common' implementation 'com.google.api.grpc:proto-google-common-protos' implementation 'com.google.guava:guava' implementation 'com.google.protobuf:protobuf-java' implementation 'com.google.protobuf:protobuf-java-util' - implementation 'commons-codec:commons-codec:1.15' + implementation 'commons-codec:commons-codec' implementation 'com.uber.concurrency-loadbalancer:concurrency-loadbalancer-core' implementation 'com.uber.m3:tally-core' + implementation 'com.uber.m3:tally-m3' implementation 'com.uber.m3:tally-statsd' implementation 'io.grpc:grpc-netty-shaded' implementation 'io.grpc:grpc-protobuf' diff --git a/uforwarder-core/src/main/java/com/uber/data/kafka/datatransfer/common/MetricsConfiguration.java b/uforwarder-core/src/main/java/com/uber/data/kafka/datatransfer/common/MetricsConfiguration.java index 6a23a30..f53e952 100644 --- a/uforwarder-core/src/main/java/com/uber/data/kafka/datatransfer/common/MetricsConfiguration.java +++ b/uforwarder-core/src/main/java/com/uber/data/kafka/datatransfer/common/MetricsConfiguration.java @@ -4,6 +4,8 @@ import com.timgroup.statsd.StatsDClient; import com.uber.m3.tally.RootScopeBuilder; import com.uber.m3.tally.Scope; +import com.uber.m3.tally.StatsReporter; +import com.uber.m3.tally.m3.M3Reporter; import com.uber.m3.tally.statsd.StatsdReporter; import com.uber.m3.util.Duration; import com.uber.m3.util.ImmutableMap; @@ -21,6 +23,21 @@ public class MetricsConfiguration { @Nullable static Scope INSTANCE; + private static final String METRICS_REPORTER_STATSD = "statsd"; + private static final String METRICS_REPORTER_M3 = "m3"; + + // The metrics reporter to use, currently only M3 and statsd is supported now + // see https://github.com/uber-java/tally for more details + private String metricsReporter = METRICS_REPORTER_M3; + + public String getMetricsReporter() { + return metricsReporter; + } + + public void setMetricsReporter(String metricsReporter) { + this.metricsReporter = metricsReporter; + } + @Bean @Singleton @ConditionalOnProperty( @@ -32,13 +49,18 @@ public class MetricsConfiguration { @ConditionalOnMissingBean public Scope rootScope(@Value("${tally.publish.interval.sec:5}") int tallyPublishIntervalSec) { if (INSTANCE == null) { - StatsDClient statsd = new NonBlockingStatsDClientBuilder() - .prefix("kforwarder") - .hostname("localhost") - .port(8125) - .build(); + StatsReporter statsReporter = null; + if (metricsReporter.equals(METRICS_REPORTER_STATSD)) { + StatsDClient statsd = new NonBlockingStatsDClientBuilder() + .prefix(METRICS_REPORTER_STATSD) + .hostname("localhost") + .port(8125) + .build(); + statsReporter = new StatsdReporter(statsd); + } INSTANCE = - new RootScopeBuilder().reporter(new StatsdReporter(statsd)) + new RootScopeBuilder() + .reporter(statsReporter) .tags(new ImmutableMap.Builder().build()) .reportEvery(Duration.ofSeconds(tallyPublishIntervalSec)); } diff --git a/uforwarder/build.gradle b/uforwarder/build.gradle index 1cdf554..4914b04 100644 --- a/uforwarder/build.gradle +++ b/uforwarder/build.gradle @@ -16,6 +16,7 @@ dependencies { implementation 'com.netflix.concurrency-limits:concurrency-limits-core' implementation 'com.uber.concurrency-loadbalancer:concurrency-loadbalancer-core' implementation 'com.uber.m3:tally-core' + implementation 'com.uber.m3:tally-m3' implementation 'com.uber.m3:tally-statsd' implementation 'io.grpc:grpc-core' implementation 'io.grpc:grpc-netty-shaded' diff --git a/uforwarder/src/main/resources/application-uforwarder-controller.yaml b/uforwarder/src/main/resources/application-uforwarder-controller.yaml index 000b115..c893eef 100644 --- a/uforwarder/src/main/resources/application-uforwarder-controller.yaml +++ b/uforwarder/src/main/resources/application-uforwarder-controller.yaml @@ -23,6 +23,7 @@ metrics: # we add host tags manually to only metrics that require it. # The majority of host level information should leverage the debug pages. includeHostTag: false + metricsReporter: statsd # Enable async-profiler debug pages. jvm: diff --git a/uforwarder/src/main/resources/application-uforwarder-worker.yaml b/uforwarder/src/main/resources/application-uforwarder-worker.yaml index 94b03f4..eb62e96 100644 --- a/uforwarder/src/main/resources/application-uforwarder-worker.yaml +++ b/uforwarder/src/main/resources/application-uforwarder-worker.yaml @@ -21,6 +21,7 @@ metrics: # we add host tags manually to only metrics that require it. # The majority of host level information should leverage the debug pages. includeHostTag: false + metricsReporter: statsd # Enable async-profiler debug pages. jvm: From 41b0671b002371b3b479b882ec22467bb4918490 Mon Sep 17 00:00:00 2001 From: suez1224 Date: Sun, 21 Apr 2024 23:46:24 -0700 Subject: [PATCH 3/6] keep m3 as default --- .../src/main/resources/application-uforwarder-controller.yaml | 2 +- .../src/main/resources/application-uforwarder-worker.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/uforwarder/src/main/resources/application-uforwarder-controller.yaml b/uforwarder/src/main/resources/application-uforwarder-controller.yaml index c893eef..f7d3861 100644 --- a/uforwarder/src/main/resources/application-uforwarder-controller.yaml +++ b/uforwarder/src/main/resources/application-uforwarder-controller.yaml @@ -23,7 +23,7 @@ metrics: # we add host tags manually to only metrics that require it. # The majority of host level information should leverage the debug pages. includeHostTag: false - metricsReporter: statsd + metricsReporter: m3 # Enable async-profiler debug pages. jvm: diff --git a/uforwarder/src/main/resources/application-uforwarder-worker.yaml b/uforwarder/src/main/resources/application-uforwarder-worker.yaml index eb62e96..7a68a5f 100644 --- a/uforwarder/src/main/resources/application-uforwarder-worker.yaml +++ b/uforwarder/src/main/resources/application-uforwarder-worker.yaml @@ -21,7 +21,7 @@ metrics: # we add host tags manually to only metrics that require it. # The majority of host level information should leverage the debug pages. includeHostTag: false - metricsReporter: statsd + metricsReporter: m3 # Enable async-profiler debug pages. jvm: From 6e697993165c633efc3ebf0501df1e0eff556ebf Mon Sep 17 00:00:00 2001 From: suez1224 Date: Mon, 22 Apr 2024 09:55:31 -0700 Subject: [PATCH 4/6] use host.docker.internal for statsd reporting host instead --- .../data/kafka/datatransfer/common/MetricsConfiguration.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uforwarder-core/src/main/java/com/uber/data/kafka/datatransfer/common/MetricsConfiguration.java b/uforwarder-core/src/main/java/com/uber/data/kafka/datatransfer/common/MetricsConfiguration.java index f53e952..8a64ddb 100644 --- a/uforwarder-core/src/main/java/com/uber/data/kafka/datatransfer/common/MetricsConfiguration.java +++ b/uforwarder-core/src/main/java/com/uber/data/kafka/datatransfer/common/MetricsConfiguration.java @@ -23,6 +23,7 @@ public class MetricsConfiguration { @Nullable static Scope INSTANCE; + private static final String DOCKER_HOST_INTERNAL_ADDRESS = "host.docker.internal"; private static final String METRICS_REPORTER_STATSD = "statsd"; private static final String METRICS_REPORTER_M3 = "m3"; @@ -53,7 +54,7 @@ public Scope rootScope(@Value("${tally.publish.interval.sec:5}") int tallyPublis if (metricsReporter.equals(METRICS_REPORTER_STATSD)) { StatsDClient statsd = new NonBlockingStatsDClientBuilder() .prefix(METRICS_REPORTER_STATSD) - .hostname("localhost") + .hostname(DOCKER_HOST_INTERNAL_ADDRESS) .port(8125) .build(); statsReporter = new StatsdReporter(statsd); From b16970d2beb8ecbce4d72a815f1557c995c79dde Mon Sep 17 00:00:00 2001 From: suez1224 Date: Mon, 22 Apr 2024 10:21:21 -0700 Subject: [PATCH 5/6] update metricsReporter comment --- .../src/main/resources/application-uforwarder-controller.yaml | 1 + uforwarder/src/main/resources/application-uforwarder-worker.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/uforwarder/src/main/resources/application-uforwarder-controller.yaml b/uforwarder/src/main/resources/application-uforwarder-controller.yaml index f7d3861..1e3e665 100644 --- a/uforwarder/src/main/resources/application-uforwarder-controller.yaml +++ b/uforwarder/src/main/resources/application-uforwarder-controller.yaml @@ -23,6 +23,7 @@ metrics: # we add host tags manually to only metrics that require it. # The majority of host level information should leverage the debug pages. includeHostTag: false + # Currently, the supported metrics reporter is m3 and statsd metricsReporter: m3 # Enable async-profiler debug pages. diff --git a/uforwarder/src/main/resources/application-uforwarder-worker.yaml b/uforwarder/src/main/resources/application-uforwarder-worker.yaml index 7a68a5f..8fd3fc3 100644 --- a/uforwarder/src/main/resources/application-uforwarder-worker.yaml +++ b/uforwarder/src/main/resources/application-uforwarder-worker.yaml @@ -21,6 +21,7 @@ metrics: # we add host tags manually to only metrics that require it. # The majority of host level information should leverage the debug pages. includeHostTag: false + # Currently, the supported metrics reporter is m3 and statsd metricsReporter: m3 # Enable async-profiler debug pages. From 782eb9453217fd1951bcb84743ff434201dd753c Mon Sep 17 00:00:00 2001 From: suez1224 Date: Mon, 22 Apr 2024 20:10:30 -0700 Subject: [PATCH 6/6] only report metrics if environment variable UFORWARDER_REPORT_METRICS is set --- .../common/MetricsConfiguration.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/uforwarder-core/src/main/java/com/uber/data/kafka/datatransfer/common/MetricsConfiguration.java b/uforwarder-core/src/main/java/com/uber/data/kafka/datatransfer/common/MetricsConfiguration.java index 8a64ddb..f017a4b 100644 --- a/uforwarder-core/src/main/java/com/uber/data/kafka/datatransfer/common/MetricsConfiguration.java +++ b/uforwarder-core/src/main/java/com/uber/data/kafka/datatransfer/common/MetricsConfiguration.java @@ -23,6 +23,7 @@ public class MetricsConfiguration { @Nullable static Scope INSTANCE; + private static final String ENV_UFORWARDER_REPORT_METRICS = "UFORWARDER_REPORT_METRICS"; private static final String DOCKER_HOST_INTERNAL_ADDRESS = "host.docker.internal"; private static final String METRICS_REPORTER_STATSD = "statsd"; private static final String METRICS_REPORTER_M3 = "m3"; @@ -51,13 +52,18 @@ public void setMetricsReporter(String metricsReporter) { public Scope rootScope(@Value("${tally.publish.interval.sec:5}") int tallyPublishIntervalSec) { if (INSTANCE == null) { StatsReporter statsReporter = null; - if (metricsReporter.equals(METRICS_REPORTER_STATSD)) { - StatsDClient statsd = new NonBlockingStatsDClientBuilder() - .prefix(METRICS_REPORTER_STATSD) - .hostname(DOCKER_HOST_INTERNAL_ADDRESS) - .port(8125) - .build(); - statsReporter = new StatsdReporter(statsd); + // If UFORWARDER_REPORT_METRICS is set, we will set metric reporter + // according to the metricsReporter property. + String reportMetrics = System.getenv().get(ENV_UFORWARDER_REPORT_METRICS); + if (reportMetrics != null) { + if (metricsReporter.equals(METRICS_REPORTER_STATSD)) { + StatsDClient statsd = new NonBlockingStatsDClientBuilder() + .prefix(METRICS_REPORTER_STATSD) + .hostname(DOCKER_HOST_INTERNAL_ADDRESS) + .port(8125) + .build(); + statsReporter = new StatsdReporter(statsd); + } } INSTANCE = new RootScopeBuilder()