From e7432dba5c713a6bffb858722ad95e6b4f823752 Mon Sep 17 00:00:00 2001 From: Jiwon Park Date: Thu, 12 Feb 2026 09:55:29 +0900 Subject: [PATCH 1/2] Add metric category filtering support to Prometheus exporter Allow users to enable/disable metric categories (self-optimizing, optimizer-group, orphan-files, ams-jvm, table-summary) via metric-reporters.yaml properties. All categories are enabled by default for backward compatibility. Also update Helm chart values and configmap tests to reflect the new configuration options. Signed-off-by: jiwonpark Signed-off-by: Jiwon Park --- .../metrics/promethues/MetricCategory.java | 84 ++++++++++ .../metrics/promethues/MetricsCollector.java | 20 ++- .../promethues/PrometheusMetricsReporter.java | 7 +- .../promethues/MetricCategoryTest.java | 136 +++++++++++++++ .../MetricsCollectorFilterTest.java | 155 ++++++++++++++++++ charts/amoro/tests/amoro-configmap_test.yaml | 17 +- charts/amoro/values.yaml | 9 +- .../conf/plugins/metric-reporters.yaml | 6 + 8 files changed, 430 insertions(+), 4 deletions(-) create mode 100644 amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricCategory.java create mode 100644 amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricCategoryTest.java create mode 100644 amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricsCollectorFilterTest.java diff --git a/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricCategory.java b/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricCategory.java new file mode 100644 index 0000000000..7fff697674 --- /dev/null +++ b/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricCategory.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.metrics.promethues; + +import java.util.Arrays; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** Metric categories for Prometheus exporter filtering. */ +public enum MetricCategory { + SELF_OPTIMIZING("self-optimizing", "table_optimizing_"), + OPTIMIZER_GROUP("optimizer-group", "optimizer_group_"), + ORPHAN_FILES("orphan-files", "table_orphan_", "table_expected_orphan_"), + AMS_JVM("ams-jvm", "ams_jvm_", "ams_ha_"), + TABLE_SUMMARY("table-summary", "table_summary_"); + + private static final String CATEGORY_PROPERTY_PREFIX = "category."; + private static final String CATEGORY_PROPERTY_SUFFIX = ".enabled"; + + private final String configName; + private final String[] prefixes; + + MetricCategory(String configName, String... prefixes) { + this.configName = configName; + this.prefixes = prefixes; + } + + public String getConfigName() { + return configName; + } + + /** Check if a metric name belongs to this category. */ + public boolean matches(String metricName) { + for (String prefix : prefixes) { + if (metricName.startsWith(prefix)) { + return true; + } + } + return false; + } + + /** + * Parse properties and return the set of disabled categories. Categories not mentioned in + * properties are enabled by default. + */ + public static Set parseDisabledCategories(Map properties) { + return Arrays.stream(values()) + .filter( + category -> { + String key = + CATEGORY_PROPERTY_PREFIX + category.configName + CATEGORY_PROPERTY_SUFFIX; + String value = properties.get(key); + return "false".equalsIgnoreCase(value); + }) + .collect(Collectors.toSet()); + } + + /** Find the category that a metric name belongs to. Returns null if no category matches. */ + public static MetricCategory findCategory(String metricName) { + for (MetricCategory category : values()) { + if (category.matches(metricName)) { + return category; + } + } + return null; + } +} diff --git a/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricsCollector.java b/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricsCollector.java index adc2b75088..5c60f8dcdf 100644 --- a/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricsCollector.java +++ b/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricsCollector.java @@ -30,8 +30,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -43,9 +45,16 @@ public class MetricsCollector extends Collector { private static final Pattern NAME_PATTERN = Pattern.compile("[a-zA-Z_:][a-zA-Z0-9_:]*"); private static final Pattern LABEL_PATTERN = Pattern.compile("[a-zA-Z_][a-zA-Z0-9_]*"); MetricSet metrics; + private final Set disabledCategories; public MetricsCollector(MetricSet metrics) { + this(metrics, Collections.emptySet()); + } + + public MetricsCollector(MetricSet metrics, Set disabledCategories) { this.metrics = metrics; + this.disabledCategories = + disabledCategories != null ? disabledCategories : Collections.emptySet(); } @Override @@ -76,8 +85,17 @@ private boolean isValidMetric(MetricDefine define) { boolean valid = nameIsValid && labelIsValid; if (!valid) { LOGGER.warn("Metric {} is not a valid prometheus metric.", define); + return false; + } + + if (!disabledCategories.isEmpty()) { + MetricCategory category = MetricCategory.findCategory(define.getName()); + if (category != null && disabledCategories.contains(category)) { + return false; + } } - return valid; + + return true; } private MetricFamilySamples createFamilySample( diff --git a/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/PrometheusMetricsReporter.java b/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/PrometheusMetricsReporter.java index fd8f6f8a81..f973fac439 100644 --- a/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/PrometheusMetricsReporter.java +++ b/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/PrometheusMetricsReporter.java @@ -23,8 +23,10 @@ import org.apache.amoro.metrics.MetricSet; import java.io.IOException; +import java.util.Collections; import java.util.Map; import java.util.Optional; +import java.util.Set; /** Prometheus exporter */ public class PrometheusMetricsReporter implements MetricReporter { @@ -32,6 +34,7 @@ public class PrometheusMetricsReporter implements MetricReporter { public static final String PORT = "port"; private HTTPServer server; + private Set disabledCategories = Collections.emptySet(); @Override public void open(Map properties) { @@ -40,6 +43,8 @@ public void open(Map properties) { .map(Integer::valueOf) .orElseThrow(() -> new IllegalArgumentException("Lack required property: " + PORT)); + this.disabledCategories = MetricCategory.parseDisabledCategories(properties); + try { this.server = new HTTPServer(port); } catch (IOException e) { @@ -59,7 +64,7 @@ public String name() { @Override public void setGlobalMetricSet(MetricSet globalMetricSet) { - MetricsCollector collector = new MetricsCollector(globalMetricSet); + MetricsCollector collector = new MetricsCollector(globalMetricSet, disabledCategories); collector.register(); } } diff --git a/amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricCategoryTest.java b/amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricCategoryTest.java new file mode 100644 index 0000000000..d97af78bb7 --- /dev/null +++ b/amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricCategoryTest.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.metrics.promethues; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public class MetricCategoryTest { + + @Test + public void testMatchesSelfOptimizing() { + assertTrue( + MetricCategory.SELF_OPTIMIZING.matches("table_optimizing_status_idle_duration_mills")); + assertTrue(MetricCategory.SELF_OPTIMIZING.matches("table_optimizing_process_total_count")); + assertFalse(MetricCategory.SELF_OPTIMIZING.matches("optimizer_group_pending_tasks")); + assertFalse(MetricCategory.SELF_OPTIMIZING.matches("table_summary_total_files")); + } + + @Test + public void testMatchesOptimizerGroup() { + assertTrue(MetricCategory.OPTIMIZER_GROUP.matches("optimizer_group_pending_tasks")); + assertTrue(MetricCategory.OPTIMIZER_GROUP.matches("optimizer_group_threads")); + assertFalse(MetricCategory.OPTIMIZER_GROUP.matches("table_optimizing_status_in_idle")); + } + + @Test + public void testMatchesOrphanFiles() { + assertTrue(MetricCategory.ORPHAN_FILES.matches("table_orphan_content_file_cleaning_count")); + assertTrue( + MetricCategory.ORPHAN_FILES.matches("table_expected_orphan_metadata_file_cleaning_count")); + assertFalse(MetricCategory.ORPHAN_FILES.matches("table_summary_total_files")); + } + + @Test + public void testMatchesAmsJvm() { + assertTrue(MetricCategory.AMS_JVM.matches("ams_jvm_cpu_load")); + assertTrue(MetricCategory.AMS_JVM.matches("ams_jvm_memory_heap_used")); + assertTrue(MetricCategory.AMS_JVM.matches("ams_ha_state")); + assertFalse(MetricCategory.AMS_JVM.matches("table_summary_total_files")); + } + + @Test + public void testMatchesTableSummary() { + assertTrue(MetricCategory.TABLE_SUMMARY.matches("table_summary_total_files")); + assertTrue(MetricCategory.TABLE_SUMMARY.matches("table_summary_health_score")); + assertFalse(MetricCategory.TABLE_SUMMARY.matches("table_optimizing_status_in_idle")); + } + + @Test + public void testFindCategory() { + assertEquals( + MetricCategory.SELF_OPTIMIZING, + MetricCategory.findCategory("table_optimizing_status_idle_duration_mills")); + assertEquals( + MetricCategory.OPTIMIZER_GROUP, + MetricCategory.findCategory("optimizer_group_pending_tasks")); + assertEquals( + MetricCategory.ORPHAN_FILES, + MetricCategory.findCategory("table_orphan_content_file_cleaning_count")); + assertEquals(MetricCategory.AMS_JVM, MetricCategory.findCategory("ams_jvm_cpu_load")); + assertEquals(MetricCategory.AMS_JVM, MetricCategory.findCategory("ams_ha_state")); + assertEquals( + MetricCategory.TABLE_SUMMARY, MetricCategory.findCategory("table_summary_total_files")); + assertNull(MetricCategory.findCategory("unknown_metric")); + } + + @Test + public void testParseDisabledCategoriesEmpty() { + Set disabled = MetricCategory.parseDisabledCategories(Collections.emptyMap()); + assertTrue(disabled.isEmpty()); + } + + @Test + public void testParseDisabledCategoriesSomeDisabled() { + Map properties = new HashMap<>(); + properties.put("category.self-optimizing.enabled", "false"); + properties.put("category.table-summary.enabled", "false"); + properties.put("category.ams-jvm.enabled", "true"); + + Set disabled = MetricCategory.parseDisabledCategories(properties); + + assertEquals(2, disabled.size()); + assertTrue(disabled.contains(MetricCategory.SELF_OPTIMIZING)); + assertTrue(disabled.contains(MetricCategory.TABLE_SUMMARY)); + assertFalse(disabled.contains(MetricCategory.AMS_JVM)); + assertFalse(disabled.contains(MetricCategory.OPTIMIZER_GROUP)); + } + + @Test + public void testParseDisabledCategoriesCaseInsensitive() { + Map properties = new HashMap<>(); + properties.put("category.optimizer-group.enabled", "FALSE"); + properties.put("category.orphan-files.enabled", "False"); + + Set disabled = MetricCategory.parseDisabledCategories(properties); + + assertEquals(2, disabled.size()); + assertTrue(disabled.contains(MetricCategory.OPTIMIZER_GROUP)); + assertTrue(disabled.contains(MetricCategory.ORPHAN_FILES)); + } + + @Test + public void testParseDisabledCategoriesInvalidValueTreatedAsEnabled() { + Map properties = new HashMap<>(); + properties.put("category.ams-jvm.enabled", "invalid"); + + Set disabled = MetricCategory.parseDisabledCategories(properties); + + assertTrue(disabled.isEmpty()); + } +} diff --git a/amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricsCollectorFilterTest.java b/amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricsCollectorFilterTest.java new file mode 100644 index 0000000000..705f02453a --- /dev/null +++ b/amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricsCollectorFilterTest.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.metrics.promethues; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.prometheus.client.Collector; +import org.apache.amoro.metrics.Counter; +import org.apache.amoro.metrics.Gauge; +import org.apache.amoro.metrics.Metric; +import org.apache.amoro.metrics.MetricDefine; +import org.apache.amoro.metrics.MetricKey; +import org.apache.amoro.metrics.MetricSet; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class MetricsCollectorFilterTest { + + private MetricSet createMetricSet(Map metrics) { + return () -> Collections.unmodifiableMap(metrics); + } + + private MetricKey registerMetric( + Map metrics, String name, Metric metric, String... tags) { + MetricDefine.Builder builder; + if (metric instanceof Counter) { + builder = MetricDefine.defineCounter(name); + } else { + builder = MetricDefine.defineGauge(name); + } + if (tags.length > 0) { + builder.withTags(tags); + } + MetricDefine define = builder.build(); + + Map tagValues = new HashMap<>(); + for (String tag : tags) { + tagValues.put(tag, "test_value"); + } + MetricKey key = new MetricKey(define, tagValues); + metrics.put(key, metric); + return key; + } + + @Test + public void testCollectWithNoFilter() { + Map metrics = new HashMap<>(); + registerMetric(metrics, "table_optimizing_status_in_idle", (Gauge) () -> 1L); + registerMetric(metrics, "optimizer_group_pending_tasks", (Gauge) () -> 5L); + registerMetric(metrics, "ams_jvm_cpu_load", (Gauge) () -> 0.5); + + MetricsCollector collector = new MetricsCollector(createMetricSet(metrics)); + List result = collector.collect(); + + assertEquals(3, result.size()); + } + + @Test + public void testCollectWithDisabledCategory() { + Map metrics = new HashMap<>(); + registerMetric(metrics, "table_optimizing_status_in_idle", (Gauge) () -> 1L); + registerMetric(metrics, "optimizer_group_pending_tasks", (Gauge) () -> 5L); + registerMetric(metrics, "ams_jvm_cpu_load", (Gauge) () -> 0.5); + + Set disabled = EnumSet.of(MetricCategory.SELF_OPTIMIZING); + MetricsCollector collector = new MetricsCollector(createMetricSet(metrics), disabled); + List result = collector.collect(); + + assertEquals(2, result.size()); + Set names = result.stream().map(s -> s.name).collect(Collectors.toSet()); + assertFalse(names.contains("amoro_table_optimizing_status_in_idle")); + assertTrue(names.contains("amoro_optimizer_group_pending_tasks")); + assertTrue(names.contains("amoro_ams_jvm_cpu_load")); + } + + @Test + public void testCollectWithMultipleDisabledCategories() { + Map metrics = new HashMap<>(); + registerMetric(metrics, "table_optimizing_status_in_idle", (Gauge) () -> 1L); + registerMetric(metrics, "optimizer_group_pending_tasks", (Gauge) () -> 5L); + registerMetric(metrics, "ams_jvm_cpu_load", (Gauge) () -> 0.5); + registerMetric(metrics, "table_summary_total_files", (Gauge) () -> 100L); + + Set disabled = + EnumSet.of(MetricCategory.SELF_OPTIMIZING, MetricCategory.TABLE_SUMMARY); + MetricsCollector collector = new MetricsCollector(createMetricSet(metrics), disabled); + List result = collector.collect(); + + assertEquals(2, result.size()); + Set names = result.stream().map(s -> s.name).collect(Collectors.toSet()); + assertTrue(names.contains("amoro_optimizer_group_pending_tasks")); + assertTrue(names.contains("amoro_ams_jvm_cpu_load")); + } + + @Test + public void testUnknownMetricNotFiltered() { + Map metrics = new HashMap<>(); + registerMetric(metrics, "custom_unknown_metric", (Gauge) () -> 42L); + + Set disabled = + EnumSet.of( + MetricCategory.SELF_OPTIMIZING, + MetricCategory.OPTIMIZER_GROUP, + MetricCategory.ORPHAN_FILES, + MetricCategory.AMS_JVM, + MetricCategory.TABLE_SUMMARY); + MetricsCollector collector = new MetricsCollector(createMetricSet(metrics), disabled); + List result = collector.collect(); + + assertEquals(1, result.size()); + assertEquals("amoro_custom_unknown_metric", result.get(0).name); + } + + @Test + public void testOrphanFilesMultiplePrefixes() { + Map metrics = new HashMap<>(); + Counter counter1 = new Counter(); + Counter counter2 = new Counter(); + registerMetric(metrics, "table_orphan_content_file_cleaning_count", counter1); + registerMetric(metrics, "table_expected_orphan_metadata_file_cleaning_count", counter2); + registerMetric(metrics, "ams_jvm_cpu_load", (Gauge) () -> 0.5); + + Set disabled = EnumSet.of(MetricCategory.ORPHAN_FILES); + MetricsCollector collector = new MetricsCollector(createMetricSet(metrics), disabled); + List result = collector.collect(); + + assertEquals(1, result.size()); + assertEquals("amoro_ams_jvm_cpu_load", result.get(0).name); + } +} diff --git a/charts/amoro/tests/amoro-configmap_test.yaml b/charts/amoro/tests/amoro-configmap_test.yaml index 0c68186936..70f2948865 100644 --- a/charts/amoro/tests/amoro-configmap_test.yaml +++ b/charts/amoro/tests/amoro-configmap_test.yaml @@ -151,4 +151,19 @@ tests: path: data["metric-reporters.yaml"] pattern: | - name: prometheus-exporter \ No newline at end of file + name: prometheus-exporter + - it: Amoro configMap should reflect metric category filtering + set: + plugin: + metricReporters: + prometheusExporter: + name: prometheus-exporter + enabled: true + properties: + port: 7001 + category.self-optimizing.enabled: "true" + category.table-summary.enabled: "false" + asserts: + - matchRegex: + path: data["metric-reporters.yaml"] + pattern: "category.table-summary.enabled.*false" \ No newline at end of file diff --git a/charts/amoro/values.yaml b/charts/amoro/values.yaml index 312fefaea0..39aa7d0953 100644 --- a/charts/amoro/values.yaml +++ b/charts/amoro/values.yaml @@ -287,7 +287,7 @@ plugin: ## metric reporters ## metricReporters: ~ -## e.g: +## e.g: # metricReporters: # ## @param Configure Prometheus exporter # ## @@ -302,6 +302,13 @@ plugin: # ## @param Prometheus port # ## # port: 7001 +# ## @param Metric category filtering (all enabled by default) +# ## +# category.self-optimizing.enabled: "true" +# category.optimizer-group.enabled: "true" +# category.orphan-files.enabled: "true" +# category.ams-jvm.enabled: "true" +# category.table-summary.enabled: "true" ## Configure the ingress resource that allows you to access the ## Amoro installation. Set up the URL diff --git a/dist/src/main/amoro-bin/conf/plugins/metric-reporters.yaml b/dist/src/main/amoro-bin/conf/plugins/metric-reporters.yaml index 40842e8177..043c315902 100644 --- a/dist/src/main/amoro-bin/conf/plugins/metric-reporters.yaml +++ b/dist/src/main/amoro-bin/conf/plugins/metric-reporters.yaml @@ -23,4 +23,10 @@ metric-reporters: # enabled: false # properties: # port: 7001 +# # Metric category filtering (all enabled by default) +# # category.self-optimizing.enabled: "true" +# # category.optimizer-group.enabled: "true" +# # category.orphan-files.enabled: "true" +# # category.ams-jvm.enabled: "true" +# # category.table-summary.enabled: "true" From 359aa5e863cb1174cf74c233152a5d68696e82ec Mon Sep 17 00:00:00 2001 From: Jiwon Park Date: Fri, 13 Feb 2026 17:23:55 +0900 Subject: [PATCH 2/2] Replace category-based filtering with regex-based metric filtering Replace the fixed category enum approach with a flexible regex-based metric filter. Users can now specify include/exclude patterns via metric-filter.includes and metric-filter.excludes properties to freely configure which metrics are exported. Signed-off-by: Jiwon Park --- .../metrics/promethues/MetricCategory.java | 84 ----------- .../metrics/promethues/MetricFilter.java | 64 +++++++++ .../metrics/promethues/MetricsCollector.java | 18 +-- .../promethues/PrometheusMetricsReporter.java | 8 +- .../promethues/MetricCategoryTest.java | 136 ------------------ .../metrics/promethues/MetricFilterTest.java | 105 ++++++++++++++ ...a => MetricsCollectorRegexFilterTest.java} | 84 +++++------ charts/amoro/tests/amoro-configmap_test.yaml | 7 +- charts/amoro/values.yaml | 9 +- .../conf/plugins/metric-reporters.yaml | 9 +- 10 files changed, 220 insertions(+), 304 deletions(-) delete mode 100644 amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricCategory.java create mode 100644 amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricFilter.java delete mode 100644 amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricCategoryTest.java create mode 100644 amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricFilterTest.java rename amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/{MetricsCollectorFilterTest.java => MetricsCollectorRegexFilterTest.java} (65%) diff --git a/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricCategory.java b/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricCategory.java deleted file mode 100644 index 7fff697674..0000000000 --- a/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricCategory.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.amoro.metrics.promethues; - -import java.util.Arrays; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -/** Metric categories for Prometheus exporter filtering. */ -public enum MetricCategory { - SELF_OPTIMIZING("self-optimizing", "table_optimizing_"), - OPTIMIZER_GROUP("optimizer-group", "optimizer_group_"), - ORPHAN_FILES("orphan-files", "table_orphan_", "table_expected_orphan_"), - AMS_JVM("ams-jvm", "ams_jvm_", "ams_ha_"), - TABLE_SUMMARY("table-summary", "table_summary_"); - - private static final String CATEGORY_PROPERTY_PREFIX = "category."; - private static final String CATEGORY_PROPERTY_SUFFIX = ".enabled"; - - private final String configName; - private final String[] prefixes; - - MetricCategory(String configName, String... prefixes) { - this.configName = configName; - this.prefixes = prefixes; - } - - public String getConfigName() { - return configName; - } - - /** Check if a metric name belongs to this category. */ - public boolean matches(String metricName) { - for (String prefix : prefixes) { - if (metricName.startsWith(prefix)) { - return true; - } - } - return false; - } - - /** - * Parse properties and return the set of disabled categories. Categories not mentioned in - * properties are enabled by default. - */ - public static Set parseDisabledCategories(Map properties) { - return Arrays.stream(values()) - .filter( - category -> { - String key = - CATEGORY_PROPERTY_PREFIX + category.configName + CATEGORY_PROPERTY_SUFFIX; - String value = properties.get(key); - return "false".equalsIgnoreCase(value); - }) - .collect(Collectors.toSet()); - } - - /** Find the category that a metric name belongs to. Returns null if no category matches. */ - public static MetricCategory findCategory(String metricName) { - for (MetricCategory category : values()) { - if (category.matches(metricName)) { - return category; - } - } - return null; - } -} diff --git a/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricFilter.java b/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricFilter.java new file mode 100644 index 0000000000..010abe5ec4 --- /dev/null +++ b/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricFilter.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.metrics.promethues; + +import java.util.Map; +import java.util.regex.Pattern; + +/** Regex-based metric filter for Prometheus exporter. */ +public class MetricFilter { + + public static final String INCLUDES_KEY = "metric-filter.includes"; + public static final String EXCLUDES_KEY = "metric-filter.excludes"; + + public static final MetricFilter ACCEPT_ALL = new MetricFilter(null, null); + + private final Pattern includePattern; + private final Pattern excludePattern; + + public MetricFilter(Pattern includePattern, Pattern excludePattern) { + this.includePattern = includePattern; + this.excludePattern = excludePattern; + } + + /** Parse metric filter from reporter properties. */ + public static MetricFilter fromProperties(Map properties) { + String includes = properties.get(INCLUDES_KEY); + String excludes = properties.get(EXCLUDES_KEY); + + if (includes == null && excludes == null) { + return ACCEPT_ALL; + } + + Pattern includePattern = includes != null ? Pattern.compile(includes) : null; + Pattern excludePattern = excludes != null ? Pattern.compile(excludes) : null; + return new MetricFilter(includePattern, excludePattern); + } + + /** Check if a metric name passes the filter. */ + public boolean matches(String metricName) { + if (includePattern != null && !includePattern.matcher(metricName).matches()) { + return false; + } + if (excludePattern != null && excludePattern.matcher(metricName).matches()) { + return false; + } + return true; + } +} diff --git a/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricsCollector.java b/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricsCollector.java index 5c60f8dcdf..9e117c0a5d 100644 --- a/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricsCollector.java +++ b/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/MetricsCollector.java @@ -30,10 +30,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.function.Function; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -45,16 +43,15 @@ public class MetricsCollector extends Collector { private static final Pattern NAME_PATTERN = Pattern.compile("[a-zA-Z_:][a-zA-Z0-9_:]*"); private static final Pattern LABEL_PATTERN = Pattern.compile("[a-zA-Z_][a-zA-Z0-9_]*"); MetricSet metrics; - private final Set disabledCategories; + private final MetricFilter metricFilter; public MetricsCollector(MetricSet metrics) { - this(metrics, Collections.emptySet()); + this(metrics, MetricFilter.ACCEPT_ALL); } - public MetricsCollector(MetricSet metrics, Set disabledCategories) { + public MetricsCollector(MetricSet metrics, MetricFilter metricFilter) { this.metrics = metrics; - this.disabledCategories = - disabledCategories != null ? disabledCategories : Collections.emptySet(); + this.metricFilter = metricFilter != null ? metricFilter : MetricFilter.ACCEPT_ALL; } @Override @@ -88,11 +85,8 @@ private boolean isValidMetric(MetricDefine define) { return false; } - if (!disabledCategories.isEmpty()) { - MetricCategory category = MetricCategory.findCategory(define.getName()); - if (category != null && disabledCategories.contains(category)) { - return false; - } + if (!metricFilter.matches(define.getName())) { + return false; } return true; diff --git a/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/PrometheusMetricsReporter.java b/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/PrometheusMetricsReporter.java index f973fac439..4ba175f5d0 100644 --- a/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/PrometheusMetricsReporter.java +++ b/amoro-metrics/amoro-metrics-prometheus/src/main/java/org/apache/amoro/metrics/promethues/PrometheusMetricsReporter.java @@ -23,10 +23,8 @@ import org.apache.amoro.metrics.MetricSet; import java.io.IOException; -import java.util.Collections; import java.util.Map; import java.util.Optional; -import java.util.Set; /** Prometheus exporter */ public class PrometheusMetricsReporter implements MetricReporter { @@ -34,7 +32,7 @@ public class PrometheusMetricsReporter implements MetricReporter { public static final String PORT = "port"; private HTTPServer server; - private Set disabledCategories = Collections.emptySet(); + private MetricFilter metricFilter = MetricFilter.ACCEPT_ALL; @Override public void open(Map properties) { @@ -43,7 +41,7 @@ public void open(Map properties) { .map(Integer::valueOf) .orElseThrow(() -> new IllegalArgumentException("Lack required property: " + PORT)); - this.disabledCategories = MetricCategory.parseDisabledCategories(properties); + this.metricFilter = MetricFilter.fromProperties(properties); try { this.server = new HTTPServer(port); @@ -64,7 +62,7 @@ public String name() { @Override public void setGlobalMetricSet(MetricSet globalMetricSet) { - MetricsCollector collector = new MetricsCollector(globalMetricSet, disabledCategories); + MetricsCollector collector = new MetricsCollector(globalMetricSet, metricFilter); collector.register(); } } diff --git a/amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricCategoryTest.java b/amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricCategoryTest.java deleted file mode 100644 index d97af78bb7..0000000000 --- a/amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricCategoryTest.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.amoro.metrics.promethues; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import org.junit.jupiter.api.Test; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -public class MetricCategoryTest { - - @Test - public void testMatchesSelfOptimizing() { - assertTrue( - MetricCategory.SELF_OPTIMIZING.matches("table_optimizing_status_idle_duration_mills")); - assertTrue(MetricCategory.SELF_OPTIMIZING.matches("table_optimizing_process_total_count")); - assertFalse(MetricCategory.SELF_OPTIMIZING.matches("optimizer_group_pending_tasks")); - assertFalse(MetricCategory.SELF_OPTIMIZING.matches("table_summary_total_files")); - } - - @Test - public void testMatchesOptimizerGroup() { - assertTrue(MetricCategory.OPTIMIZER_GROUP.matches("optimizer_group_pending_tasks")); - assertTrue(MetricCategory.OPTIMIZER_GROUP.matches("optimizer_group_threads")); - assertFalse(MetricCategory.OPTIMIZER_GROUP.matches("table_optimizing_status_in_idle")); - } - - @Test - public void testMatchesOrphanFiles() { - assertTrue(MetricCategory.ORPHAN_FILES.matches("table_orphan_content_file_cleaning_count")); - assertTrue( - MetricCategory.ORPHAN_FILES.matches("table_expected_orphan_metadata_file_cleaning_count")); - assertFalse(MetricCategory.ORPHAN_FILES.matches("table_summary_total_files")); - } - - @Test - public void testMatchesAmsJvm() { - assertTrue(MetricCategory.AMS_JVM.matches("ams_jvm_cpu_load")); - assertTrue(MetricCategory.AMS_JVM.matches("ams_jvm_memory_heap_used")); - assertTrue(MetricCategory.AMS_JVM.matches("ams_ha_state")); - assertFalse(MetricCategory.AMS_JVM.matches("table_summary_total_files")); - } - - @Test - public void testMatchesTableSummary() { - assertTrue(MetricCategory.TABLE_SUMMARY.matches("table_summary_total_files")); - assertTrue(MetricCategory.TABLE_SUMMARY.matches("table_summary_health_score")); - assertFalse(MetricCategory.TABLE_SUMMARY.matches("table_optimizing_status_in_idle")); - } - - @Test - public void testFindCategory() { - assertEquals( - MetricCategory.SELF_OPTIMIZING, - MetricCategory.findCategory("table_optimizing_status_idle_duration_mills")); - assertEquals( - MetricCategory.OPTIMIZER_GROUP, - MetricCategory.findCategory("optimizer_group_pending_tasks")); - assertEquals( - MetricCategory.ORPHAN_FILES, - MetricCategory.findCategory("table_orphan_content_file_cleaning_count")); - assertEquals(MetricCategory.AMS_JVM, MetricCategory.findCategory("ams_jvm_cpu_load")); - assertEquals(MetricCategory.AMS_JVM, MetricCategory.findCategory("ams_ha_state")); - assertEquals( - MetricCategory.TABLE_SUMMARY, MetricCategory.findCategory("table_summary_total_files")); - assertNull(MetricCategory.findCategory("unknown_metric")); - } - - @Test - public void testParseDisabledCategoriesEmpty() { - Set disabled = MetricCategory.parseDisabledCategories(Collections.emptyMap()); - assertTrue(disabled.isEmpty()); - } - - @Test - public void testParseDisabledCategoriesSomeDisabled() { - Map properties = new HashMap<>(); - properties.put("category.self-optimizing.enabled", "false"); - properties.put("category.table-summary.enabled", "false"); - properties.put("category.ams-jvm.enabled", "true"); - - Set disabled = MetricCategory.parseDisabledCategories(properties); - - assertEquals(2, disabled.size()); - assertTrue(disabled.contains(MetricCategory.SELF_OPTIMIZING)); - assertTrue(disabled.contains(MetricCategory.TABLE_SUMMARY)); - assertFalse(disabled.contains(MetricCategory.AMS_JVM)); - assertFalse(disabled.contains(MetricCategory.OPTIMIZER_GROUP)); - } - - @Test - public void testParseDisabledCategoriesCaseInsensitive() { - Map properties = new HashMap<>(); - properties.put("category.optimizer-group.enabled", "FALSE"); - properties.put("category.orphan-files.enabled", "False"); - - Set disabled = MetricCategory.parseDisabledCategories(properties); - - assertEquals(2, disabled.size()); - assertTrue(disabled.contains(MetricCategory.OPTIMIZER_GROUP)); - assertTrue(disabled.contains(MetricCategory.ORPHAN_FILES)); - } - - @Test - public void testParseDisabledCategoriesInvalidValueTreatedAsEnabled() { - Map properties = new HashMap<>(); - properties.put("category.ams-jvm.enabled", "invalid"); - - Set disabled = MetricCategory.parseDisabledCategories(properties); - - assertTrue(disabled.isEmpty()); - } -} diff --git a/amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricFilterTest.java b/amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricFilterTest.java new file mode 100644 index 0000000000..7da7f00558 --- /dev/null +++ b/amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricFilterTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.metrics.promethues; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class MetricFilterTest { + + @Test + public void testAcceptAllMatchesEverything() { + MetricFilter filter = MetricFilter.ACCEPT_ALL; + assertTrue(filter.matches("table_optimizing_status_in_idle")); + assertTrue(filter.matches("optimizer_group_pending_tasks")); + assertTrue(filter.matches("any_metric_name")); + } + + @Test + public void testFromPropertiesEmptyReturnsAcceptAll() { + MetricFilter filter = MetricFilter.fromProperties(Collections.emptyMap()); + assertSame(MetricFilter.ACCEPT_ALL, filter); + } + + @Test + public void testIncludesOnly() { + Map props = new HashMap<>(); + props.put("metric-filter.includes", "table_optimizing_.*|optimizer_group_.*"); + + MetricFilter filter = MetricFilter.fromProperties(props); + assertTrue(filter.matches("table_optimizing_status_in_idle")); + assertTrue(filter.matches("optimizer_group_pending_tasks")); + assertFalse(filter.matches("table_summary_total_files")); + assertFalse(filter.matches("ams_jvm_cpu_load")); + } + + @Test + public void testExcludesOnly() { + Map props = new HashMap<>(); + props.put("metric-filter.excludes", "table_summary_.*"); + + MetricFilter filter = MetricFilter.fromProperties(props); + assertTrue(filter.matches("table_optimizing_status_in_idle")); + assertTrue(filter.matches("ams_jvm_cpu_load")); + assertFalse(filter.matches("table_summary_total_files")); + assertFalse(filter.matches("table_summary_health_score")); + } + + @Test + public void testIncludesAndExcludes() { + Map props = new HashMap<>(); + props.put("metric-filter.includes", "table_.*"); + props.put("metric-filter.excludes", "table_summary_.*"); + + MetricFilter filter = MetricFilter.fromProperties(props); + assertTrue(filter.matches("table_optimizing_status_in_idle")); + assertTrue(filter.matches("table_orphan_content_file_cleaning_count")); + assertFalse(filter.matches("table_summary_total_files")); + assertFalse(filter.matches("optimizer_group_pending_tasks")); + } + + @Test + public void testExcludesTakesPrecedenceOverIncludes() { + Map props = new HashMap<>(); + props.put("metric-filter.includes", ".*"); + props.put("metric-filter.excludes", "ams_jvm_.*"); + + MetricFilter filter = MetricFilter.fromProperties(props); + assertTrue(filter.matches("table_optimizing_status_in_idle")); + assertFalse(filter.matches("ams_jvm_cpu_load")); + assertFalse(filter.matches("ams_jvm_memory_heap_used")); + } + + @Test + public void testExactMatchPattern() { + Map props = new HashMap<>(); + props.put("metric-filter.includes", "ams_jvm_cpu_load"); + + MetricFilter filter = MetricFilter.fromProperties(props); + assertTrue(filter.matches("ams_jvm_cpu_load")); + assertFalse(filter.matches("ams_jvm_cpu_time")); + } +} diff --git a/amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricsCollectorFilterTest.java b/amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricsCollectorRegexFilterTest.java similarity index 65% rename from amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricsCollectorFilterTest.java rename to amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricsCollectorRegexFilterTest.java index 705f02453a..f50ff503b5 100644 --- a/amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricsCollectorFilterTest.java +++ b/amoro-metrics/amoro-metrics-prometheus/src/test/java/org/apache/amoro/metrics/promethues/MetricsCollectorRegexFilterTest.java @@ -32,39 +32,29 @@ import org.junit.jupiter.api.Test; import java.util.Collections; -import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Pattern; import java.util.stream.Collectors; -public class MetricsCollectorFilterTest { +public class MetricsCollectorRegexFilterTest { private MetricSet createMetricSet(Map metrics) { return () -> Collections.unmodifiableMap(metrics); } - private MetricKey registerMetric( - Map metrics, String name, Metric metric, String... tags) { + private void registerMetric(Map metrics, String name, Metric metric) { MetricDefine.Builder builder; if (metric instanceof Counter) { builder = MetricDefine.defineCounter(name); } else { builder = MetricDefine.defineGauge(name); } - if (tags.length > 0) { - builder.withTags(tags); - } MetricDefine define = builder.build(); - - Map tagValues = new HashMap<>(); - for (String tag : tags) { - tagValues.put(tag, "test_value"); - } - MetricKey key = new MetricKey(define, tagValues); + MetricKey key = new MetricKey(define, Collections.emptyMap()); metrics.put(key, metric); - return key; } @Test @@ -81,75 +71,67 @@ public void testCollectWithNoFilter() { } @Test - public void testCollectWithDisabledCategory() { + public void testCollectWithIncludesFilter() { Map metrics = new HashMap<>(); registerMetric(metrics, "table_optimizing_status_in_idle", (Gauge) () -> 1L); registerMetric(metrics, "optimizer_group_pending_tasks", (Gauge) () -> 5L); registerMetric(metrics, "ams_jvm_cpu_load", (Gauge) () -> 0.5); - Set disabled = EnumSet.of(MetricCategory.SELF_OPTIMIZING); - MetricsCollector collector = new MetricsCollector(createMetricSet(metrics), disabled); + MetricFilter filter = new MetricFilter(Pattern.compile("table_optimizing_.*"), null); + MetricsCollector collector = new MetricsCollector(createMetricSet(metrics), filter); List result = collector.collect(); - assertEquals(2, result.size()); - Set names = result.stream().map(s -> s.name).collect(Collectors.toSet()); - assertFalse(names.contains("amoro_table_optimizing_status_in_idle")); - assertTrue(names.contains("amoro_optimizer_group_pending_tasks")); - assertTrue(names.contains("amoro_ams_jvm_cpu_load")); + assertEquals(1, result.size()); + assertEquals("amoro_table_optimizing_status_in_idle", result.get(0).name); } @Test - public void testCollectWithMultipleDisabledCategories() { + public void testCollectWithExcludesFilter() { Map metrics = new HashMap<>(); registerMetric(metrics, "table_optimizing_status_in_idle", (Gauge) () -> 1L); registerMetric(metrics, "optimizer_group_pending_tasks", (Gauge) () -> 5L); - registerMetric(metrics, "ams_jvm_cpu_load", (Gauge) () -> 0.5); registerMetric(metrics, "table_summary_total_files", (Gauge) () -> 100L); - Set disabled = - EnumSet.of(MetricCategory.SELF_OPTIMIZING, MetricCategory.TABLE_SUMMARY); - MetricsCollector collector = new MetricsCollector(createMetricSet(metrics), disabled); + MetricFilter filter = new MetricFilter(null, Pattern.compile("table_summary_.*")); + MetricsCollector collector = new MetricsCollector(createMetricSet(metrics), filter); List result = collector.collect(); assertEquals(2, result.size()); Set names = result.stream().map(s -> s.name).collect(Collectors.toSet()); + assertFalse(names.contains("amoro_table_summary_total_files")); + assertTrue(names.contains("amoro_table_optimizing_status_in_idle")); assertTrue(names.contains("amoro_optimizer_group_pending_tasks")); - assertTrue(names.contains("amoro_ams_jvm_cpu_load")); } @Test - public void testUnknownMetricNotFiltered() { + public void testCollectWithIncludesAndExcludes() { Map metrics = new HashMap<>(); - registerMetric(metrics, "custom_unknown_metric", (Gauge) () -> 42L); - - Set disabled = - EnumSet.of( - MetricCategory.SELF_OPTIMIZING, - MetricCategory.OPTIMIZER_GROUP, - MetricCategory.ORPHAN_FILES, - MetricCategory.AMS_JVM, - MetricCategory.TABLE_SUMMARY); - MetricsCollector collector = new MetricsCollector(createMetricSet(metrics), disabled); + registerMetric(metrics, "table_optimizing_status_in_idle", (Gauge) () -> 1L); + registerMetric(metrics, "table_summary_total_files", (Gauge) () -> 100L); + registerMetric(metrics, "table_orphan_content_file_cleaning_count", new Counter()); + registerMetric(metrics, "optimizer_group_pending_tasks", (Gauge) () -> 5L); + + MetricFilter filter = + new MetricFilter(Pattern.compile("table_.*"), Pattern.compile("table_summary_.*")); + MetricsCollector collector = new MetricsCollector(createMetricSet(metrics), filter); List result = collector.collect(); - assertEquals(1, result.size()); - assertEquals("amoro_custom_unknown_metric", result.get(0).name); + assertEquals(2, result.size()); + Set names = result.stream().map(s -> s.name).collect(Collectors.toSet()); + assertTrue(names.contains("amoro_table_optimizing_status_in_idle")); + assertTrue(names.contains("amoro_table_orphan_content_file_cleaning_count")); } @Test - public void testOrphanFilesMultiplePrefixes() { + public void testCollectWithAcceptAllFilter() { Map metrics = new HashMap<>(); - Counter counter1 = new Counter(); - Counter counter2 = new Counter(); - registerMetric(metrics, "table_orphan_content_file_cleaning_count", counter1); - registerMetric(metrics, "table_expected_orphan_metadata_file_cleaning_count", counter2); - registerMetric(metrics, "ams_jvm_cpu_load", (Gauge) () -> 0.5); + registerMetric(metrics, "table_optimizing_status_in_idle", (Gauge) () -> 1L); + registerMetric(metrics, "custom_metric", (Gauge) () -> 42L); - Set disabled = EnumSet.of(MetricCategory.ORPHAN_FILES); - MetricsCollector collector = new MetricsCollector(createMetricSet(metrics), disabled); + MetricsCollector collector = + new MetricsCollector(createMetricSet(metrics), MetricFilter.ACCEPT_ALL); List result = collector.collect(); - assertEquals(1, result.size()); - assertEquals("amoro_ams_jvm_cpu_load", result.get(0).name); + assertEquals(2, result.size()); } } diff --git a/charts/amoro/tests/amoro-configmap_test.yaml b/charts/amoro/tests/amoro-configmap_test.yaml index 70f2948865..b6891052a9 100644 --- a/charts/amoro/tests/amoro-configmap_test.yaml +++ b/charts/amoro/tests/amoro-configmap_test.yaml @@ -152,7 +152,7 @@ tests: pattern: | name: prometheus-exporter - - it: Amoro configMap should reflect metric category filtering + - it: Amoro configMap should reflect metric regex filtering set: plugin: metricReporters: @@ -161,9 +161,8 @@ tests: enabled: true properties: port: 7001 - category.self-optimizing.enabled: "true" - category.table-summary.enabled: "false" + metric-filter.excludes: "table_summary_.*" asserts: - matchRegex: path: data["metric-reporters.yaml"] - pattern: "category.table-summary.enabled.*false" \ No newline at end of file + pattern: "metric-filter.excludes.*table_summary_" \ No newline at end of file diff --git a/charts/amoro/values.yaml b/charts/amoro/values.yaml index 39aa7d0953..9334977b0d 100644 --- a/charts/amoro/values.yaml +++ b/charts/amoro/values.yaml @@ -302,13 +302,10 @@ plugin: # ## @param Prometheus port # ## # port: 7001 -# ## @param Metric category filtering (all enabled by default) +# ## @param Metric filtering by regex (optional, all metrics included by default) # ## -# category.self-optimizing.enabled: "true" -# category.optimizer-group.enabled: "true" -# category.orphan-files.enabled: "true" -# category.ams-jvm.enabled: "true" -# category.table-summary.enabled: "true" +# metric-filter.includes: "table_optimizing_.*|optimizer_group_.*" +# metric-filter.excludes: "table_summary_.*" ## Configure the ingress resource that allows you to access the ## Amoro installation. Set up the URL diff --git a/dist/src/main/amoro-bin/conf/plugins/metric-reporters.yaml b/dist/src/main/amoro-bin/conf/plugins/metric-reporters.yaml index 043c315902..07dd778af8 100644 --- a/dist/src/main/amoro-bin/conf/plugins/metric-reporters.yaml +++ b/dist/src/main/amoro-bin/conf/plugins/metric-reporters.yaml @@ -23,10 +23,7 @@ metric-reporters: # enabled: false # properties: # port: 7001 -# # Metric category filtering (all enabled by default) -# # category.self-optimizing.enabled: "true" -# # category.optimizer-group.enabled: "true" -# # category.orphan-files.enabled: "true" -# # category.ams-jvm.enabled: "true" -# # category.table-summary.enabled: "true" +# # Metric filtering by regex (optional, all metrics included by default) +# # metric-filter.includes: "table_optimizing_.*|optimizer_group_.*" +# # metric-filter.excludes: "table_summary_.*"