From a6517ff6b3191492520dafd959bb71251b69542f Mon Sep 17 00:00:00 2001 From: YxYL Date: Wed, 18 Jun 2025 17:32:16 +0800 Subject: [PATCH 1/6] [feature:redis-slowlog] add redis slow query monitor; update doc for slow query in both EN and CN --- .../collect/redis/RedisCommonCollectImpl.java | 102 +++++++++++++++++- .../src/main/resources/define/app-redis.yml | 64 ++++++++++- home/docs/help/redis.md | 16 +++ .../current/help/redis.md | 18 +++- .../version-v1.6.x/help/redis.md | 18 +++- .../version-v1.6.x/help/redis.md | 18 +++- 6 files changed, 229 insertions(+), 7 deletions(-) diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/redis/RedisCommonCollectImpl.java b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/redis/RedisCommonCollectImpl.java index 90b258545db..ffaf3563b00 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/redis/RedisCommonCollectImpl.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/redis/RedisCommonCollectImpl.java @@ -38,6 +38,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.collector.collect.AbstractCollect; import org.apache.hertzbeat.collector.collect.common.cache.AbstractConnection; @@ -71,6 +72,8 @@ public class RedisCommonCollectImpl extends AbstractCollect { private static final String UNIQUE_IDENTITY = "identity"; + private static final String SLOW_LOG = "slowlog"; + private final ClientResources defaultClientResources; private final GlobalConnectionCache connectionCache = GlobalConnectionCache.getInstance(); @@ -90,7 +93,15 @@ public void preCheck(Metrics metrics) throws IllegalArgumentException{ @Override public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) { try { - if (Objects.nonNull(metrics.getRedis().getPattern()) && Objects.equals(metrics.getRedis().getPattern(), CLUSTER)) { + if (Objects.equals(metrics.getName(), SLOW_LOG)) { + if (Objects.nonNull(metrics.getRedis().getPattern()) && Objects.equals(metrics.getRedis().getPattern(), CLUSTER)) { + List> redisSlowLogList = getClusterRedisSlowLog(metrics); + doMetricsDataList(builder, redisSlowLogList, metrics); + } else { + List> redisSlowLog = getSingleRedisSlowLog(metrics); + doMetricsDataList(builder, redisSlowLog, metrics); + } + } else if (Objects.nonNull(metrics.getRedis().getPattern()) && Objects.equals(metrics.getRedis().getPattern(), CLUSTER)) { List> redisInfoList = getClusterRedisInfo(metrics); doMetricsDataList(builder, redisInfoList, metrics); } else { @@ -371,4 +382,93 @@ public String supportProtocol() { return DispatchConstants.PROTOCOL_REDIS; } + /** + * Get slow log data from a single Redis instance + * @param metrics metrics config + * @return data + */ + private List> getSingleRedisSlowLog(Metrics metrics) throws GeneralSecurityException, IOException { + StatefulRedisConnection connection = getSingleConnection(metrics.getRedis()); + // Get the last 100 slow logs by default + List slowLogs = connection.sync().slowlogGet(100); + + return parseSlowLogs(slowLogs); + } + + /** + * Get slow log data from Redis cluster nodes + * @param metrics metrics config + * @return data + */ + private List> getClusterRedisSlowLog(Metrics metrics) throws GeneralSecurityException, IOException { + Map> connectionMap = getConnectionList(metrics.getRedis()); + List> allSlowLogs = new ArrayList<>(); + + connectionMap.forEach((identity, connection) -> { + // Get the last 100 slow logs by default + List slowLogs = connection.sync().slowlogGet(100); + List> parsedLogs = parseSlowLogs(slowLogs); + + // Add node identity to each log entry + parsedLogs.forEach(log -> log.put(UNIQUE_IDENTITY, identity)); + + allSlowLogs.addAll(parsedLogs); + }); + + return allSlowLogs; + } + + /** + * Parse Redis slow log response into a list of maps + * @param slowLogs raw slow log data from Redis + * @return parsed slow logs + */ + private List> parseSlowLogs(List slowLogs) { + List> result = new ArrayList<>(); + + if (Objects.isNull(slowLogs) || slowLogs.isEmpty()) { + return result; + } + + for (Object entry : slowLogs) { + if (entry instanceof List) { + List logEntry = (List) entry; + if (logEntry.size() >= 4) { + Map logMap = new HashMap<>(); + + // ID + logMap.put("id", String.valueOf(logEntry.get(0))); + + // Timestamp + logMap.put("timestamp", String.valueOf(logEntry.get(1))); + + // Execution time in microseconds + logMap.put("execution_time", String.valueOf(logEntry.get(2))); + + // Command and arguments + if (logEntry.get(3) instanceof List) { + List cmdArgs = (List) logEntry.get(3); + String command = cmdArgs.stream() + .map(String::valueOf) + .collect(Collectors.joining(" ")); + logMap.put("command", command); + } + + // Client IP and name (if available) + if (logEntry.size() > 4 && logEntry.get(4) instanceof String) { + logMap.put("client_ip", String.valueOf(logEntry.get(4))); + } + + if (logEntry.size() > 5 && logEntry.get(5) instanceof String) { + logMap.put("client_name", String.valueOf(logEntry.get(5))); + } + + result.add(logMap); + } + } + } + + return result; + } + } diff --git a/hertzbeat-manager/src/main/resources/define/app-redis.yml b/hertzbeat-manager/src/main/resources/define/app-redis.yml index 9799b9ce51d..7508be5ea04 100644 --- a/hertzbeat-manager/src/main/resources/define/app-redis.yml +++ b/hertzbeat-manager/src/main/resources/define/app-redis.yml @@ -23,9 +23,9 @@ name: en-US: Redis # The description and help of this monitoring type help: - zh-CN: HertzBeat 对 REDIS 数据库的通用性能指标进行采集监控(server、clients、memory、persistence、stats、replication、cpu、errorstats、cluster、commandstats)，支持版本为 REDIS1.0+。
您可以点击“新建 REDIS 数据库”并进行配置，或者选择“更多操作”，导入已有配置。 - en-US: HertzBeat monitors REDIS database of general performance metrics such as memory, persistence, replication and so on. The versions we support is REDIS1.0+.
You could click the "New REDIS" button and proceed with the configuration or import an existing setup through the "More Actions" menu. - zh-TW: HertzBeat 對 REDIS 數據庫的通用性能指標進行采集監控(server、clients、memory、persistence、stats、replication、cpu、errorstats、cluster、commandstats)，支持版本爲 REDIS1.0+。
您可以點擊“新建 REDIS 數據庫”並進行配置，或者選擇“更多操作”，導入已有配置。 + zh-CN: HertzBeat 对 REDIS 数据库的通用性能指标进行采集监控(server、clients、memory、persistence、stats、replication、cpu、errorstats、cluster、commandstats、slowlog)，支持版本为 REDIS1.0+。
您可以点击"新建 REDIS 数据库"并进行配置，或者选择"更多操作"，导入已有配置。 + en-US: HertzBeat monitors REDIS database of general performance metrics such as memory, persistence, replication, slow query logs and so on. The versions we support is REDIS1.0+.
You could click the "New REDIS" button and proceed with the configuration or import an existing setup through the "More Actions" menu. + zh-TW: HertzBeat 對 REDIS 數據庫的通用性能指標進行采集監控(server、clients、memory、persistence、stats、replication、cpu、errorstats、cluster、commandstats、slowlog)，支持版本爲 REDIS1.0+。
您可以點擊"新建 REDIS 數據庫"並進行配置，或者選擇"更多操作"，導入已有配置。 helpLink: zh-CN: https://hertzbeat.apache.org/zh-cn/docs/help/redis en-US: https://hertzbeat.apache.org/docs/help/redis @@ -1449,3 +1449,61 @@ metrics: privateKey: ^_^sshPrivateKey^_^ privateKeyPassphrase: ^_^sshPrivateKeyPassphrase^_^ shareConnection: ^_^sshShareConnection^_^ + + # metrics - slowlog + - name: slowlog + # metrics scheduling priority(0->127)->(high->low), metrics with the same priority will be scheduled in parallel + priority: 9 + i18n: + zh-CN: 慢查询日志 + en-US: Slow Query Log + # collect metrics content + fields: + - field: id + type: 0 + i18n: + zh-CN: ID + en-US: ID + - field: timestamp + type: 0 + i18n: + zh-CN: 时间戳 + en-US: Timestamp + - field: execution_time + type: 0 + unit: μs + i18n: + zh-CN: 执行时间 + en-US: Execution Time + - field: command + type: 1 + i18n: + zh-CN: 命令 + en-US: Command + - field: client_ip + type: 1 + i18n: + zh-CN: 客户端IP + en-US: Client IP + - field: client_name + type: 1 + i18n: + zh-CN: 客户端名称 + en-US: Client Name + protocol: redis + redis: + host: ^_^host^_^ + port: ^_^port^_^ + username: ^_^username^_^ + password: ^_^password^_^ + timeout: ^_^timeout^_^ + sshTunnel: + enable: ^_^enableSshTunnel^_^ + host: ^_^sshHost^_^ + port: ^_^sshPort^_^ + timeout: ^_^sshTimeout^_^ + username: ^_^sshUsername^_^ + password: ^_^sshPassword^_^ + privateKey: ^_^sshPrivateKey^_^ + privateKeyPassphrase: ^_^sshPrivateKeyPassphrase^_^ + shareConnection: ^_^sshShareConnection^_^ diff --git a/home/docs/help/redis.md b/home/docs/help/redis.md index 2edbc47f482..ff4d9a9a59d 100644 --- a/home/docs/help/redis.md +++ b/home/docs/help/redis.md @@ -232,3 +232,19 @@ keywords: [ open source monitoring tool, open source Redis monitoring tool, moni | cmdstat_lpop | none | lpop command stat | | cmdstat_rpop | none | rpop command stat | | cmdstat_llen | none | llen command stat | + +#### Metric set：slowlog + +| Metric name | Metric unit | Metric help description | +|---------------------------|----------|-----------------------------------------------------------------------------------------------| +| id | none | Unique progressive identifier for every slow log entry | +| timestamp | none | Unix timestamp at which the logged command was processed | +| execution_time | μs | The amount of time needed for its execution, in microseconds | +| command | none | The command and its arguments | +| client_ip | none | Client IP address (available for Redis 4.0 or higher) | +| client_name | none | Client name if set via the CLIENT SETNAME command (available for Redis 4.0 or higher) | + +> Note: To configure Redis slow log settings, use the following Redis commands: +> - `CONFIG SET slowlog-log-slower-than 10000` - Log commands that take longer than 10,000 microseconds (10ms) +> - `CONFIG SET slowlog-max-len 128` - Keep the last 128 slow log entries +> - The slow log metrics are displayed in real-time and updated automatically. diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/redis.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/redis.md index d34ac3ca56c..ec2cc344daf 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/redis.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/redis.md @@ -5,7 +5,7 @@ sidebar_label: Redis 数据库 keywords: [开源监控系统, 开源数据库监控, Redis数据库监控] --- -> 对REDIS数据库的通用性能指标进行采集监控。支持REDIS1.0+。 +> 对REDIS数据库的通用性能指标（包括慢查询日志）进行采集监控。支持REDIS1.0+。 ### 配置参数 @@ -237,3 +237,19 @@ keywords: [开源监控系统, 开源数据库监控, Redis数据库监控] | cmdstat_lpop | 无 | lpop命令的统计信息 | | cmdstat_rpop | 无 | rpop命令的统计信息 | | cmdstat_llen | 无 | llen命令的统计信息 | + +#### 指标集合：slowlog + +| 指标名称 | 指标单位 | 指标帮助描述 | +|---------------------------|----------|-----------------------------------------------------------------------------------------------| +| id | 无 | 慢查询日志的唯一递增标识符 | +| timestamp | 无 | 记录命令被处理时的Unix时间戳 | +| execution_time | μs | 命令执行所需的时间（微秒） | +| command | 无 | 执行的命令及其参数 | +| client_ip | 无 | 客户端IP地址（Redis 4.0或更高版本可用） | +| client_name | 无 | 客户端名称，如果通过CLIENT SETNAME命令设置（Redis 4.0或更高版本可用） | + +> 注意：要配置Redis慢查询日志设置，请使用以下Redis命令： +> - `CONFIG SET slowlog-log-slower-than 10000` - 记录执行时间超过10,000微秒（10毫秒）的命令 +> - `CONFIG SET slowlog-max-len 128` - 保留最近128条慢查询日志条目 +> - 慢查询日志指标会实时显示并自动更新。 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/version-v1.6.x/help/redis.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/version-v1.6.x/help/redis.md index d34ac3ca56c..ec2cc344daf 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/version-v1.6.x/help/redis.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/version-v1.6.x/help/redis.md @@ -5,7 +5,7 @@ sidebar_label: Redis 数据库 keywords: [开源监控系统, 开源数据库监控, Redis数据库监控] --- -> 对REDIS数据库的通用性能指标进行采集监控。支持REDIS1.0+。 +> 对REDIS数据库的通用性能指标（包括慢查询日志）进行采集监控。支持REDIS1.0+。 ### 配置参数 @@ -237,3 +237,19 @@ keywords: [开源监控系统, 开源数据库监控, Redis数据库监控] | cmdstat_lpop | 无 | lpop命令的统计信息 | | cmdstat_rpop | 无 | rpop命令的统计信息 | | cmdstat_llen | 无 | llen命令的统计信息 | + +#### 指标集合：slowlog + +| 指标名称 | 指标单位 | 指标帮助描述 | +|---------------------------|----------|-----------------------------------------------------------------------------------------------| +| id | 无 | 慢查询日志的唯一递增标识符 | +| timestamp | 无 | 记录命令被处理时的Unix时间戳 | +| execution_time | μs | 命令执行所需的时间（微秒） | +| command | 无 | 执行的命令及其参数 | +| client_ip | 无 | 客户端IP地址（Redis 4.0或更高版本可用） | +| client_name | 无 | 客户端名称，如果通过CLIENT SETNAME命令设置（Redis 4.0或更高版本可用） | + +> 注意：要配置Redis慢查询日志设置，请使用以下Redis命令： +> - `CONFIG SET slowlog-log-slower-than 10000` - 记录执行时间超过10,000微秒（10毫秒）的命令 +> - `CONFIG SET slowlog-max-len 128` - 保留最近128条慢查询日志条目 +> - 慢查询日志指标会实时显示并自动更新。 diff --git a/home/versioned_docs/version-v1.6.x/help/redis.md b/home/versioned_docs/version-v1.6.x/help/redis.md index 2edbc47f482..2ab7c3a086c 100644 --- a/home/versioned_docs/version-v1.6.x/help/redis.md +++ b/home/versioned_docs/version-v1.6.x/help/redis.md @@ -5,7 +5,7 @@ sidebar_label: REDIS keywords: [ open source monitoring tool, open source Redis monitoring tool, monitoring Redis metrics ] --- -> Collect and monitor the general performance Metrics of Redis database. Support REDIS1.0+. +> Collect and monitor the general performance Metrics of Redis database including slow query logs. Support REDIS1.0+. ### Configuration parameter @@ -232,3 +232,19 @@ keywords: [ open source monitoring tool, open source Redis monitoring tool, moni | cmdstat_lpop | none | lpop command stat | | cmdstat_rpop | none | rpop command stat | | cmdstat_llen | none | llen command stat | + +#### Metric set：slowlog + +| Metric name | Metric unit | Metric help description | +|---------------------------|----------|-----------------------------------------------------------------------------------------------| +| id | none | Unique progressive identifier for every slow log entry | +| timestamp | none | Unix timestamp at which the logged command was processed | +| execution_time | μs | The amount of time needed for its execution, in microseconds | +| command | none | The command and its arguments | +| client_ip | none | Client IP address (available for Redis 4.0 or higher) | +| client_name | none | Client name if set via the CLIENT SETNAME command (available for Redis 4.0 or higher) | + +> Note: To configure Redis slow log settings, use the following Redis commands: +> - `CONFIG SET slowlog-log-slower-than 10000` - Log commands that take longer than 10,000 microseconds (10ms) +> - `CONFIG SET slowlog-max-len 128` - Keep the last 128 slow log entries +> - The slow log metrics are displayed in real-time and updated automatically. From fc050f9db65523ab590b84e441e8f0206c31051f Mon Sep 17 00:00:00 2001 From: YxYL Date: Thu, 19 Jun 2025 09:55:53 +0800 Subject: [PATCH 2/6] [feature:redis-slowlog] fix doc format --- home/docs/help/redis.md | 1 + .../zh-cn/docusaurus-plugin-content-docs/current/help/redis.md | 1 + .../docusaurus-plugin-content-docs/version-v1.6.x/help/redis.md | 1 + home/versioned_docs/version-v1.6.x/help/redis.md | 1 + 4 files changed, 4 insertions(+) diff --git a/home/docs/help/redis.md b/home/docs/help/redis.md index ff4d9a9a59d..e02f691292e 100644 --- a/home/docs/help/redis.md +++ b/home/docs/help/redis.md @@ -245,6 +245,7 @@ keywords: [ open source monitoring tool, open source Redis monitoring tool, moni | client_name | none | Client name if set via the CLIENT SETNAME command (available for Redis 4.0 or higher) | > Note: To configure Redis slow log settings, use the following Redis commands: +> > - `CONFIG SET slowlog-log-slower-than 10000` - Log commands that take longer than 10,000 microseconds (10ms) > - `CONFIG SET slowlog-max-len 128` - Keep the last 128 slow log entries > - The slow log metrics are displayed in real-time and updated automatically. diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/redis.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/redis.md index ec2cc344daf..ea9312eac35 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/redis.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/redis.md @@ -250,6 +250,7 @@ keywords: [开源监控系统, 开源数据库监控, Redis数据库监控] | client_name | 无 | 客户端名称，如果通过CLIENT SETNAME命令设置（Redis 4.0或更高版本可用） | > 注意：要配置Redis慢查询日志设置，请使用以下Redis命令： +> > - `CONFIG SET slowlog-log-slower-than 10000` - 记录执行时间超过10,000微秒（10毫秒）的命令 > - `CONFIG SET slowlog-max-len 128` - 保留最近128条慢查询日志条目 > - 慢查询日志指标会实时显示并自动更新。 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/version-v1.6.x/help/redis.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/version-v1.6.x/help/redis.md index ec2cc344daf..ea9312eac35 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/version-v1.6.x/help/redis.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/version-v1.6.x/help/redis.md @@ -250,6 +250,7 @@ keywords: [开源监控系统, 开源数据库监控, Redis数据库监控] | client_name | 无 | 客户端名称，如果通过CLIENT SETNAME命令设置（Redis 4.0或更高版本可用） | > 注意：要配置Redis慢查询日志设置，请使用以下Redis命令： +> > - `CONFIG SET slowlog-log-slower-than 10000` - 记录执行时间超过10,000微秒（10毫秒）的命令 > - `CONFIG SET slowlog-max-len 128` - 保留最近128条慢查询日志条目 > - 慢查询日志指标会实时显示并自动更新。 diff --git a/home/versioned_docs/version-v1.6.x/help/redis.md b/home/versioned_docs/version-v1.6.x/help/redis.md index 2ab7c3a086c..64f1cd40aff 100644 --- a/home/versioned_docs/version-v1.6.x/help/redis.md +++ b/home/versioned_docs/version-v1.6.x/help/redis.md @@ -245,6 +245,7 @@ keywords: [ open source monitoring tool, open source Redis monitoring tool, moni | client_name | none | Client name if set via the CLIENT SETNAME command (available for Redis 4.0 or higher) | > Note: To configure Redis slow log settings, use the following Redis commands: +> > - `CONFIG SET slowlog-log-slower-than 10000` - Log commands that take longer than 10,000 microseconds (10ms) > - `CONFIG SET slowlog-max-len 128` - Keep the last 128 slow log entries > - The slow log metrics are displayed in real-time and updated automatically. From 6dbe833501f943cebc4fdf00afc292af41225d11 Mon Sep 17 00:00:00 2001 From: YxYL Date: Thu, 19 Jun 2025 10:02:23 +0800 Subject: [PATCH 3/6] [feature:redis-slowlog] fix ut --- .../redis/RedisCommonCollectE2eTest.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/redis/RedisCommonCollectE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/redis/RedisCommonCollectE2eTest.java index 673cbb0d633..06c57bd80f1 100644 --- a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/redis/RedisCommonCollectE2eTest.java +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/redis/RedisCommonCollectE2eTest.java @@ -77,6 +77,9 @@ public void setUp() throws Exception { mappedPort = redisContainer.getMappedPort(REDIS_PORT); log.info("Container started successfully with mapped port: {}", mappedPort); + + // Configure Redis and generate slow queries for testing + generateSlowQueries(); } @Test @@ -123,4 +126,29 @@ public void setupAndStartContainer() { Startables.deepStart(Stream.of(redisContainer)).join(); } + + /** + * Generate slow queries for testing slowlog metrics + */ + private void generateSlowQueries() { + try { + // Set a low threshold for slow log to ensure our commands are captured + redisContainer.execInContainer("redis-cli", "CONFIG", "SET", "slowlog-log-slower-than", "0"); + redisContainer.execInContainer("redis-cli", "CONFIG", "SET", "slowlog-max-len", "128"); + + // Execute some commands that will be captured in the slow log + redisContainer.execInContainer("redis-cli", "SET", "test_key", "test_value"); + redisContainer.execInContainer("redis-cli", "GET", "test_key"); + redisContainer.execInContainer("redis-cli", "KEYS", "*"); + + // Execute a Lua script that will definitely be slow + String luaScript = "local i=0; while i<1000 do i=i+1 end; return i"; + redisContainer.execInContainer("redis-cli", "EVAL", luaScript, "0"); + + // Verify that we have slow log entries + log.info("Generated slow queries for testing"); + } catch (Exception e) { + log.error("Failed to generate slow queries: {}", e.getMessage(), e); + } + } } From 03dbbea3538cdf721b267985f610698d547f1adc Mon Sep 17 00:00:00 2001 From: YxYL Date: Thu, 19 Jun 2025 16:48:23 +0800 Subject: [PATCH 4/6] [feature:redis-slowlog] fix ut --- .../collect/basic/redis/RedisCommonCollectE2eTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/redis/RedisCommonCollectE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/redis/RedisCommonCollectE2eTest.java index 06c57bd80f1..18691bc350c 100644 --- a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/redis/RedisCommonCollectE2eTest.java +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/redis/RedisCommonCollectE2eTest.java @@ -54,7 +54,7 @@ public class RedisCommonCollectE2eTest extends AbstractCollectE2eTest { private static final String HOST = "127.0.0.1"; private static final int REDIS_PORT = 6379; private static final String REDIS_PATTERN = "1"; - private static final List ALLOW_EMPTY_WHITE_LIST = Arrays.asList("server", "errorstats", "commandstats", "keyspace"); + private static final List ALLOW_EMPTY_WHITE_LIST = Arrays.asList("server", "errorstats", "commandstats", "keyspace", "slowlog"); private static GenericContainer redisContainer; From 6a74662f5bac1f8bce5ee462ded164368c775c2e Mon Sep 17 00:00:00 2001 From: YxYL Date: Fri, 20 Jun 2025 10:43:14 +0800 Subject: [PATCH 5/6] [feature:redis-slowlog] fix e2e test --- .../collect/basic/redis/RedisCommonCollectE2eTest.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/redis/RedisCommonCollectE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/redis/RedisCommonCollectE2eTest.java index 18691bc350c..628d1abc019 100644 --- a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/redis/RedisCommonCollectE2eTest.java +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/redis/RedisCommonCollectE2eTest.java @@ -136,6 +136,9 @@ private void generateSlowQueries() { redisContainer.execInContainer("redis-cli", "CONFIG", "SET", "slowlog-log-slower-than", "0"); redisContainer.execInContainer("redis-cli", "CONFIG", "SET", "slowlog-max-len", "128"); + // Set client name for the connection to ensure client_name field is not empty + redisContainer.execInContainer("redis-cli", "CLIENT", "SETNAME", "test-client-name"); + // Execute some commands that will be captured in the slow log redisContainer.execInContainer("redis-cli", "SET", "test_key", "test_value"); redisContainer.execInContainer("redis-cli", "GET", "test_key"); @@ -145,6 +148,13 @@ private void generateSlowQueries() { String luaScript = "local i=0; while i<1000 do i=i+1 end; return i"; redisContainer.execInContainer("redis-cli", "EVAL", luaScript, "0"); + // Execute more commands to ensure we have enough entries with client names + for (int i = 0; i < 5; i++) { + redisContainer.execInContainer("redis-cli", "--no-auth-warning", "CLIENT", "SETNAME", "test-client-" + i); + redisContainer.execInContainer("redis-cli", "--no-auth-warning", "SET", "key" + i, "value" + i); + redisContainer.execInContainer("redis-cli", "--no-auth-warning", "GET", "key" + i); + } + // Verify that we have slow log entries log.info("Generated slow queries for testing"); } catch (Exception e) { From d0dbb4fdf1687066b48f784aaf35f5ee3fa7f2dd Mon Sep 17 00:00:00 2001 From: YxYL Date: Wed, 25 Jun 2025 16:20:54 +0800 Subject: [PATCH 6/6] [feature:redis-slowlog] add more log for e2e test --- .../redis/RedisCommonCollectE2eTest.java | 38 ------------------- .../collect/AbstractCollectE2eTest.java | 2 + 2 files changed, 2 insertions(+), 38 deletions(-) diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/redis/RedisCommonCollectE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/redis/RedisCommonCollectE2eTest.java index 628d1abc019..6aff9dd9208 100644 --- a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/redis/RedisCommonCollectE2eTest.java +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/redis/RedisCommonCollectE2eTest.java @@ -77,9 +77,6 @@ public void setUp() throws Exception { mappedPort = redisContainer.getMappedPort(REDIS_PORT); log.info("Container started successfully with mapped port: {}", mappedPort); - - // Configure Redis and generate slow queries for testing - generateSlowQueries(); } @Test @@ -126,39 +123,4 @@ public void setupAndStartContainer() { Startables.deepStart(Stream.of(redisContainer)).join(); } - - /** - * Generate slow queries for testing slowlog metrics - */ - private void generateSlowQueries() { - try { - // Set a low threshold for slow log to ensure our commands are captured - redisContainer.execInContainer("redis-cli", "CONFIG", "SET", "slowlog-log-slower-than", "0"); - redisContainer.execInContainer("redis-cli", "CONFIG", "SET", "slowlog-max-len", "128"); - - // Set client name for the connection to ensure client_name field is not empty - redisContainer.execInContainer("redis-cli", "CLIENT", "SETNAME", "test-client-name"); - - // Execute some commands that will be captured in the slow log - redisContainer.execInContainer("redis-cli", "SET", "test_key", "test_value"); - redisContainer.execInContainer("redis-cli", "GET", "test_key"); - redisContainer.execInContainer("redis-cli", "KEYS", "*"); - - // Execute a Lua script that will definitely be slow - String luaScript = "local i=0; while i<1000 do i=i+1 end; return i"; - redisContainer.execInContainer("redis-cli", "EVAL", luaScript, "0"); - - // Execute more commands to ensure we have enough entries with client names - for (int i = 0; i < 5; i++) { - redisContainer.execInContainer("redis-cli", "--no-auth-warning", "CLIENT", "SETNAME", "test-client-" + i); - redisContainer.execInContainer("redis-cli", "--no-auth-warning", "SET", "key" + i, "value" + i); - redisContainer.execInContainer("redis-cli", "--no-auth-warning", "GET", "key" + i); - } - - // Verify that we have slow log entries - log.info("Generated slow queries for testing"); - } catch (Exception e) { - log.error("Failed to generate slow queries: {}", e.getMessage(), e); - } - } } diff --git a/hertzbeat-e2e/hertzbeat-collector-common-e2e/src/test/java/org/apache/hertzbeat/collector/collect/AbstractCollectE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-common-e2e/src/test/java/org/apache/hertzbeat/collector/collect/AbstractCollectE2eTest.java index 7a7aa174203..7f1e42488c9 100644 --- a/hertzbeat-e2e/hertzbeat-collector-common-e2e/src/test/java/org/apache/hertzbeat/collector/collect/AbstractCollectE2eTest.java +++ b/hertzbeat-e2e/hertzbeat-collector-common-e2e/src/test/java/org/apache/hertzbeat/collector/collect/AbstractCollectE2eTest.java @@ -110,7 +110,9 @@ protected CollectRep.MetricsData validateMetricsCollection(Metrics metricsDef, S String.format("%s metrics values should not be empty, detail: %s", metricName, metricsData.getMsg())); for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) { + log.info("valueRow: {}", valueRow); for (int i = 0; i < valueRow.getColumnsCount(); i++) { + log.info("valueRow.getColumns(i): {}", valueRow.getColumns(i)); Assertions.assertFalse(valueRow.getColumns(i).isEmpty(), String.format("%s metric column %d should not be empty", metricName, i)); if (!allowEmpty) {